From c66df1c0c21542597d219c2c12ea7522a38a043c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 27 Nov 2014 10:49:42 -0800 Subject: [PATCH 001/235] Issue #56 Pull in the Lua 5.2 os.date() fixes --- cmake/lua-5_1_5.patch | 241 +++++++++++++++++++++++++------- src/test/lua/lpeg_date_time.lua | 2 +- 2 files changed, 189 insertions(+), 54 deletions(-) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch index abc703b..d72b89a 100644 --- a/cmake/lua-5_1_5.patch +++ b/cmake/lua-5_1_5.patch @@ -1,7 +1,7 @@ -diff -Naur lua-5.1.5/CMakeLists.txt Source/lua-5_1_5/CMakeLists.txt ---- lua-5.1.5/CMakeLists.txt Wed Dec 31 16:00:00 1969 -+++ Source/lua-5_1_5/CMakeLists.txt Wed May 21 13:16:57 2014 -@@ -0,0 +1,62 @@ +diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt +--- /work/lua-5.1.5/CMakeLists.txt 1969-12-31 16:00:00.000000000 -0800 ++++ lua-5_1_5/CMakeLists.txt 2014-11-27 10:36:18.352979806 -0800 +@@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -44,6 +44,7 @@ diff -Naur lua-5.1.5/CMakeLists.txt Source/lua-5_1_5/CMakeLists.txt +set(CMAKE_INSTALL_PREFIX ${EP_BASE}) +if (WIN32) + add_library(lua SHARED ${LUA_SOURCE}) ++ add_definitions(-DLUA_WIN) + add_definitions(-DLUA_BUILD_AS_DLL) + if (MINGW) + set_target_properties(lua PROPERTIES LINK_FLAGS -s) @@ -55,82 +56,216 @@ diff -Naur lua-5.1.5/CMakeLists.txt Source/lua-5_1_5/CMakeLists.txt +elseif(APPLE) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + add_library(lua STATIC ${LUA_SOURCE}) -+ add_definitions(-DLUA_USE_LINUX) ++ add_definitions(-DLUA_USE_MACOSX) + target_link_libraries(lua -lreadline) +else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + add_library(lua STATIC ${LUA_SOURCE}) ++ add_definitions(-DLUA_USE_LINUX) +endif() + +install(TARGETS lua DESTINATION lib) +install(FILES src/lua.h src/luaconf.h src/lualib.h src/lauxlib.h DESTINATION include) -diff -Naur lua-5.1.5/src/ldebug.c Source/lua-5_1_5/src/ldebug.c ---- lua-5.1.5/src/ldebug.c Thu May 8 09:56:26 2008 -+++ Source/lua-5_1_5/src/ldebug.c Wed May 21 13:16:58 2014 +diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c +--- /work/lua-5.1.5/src/ldebug.c 2008-05-08 09:56:26.000000000 -0700 ++++ lua-5_1_5/src/ldebug.c 2014-11-27 10:32:27.052988495 -0800 @@ -80,6 +80,10 @@ return L->basehookcount; } - + +LUA_API int lua_gethookcountremaining (lua_State *L) { + return L->hookcount; +} + - + LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar) { int status; -diff -Naur lua-5.1.5/src/loslib.c Source/lua-5_1_5/src/loslib.c ---- lua-5.1.5/src/loslib.c Fri Jan 18 08:38:18 2008 -+++ Source/lua-5_1_5/src/loslib.c Wed May 21 13:16:58 2014 -@@ -154,9 +154,40 @@ - if (*s != '%' || *(s + 1) == '\0') /* no conversion specifier? */ - luaL_addchar(&b, *s); - else { -+ cc[1] = *(++s); -+#ifdef _WIN32 -+ int valid = 0; -+ switch (*s) { -+ case 'a': -+ case 'A': -+ case 'b': -+ case 'B': -+ case 'c': -+ case 'd': -+ case 'H': -+ case 'I': -+ case 'j': -+ case 'm': -+ case 'M': -+ case 'p': -+ case 'S': -+ case 'U': -+ case 'w': -+ case 'W': -+ case 'x': -+ case 'X': -+ case 'y': -+ case 'Y': -+ case 'Z': -+ case 'z': -+ case '%': -+ valid = 1; -+ break; -+ } -+ if (!valid) continue; +diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c +--- /work/lua-5.1.5/src/loslib.c 2008-01-18 08:38:18.000000000 -0800 ++++ lua-5_1_5/src/loslib.c 2014-11-27 10:32:27.052988495 -0800 +@@ -33,6 +33,61 @@ + return 3; + } + } ++/* ++** list of valid conversion specifiers for the 'strftime' function ++*/ ++#if !defined(LUA_STRFTIMEOPTIONS) ++ ++#if !defined(LUA_USE_POSIX) ++#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYz%", "" } ++#else ++#define LUA_STRFTIMEOPTIONS \ ++ { "aAbBcCdDeFgGhHIjklmMnprRSstTuUVwWxXyYzZ%", "" \ ++ "", "E", "cCxXyY", \ ++ "O", "deHImMSuUVwWy" } ++#endif ++ +#endif ++ ++ ++ ++/* ++** By default, Lua uses tmpnam except when POSIX is available, where it ++** uses mkstemp. ++*/ ++#if defined(LUA_USE_MKSTEMP) ++#include ++#define LUA_TMPNAMBUFSIZE 32 ++#define lua_tmpnam(b,e) { \ ++ strcpy(b, "/tmp/lua_XXXXXX"); \ ++ e = mkstemp(b); \ ++ if (e != -1) close(e); \ ++ e = (e == -1); } ++ ++#elif !defined(lua_tmpnam) ++ ++#define LUA_TMPNAMBUFSIZE L_tmpnam ++#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } ++ ++#endif ++ ++ ++/* ++** By default, Lua uses gmtime/localtime, except when POSIX is available, ++** where it uses gmtime_r/localtime_r ++*/ ++#if defined(LUA_USE_GMTIME_R) ++ ++#define l_gmtime(t,r) gmtime_r(t,r) ++#define l_localtime(t,r) localtime_r(t,r) ++ ++#elif !defined(l_gmtime) ++ ++#define l_gmtime(t,r) ((void)r, gmtime(t)) ++#define l_localtime(t,r) ((void)r, localtime(t)) ++ ++#endif ++ + + + static int os_execute (lua_State *L) { +@@ -121,16 +176,40 @@ + } + + ++static const char *checkoption (lua_State *L, const char *conv, char *buff) { ++ static const char *const options[] = LUA_STRFTIMEOPTIONS; ++ unsigned int i; ++ for (i = 0; i < sizeof(options)/sizeof(options[0]); i += 2) { ++ if (*conv != '\0' && strchr(options[i], *conv) != NULL) { ++ buff[1] = *conv; ++ if (*options[i + 1] == '\0') { /* one-char conversion specifier? */ ++ buff[2] = '\0'; /* end buffer */ ++ return conv + 1; ++ } ++ else if (*(conv + 1) != '\0' && ++ strchr(options[i + 1], *(conv + 1)) != NULL) { ++ buff[2] = *(conv + 1); /* valid two-char conversion specifier */ ++ buff[3] = '\0'; /* end buffer */ ++ return conv + 2; ++ } ++ } ++ } ++ luaL_argerror(L, 1, ++ lua_pushfstring(L, "invalid conversion specifier '%%%s'", conv)); ++ return conv; /* to avoid warnings */ ++} ++ ++ + static int os_date (lua_State *L) { + const char *s = luaL_optstring(L, 1, "%c"); + time_t t = luaL_opt(L, (time_t)luaL_checknumber, 2, time(NULL)); +- struct tm *stm; ++ struct tm tmr, *stm; + if (*s == '!') { /* UTC? */ +- stm = gmtime(&t); ++ stm = l_gmtime(&t, &tmr); + s++; /* skip `!' */ + } + else +- stm = localtime(&t); ++ stm = l_localtime(&t, &tmr); + if (stm == NULL) /* invalid date? */ + lua_pushnil(L); + else if (strcmp(s, "*t") == 0) { +@@ -146,17 +225,17 @@ + setboolfield(L, "isdst", stm->tm_isdst); + } + else { +- char cc[3]; ++ char cc[4]; + luaL_Buffer b; +- cc[0] = '%'; cc[2] = '\0'; ++ cc[0] = '%'; + luaL_buffinit(L, &b); +- for (; *s; s++) { +- if (*s != '%' || *(s + 1) == '\0') /* no conversion specifier? */ +- luaL_addchar(&b, *s); ++ while (*s) { ++ if (*s != '%') /* no conversion specifier? */ ++ luaL_addchar(&b, *s++); + else { size_t reslen; char buff[200]; /* should be big enough for any conversion result */ - cc[1] = *(++s); ++ s = checkoption(L, s + 1, cc); reslen = strftime(buff, sizeof(buff), cc, stm); luaL_addlstring(&b, buff, reslen); } -diff -Naur lua-5.1.5/src/lua.h Source/lua-5_1_5/src/lua.h ---- lua-5.1.5/src/lua.h Fri Jan 13 12:36:20 2012 -+++ Source/lua-5_1_5/src/lua.h Wed May 21 13:16:58 2014 +diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h +--- /work/lua-5.1.5/src/luaconf.h 2008-02-11 08:25:08.000000000 -0800 ++++ lua-5_1_5/src/luaconf.h 2014-11-27 10:32:27.052988495 -0800 +@@ -1,5 +1,5 @@ + /* +-** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ ++** $Id: luaconf.h,v 1.176.1.1 2013/04/12 18:48:47 roberto Exp $ + ** Configuration file for Lua + ** See Copyright Notice in lua.h + */ +@@ -33,14 +33,26 @@ + #define LUA_WIN + #endif + ++#if defined(LUA_WIN) ++#define LUA_DL_DLL ++#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ ++#endif ++ + #if defined(LUA_USE_LINUX) + #define LUA_USE_POSIX + #define LUA_USE_DLOPEN /* needs an extra library: -ldl */ + #define LUA_USE_READLINE /* needs some extra libraries */ ++#define LUA_USE_STRTODHEX /* assume 'strtod' handles hex formats */ ++#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ ++#define LUA_USE_LONGLONG /* assume support for long long */ + #endif + + #if defined(LUA_USE_MACOSX) + #define LUA_USE_POSIX ++#define LUA_USE_DLOPEN /* does not need -ldl */ ++#define LUA_USE_READLINE /* needs an extra library: -lreadline */ ++#define LUA_USE_STRTODHEX /* assume 'strtod' handles hex formats */ ++#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ + #define LUA_DL_DYLD /* does not need extra library */ + #endif + +@@ -56,6 +68,7 @@ + #define LUA_USE_ISATTY + #define LUA_USE_POPEN + #define LUA_USE_ULONGJMP ++#define LUA_USE_GMTIME_R + #endif + + +diff -Naur /work/lua-5.1.5/src/lua.h lua-5_1_5/src/lua.h +--- /work/lua-5.1.5/src/lua.h 2012-01-13 12:36:20.000000000 -0800 ++++ lua-5_1_5/src/lua.h 2014-11-27 10:32:27.052988495 -0800 @@ -341,6 +341,7 @@ LUA_API lua_Hook lua_gethook (lua_State *L); LUA_API int lua_gethookmask (lua_State *L); LUA_API int lua_gethookcount (lua_State *L); +LUA_API int lua_gethookcountremaining (lua_State *L); - - + + struct lua_Debug { diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua index 4da4b2e..62eed68 100644 --- a/src/test/lua/lpeg_date_time.lua +++ b/src/test/lua/lpeg_date_time.lua @@ -87,7 +87,7 @@ local function strftime_all() "%F", "%g", "%G", "%h", "%H", "%I", "%j", "%k", "%l", "%m", "%M", "%n", "%p", "%r", "%R", "%s", "%S", "%t", "%T", "%u", "%U", "%V", "%w", "%W", "%x","%X", "%y", "%Y", "%z", "%Z", - "%%", "test string", "%~"} + "%%", "test string"} end for i,v in ipairs(formats) do From a1988728246c9d9b827fa0419bc411a9b721d3da Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 1 Dec 2014 07:58:29 -0800 Subject: [PATCH 002/235] Add %Z into the Windows strftime specifiers --- cmake/lua-5_1_5.patch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch index d72b89a..56ee374 100644 --- a/cmake/lua-5_1_5.patch +++ b/cmake/lua-5_1_5.patch @@ -93,7 +93,7 @@ diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c +#if !defined(LUA_STRFTIMEOPTIONS) + +#if !defined(LUA_USE_POSIX) -+#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYz%", "" } ++#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYzZ%", "" } +#else +#define LUA_STRFTIMEOPTIONS \ + { "aAbBcCdDeFgGhHIjklmMnprRSstTuUVwWxXyYzZ%", "" \ From c7e84c0a4a633126b5802859697c1355a5c75aa8 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 8 Dec 2014 11:05:33 -0800 Subject: [PATCH 003/235] Only add a metatable to libraries if one doesn't already exist --- src/lua_sandbox_private.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/lua_sandbox_private.c b/src/lua_sandbox_private.c index a15b0c5..4bb1393 100644 --- a/src/lua_sandbox_private.c +++ b/src/lua_sandbox_private.c @@ -37,6 +37,19 @@ const char* package_table = "package"; const char* loaded_table = "loaded"; +// If necessary, add an empty metatable to flag the table as non-data +// during preservation. +static void add_empty_metatable(lua_State* lua) +{ + if (lua_getmetatable(lua, -1) == 0) { + lua_newtable(lua); + lua_setmetatable(lua, -2); + } else { + lua_pop(lua, 1); + } +} + + void load_library(lua_State* lua, const char* table, lua_CFunction f, const char** disable) { @@ -53,10 +66,7 @@ void load_library(lua_State* lua, const char* table, lua_CFunction f, lua_pushnil(lua); lua_setfield(lua, -2, disable[i]); } - // Add an empty metatable to identify core libraries during - // preservation. - lua_newtable(lua); - lua_setmetatable(lua, -2); + add_empty_metatable(lua); } } @@ -327,9 +337,7 @@ int require_library(lua_State* lua) if (luaL_dofile(lua, fn) != 0) { return luaL_error(lua, "%s", lua_tostring(lua, -1)); } - // Add an empty metatable to identify the library during preservation. - lua_newtable(lua); - lua_setmetatable(lua, -2); + add_empty_metatable(lua); } lua_pushvalue(lua, -1); lua_setfield(lua, pos, name); From f96b4495929219a886c3611b0ac85391c6e07b9a Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 20:04:51 +0000 Subject: [PATCH 004/235] Add sfl4j.lua and tests --- modules/sfl4j.lua | 71 +++++++++++++++++++++++++++++++++++++ src/test/lua/lpeg_sfl4j.lua | 32 +++++++++++++++++ src/test/test_lua_sandbox.c | 1 + 3 files changed, 104 insertions(+) create mode 100644 modules/sfl4j.lua create mode 100644 src/test/lua/lpeg_sfl4j.lua diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua new file mode 100644 index 0000000..21c50b6 --- /dev/null +++ b/modules/sfl4j.lua @@ -0,0 +1,71 @@ +-- Using this as a reference: https://github.banksimple.com/gist/sjl/92b4b5750739a46d38a7 +-- + +-- imports +local l = require "lpeg" +local dt = require "dt" +local tonumber = tonumber + +l.locale(l) + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +local space = l.space^1 +local sep = l.P"\n" +local level = l.alpha^5 -- Log level (5 characters) +local brack = l.S("[]") -- [ and ] +local colon = l.S(":") -- : +local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue +local line = (l.P(1) - sep)^0 * sep + +-- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name +local logline = l.Cg(level, "Level") -- ERROR + * space + * brack + * l.Cg(timestamp, "Timestamp") -- [2014-11-21 16:35:59,501] + * brack + * space + * l.Cg(class, "class") -- com.domain.client.jobs.OutgoingQueue + * colon + * space + * l.Cg(line, "ErrorMessage") -- Error handling output... + +-- TODO: Multiline, need to figure out how to capture the entire stack in a table +-- -- Example: ! com.domain.inet.ftp.Exception: java.io.IOException: FAILED +-- local stackline = l.P"!" +-- * space +-- * line -- com.domain.inet.ftp.Exception +-- +-- -- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) +-- local stackatline = l.P"!" +-- * space +-- * l.P"at" +-- * space +-- * line -- com.domain.inet.ftp.Exception +-- +-- -- Matches entire line, use this to gather stuff we don't parse properly +-- local miscline = line + +-- local logevent = logline +-- * stackline +-- * stackatline +-- * miscline +-- * sep + +local slf4j_severity = l.digit^1 / tonumber + +local slf4j_log_levels_text = ( + (l.P"fatal" + "FATAL") / "6" ++ (l.P"error" + "ERROR") / "5" ++ (l.P"warn" + "WARN") / "4" ++ (l.P"info" + "INFO") / "3" ++ (l.P"debug" + "DEBUG") / "2" ++ (l.P"trace" + "TRACE") / "1" +) / tonumber + +-- TODO: Map slf4j to standard log levels for "Severity" + +logevent_grammar = l.Ct(logline) + +return M diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua new file mode 100644 index 0000000..faaf4b7 --- /dev/null +++ b/src/test/lua/lpeg_sfl4j.lua @@ -0,0 +1,32 @@ +local sfl4j = require("sfl4j") + +local test_log_events = { +[[ +ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name +]] +} + +local fields = { + {"Level", "ERROR"}, + {"Timestamp", ""}, + {"Class", "com.domain.client.jobs.OutgoingQueue"}, + {"ErrorMessage", "Error handling output file with job job-name"} +} + +local function validate(fields, t) + for i, v in ipairs(fields) do + end + return false +end + +local function mainlogevent() + local t = sfl4j.logevent_grammar:match(test_log_events[1]) + if not t then return error("no match") end + validate(fields, t) +end + +function process(tc) + mainlogevent() + + return 0 +end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 858a2a7..8d56a51 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -817,6 +817,7 @@ static char* test_lpeg() , "lua/lpeg_ip_address.lua" , "lua/lpeg_mysql.lua" , "lua/lpeg_syslog.lua" + , "lua/lpeg_sfl4j.lua" , NULL }; From 0e0b6f57089fbc4fdaa3373ff5d19e05c1626026 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 20:25:11 +0000 Subject: [PATCH 005/235] A fix and force it fail --- src/test/lua/lpeg_sfl4j.lua | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index faaf4b7..09e744d 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -21,12 +21,11 @@ end local function mainlogevent() local t = sfl4j.logevent_grammar:match(test_log_events[1]) - if not t then return error("no match") end validate(fields, t) end function process(tc) mainlogevent() - return 0 + return 1 end From b59f0c563e852d0967c333055aba4d232519a250 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 9 Dec 2014 12:51:43 -0800 Subject: [PATCH 006/235] Fix the unit test assertions. --- src/test/lua/lpeg.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/lua/lpeg.lua b/src/test/lua/lpeg.lua index 6120e4d..d9553ae 100644 --- a/src/test/lua/lpeg.lua +++ b/src/test/lua/lpeg.lua @@ -11,8 +11,8 @@ local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1) function process () local t = lpeg.match(record, '1,string with spaces,"quoted string, with comma and ""quoted"" text"') - assert(t[1], "1", t[1]) - assert(t[2], "string with spaces", t[2]) - assert(t[3], 'quoted string, with comma and "quoted" text', t[3]) + assert(t[1] == "1", t[1]) + assert(t[2] == "string with spaces", t[2]) + assert(t[3] == 'quoted string, with comma and "quoted" text', t[3]) return 0 end From 90496248a9700f69b61d3b797c5d69c96ff1e68f Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 21:00:25 +0000 Subject: [PATCH 007/235] Cleanup some errors --- modules/sfl4j.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 21c50b6..ad83d9a 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -3,7 +3,6 @@ -- imports local l = require "lpeg" -local dt = require "dt" local tonumber = tonumber l.locale(l) @@ -15,6 +14,7 @@ local space = l.space^1 local sep = l.P"\n" local level = l.alpha^5 -- Log level (5 characters) local brack = l.S("[]") -- [ and ] +local timestamp = (l.P(1) - brack)^1 local colon = l.S(":") -- : local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue local line = (l.P(1) - sep)^0 * sep @@ -23,7 +23,7 @@ local line = (l.P(1) - sep)^0 * sep local logline = l.Cg(level, "Level") -- ERROR * space * brack - * l.Cg(timestamp, "Timestamp") -- [2014-11-21 16:35:59,501] + * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 * brack * space * l.Cg(class, "class") -- com.domain.client.jobs.OutgoingQueue From bc8181a6d34e5b58ce140ac58ba4ad1a2e599cfe Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 21:00:40 +0000 Subject: [PATCH 008/235] Lets create better tests --- src/test/lua/lpeg_sfl4j.lua | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index 09e744d..ba6a4a2 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -1,11 +1,5 @@ local sfl4j = require("sfl4j") -local test_log_events = { -[[ -ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name -]] -} - local fields = { {"Level", "ERROR"}, {"Timestamp", ""}, @@ -13,19 +7,14 @@ local fields = { {"ErrorMessage", "Error handling output file with job job-name"} } -local function validate(fields, t) - for i, v in ipairs(fields) do - end - return false -end - -local function mainlogevent() - local t = sfl4j.logevent_grammar:match(test_log_events[1]) - validate(fields, t) +local function single_logevent() + local log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name' + local fields = slf4j.logevent_grammar:match(log) + assert(fields.Level == "ERROR", fields.Level) end -function process(tc) - mainlogevent() +function process() + single_logevent() - return 1 + return 0 end From 6c8f7047ff0213bc8eca08dbaf7f1e97f5b9481e Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 21:41:32 +0000 Subject: [PATCH 009/235] Fix our parsing, test passes --- modules/sfl4j.lua | 29 ++++++++++++++++------------- src/test/lua/lpeg_sfl4j.lua | 10 +++++++--- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index ad83d9a..11d04a5 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -2,7 +2,7 @@ -- -- imports -local l = require "lpeg" +local l = require "lpeg" local tonumber = tonumber l.locale(l) @@ -10,26 +10,29 @@ l.locale(l) local M = {} setfenv(1, M) -- Remove external access to contain everything in the module -local space = l.space^1 -local sep = l.P"\n" -local level = l.alpha^5 -- Log level (5 characters) -local brack = l.S("[]") -- [ and ] -local timestamp = (l.P(1) - brack)^1 -local colon = l.S(":") -- : -local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue -local line = (l.P(1) - sep)^0 * sep +local space = l.space^1 +local sep = l.P"\n" +local level = l.alpha^5 -- Log level (5 characters) +local lbrack = l.S("[") +local rbrack = l.S("]") +local timestamp = (l.P(1) - rbrack)^0 +local colon = l.S(":") -- : +local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue +local errmsg = (l.P(1) - sep)^0 +local line = (l.P(1) - sep)^0 * sep -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = l.Cg(level, "Level") -- ERROR * space - * brack + * lbrack * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 - * brack + * rbrack * space - * l.Cg(class, "class") -- com.domain.client.jobs.OutgoingQueue + * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue * colon * space - * l.Cg(line, "ErrorMessage") -- Error handling output... + * l.Cg(errmsg, "ErrorMessage") -- Error handling output... + * sep -- TODO: Multiline, need to figure out how to capture the entire stack in a table -- -- Example: ! com.domain.inet.ftp.Exception: java.io.IOException: FAILED diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index ba6a4a2..d107fd0 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -2,15 +2,19 @@ local sfl4j = require("sfl4j") local fields = { {"Level", "ERROR"}, - {"Timestamp", ""}, + {"Timestamp", "2014-11-21 16:35:59,501"}, {"Class", "com.domain.client.jobs.OutgoingQueue"}, {"ErrorMessage", "Error handling output file with job job-name"} } local function single_logevent() - local log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name' - local fields = slf4j.logevent_grammar:match(log) + local log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' + local fields = sfl4j.logevent_grammar:match(log) + if not fields then error("match didn't work " .. log) end assert(fields.Level == "ERROR", fields.Level) + assert(fields.Timestamp == "2014-11-21 16:35:59,501", fields.Timestamp) + assert(fields.Class == "com.domain.client.jobs.OutgoingQueue", fields.Class) + -- assert(fields.ErrorMessage == "Error handling output file with job job-name", fields.ErrorMessage) end function process() From 437071613603e377b8bf1c07ffbf2e070ea3583b Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 21:44:15 +0000 Subject: [PATCH 010/235] Forgot to uncomment this --- src/test/lua/lpeg_sfl4j.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index d107fd0..68e9a40 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -14,7 +14,7 @@ local function single_logevent() assert(fields.Level == "ERROR", fields.Level) assert(fields.Timestamp == "2014-11-21 16:35:59,501", fields.Timestamp) assert(fields.Class == "com.domain.client.jobs.OutgoingQueue", fields.Class) - -- assert(fields.ErrorMessage == "Error handling output file with job job-name", fields.ErrorMessage) + assert(fields.ErrorMessage == "Error handling output file with job job-name", fields.ErrorMessage) end function process() From f56439e80d73e59cd050b911cb3c358a141b31da Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 9 Dec 2014 21:57:14 +0000 Subject: [PATCH 011/235] Cleanup and idiomatic structure --- src/test/lua/lpeg_sfl4j.lua | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index 68e9a40..6a2069e 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -1,20 +1,23 @@ local sfl4j = require("sfl4j") -local fields = { - {"Level", "ERROR"}, - {"Timestamp", "2014-11-21 16:35:59,501"}, - {"Class", "com.domain.client.jobs.OutgoingQueue"}, - {"ErrorMessage", "Error handling output file with job job-name"} +local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' +local single_line_test_fields = { + Level = "ERROR", + Timestamp = "2014-11-21 16:35:59,501", + Class = "com.domain.client.jobs.OutgoingQueue", + ErrorMessage = "Error handling output file with job job-name" } local function single_logevent() - local log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' - local fields = sfl4j.logevent_grammar:match(log) - if not fields then error("match didn't work " .. log) end - assert(fields.Level == "ERROR", fields.Level) - assert(fields.Timestamp == "2014-11-21 16:35:59,501", fields.Timestamp) - assert(fields.Class == "com.domain.client.jobs.OutgoingQueue", fields.Class) - assert(fields.ErrorMessage == "Error handling output file with job job-name", fields.ErrorMessage) + local fields = sfl4j.logevent_grammar:match(single_line_log) + if not fields then error("match didn't work " .. single_line_log) end + assert(fields.Level == single_line_test_fields.Level, fields.Level) + assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) + assert(fields.Class == single_line_test_fields.Class, fields.Class) + assert(fields.ErrorMessage == single_line_test_fields.ErrorMessage, fields.ErrorMessage) +end + +local function mult_line_logevent() end function process() From c395e2674e3a35f88237f06653051145963cafd3 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 10 Dec 2014 01:13:34 +0000 Subject: [PATCH 012/235] Start supporting multi-line events --- modules/sfl4j.lua | 77 +++++++++++++---------------- src/test/lua/lpeg_sfl4j.lua | 99 ++++++++++++++++++++++++++++++++----- 2 files changed, 121 insertions(+), 55 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 11d04a5..4d08d6c 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -18,57 +18,46 @@ local rbrack = l.S("]") local timestamp = (l.P(1) - rbrack)^0 local colon = l.S(":") -- : local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue -local errmsg = (l.P(1) - sep)^0 +local msg = (l.P(1) - sep)^0 local line = (l.P(1) - sep)^0 * sep -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = l.Cg(level, "Level") -- ERROR - * space - * lbrack + * space * lbrack * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 - * rbrack - * space - * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue - * colon - * space - * l.Cg(errmsg, "ErrorMessage") -- Error handling output... - * sep + * rbrack * space + * l.Cg(class, "ErrorClass") -- com.domain.client.jobs.OutgoingQueue + * colon * space + * l.Cg(msg, "ErrorMessage") -- Error handling output... -- TODO: Multiline, need to figure out how to capture the entire stack in a table --- -- Example: ! com.domain.inet.ftp.Exception: java.io.IOException: FAILED --- local stackline = l.P"!" --- * space --- * line -- com.domain.inet.ftp.Exception --- --- -- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) --- local stackatline = l.P"!" --- * space --- * l.P"at" --- * space --- * line -- com.domain.inet.ftp.Exception --- --- -- Matches entire line, use this to gather stuff we don't parse properly --- local miscline = line - --- local logevent = logline --- * stackline --- * stackatline --- * miscline --- * sep - -local slf4j_severity = l.digit^1 / tonumber - -local slf4j_log_levels_text = ( - (l.P"fatal" + "FATAL") / "6" -+ (l.P"error" + "ERROR") / "5" -+ (l.P"warn" + "WARN") / "4" -+ (l.P"info" + "INFO") / "3" -+ (l.P"debug" + "DEBUG") / "2" -+ (l.P"trace" + "TRACE") / "1" -) / tonumber - +-- Example: ! java.net.SocketTimeoutException: Read timed out +local stackline = l.P"!" * space + * l.Cg(class, "ExceptionClass") -- java.net.SocketTimeoutException + * colon * space + * l.Cg(msg, "ExceptionMessage") -- Read timed out + +-- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) + +-- Match line, use this to gather stuff we don't parse properly +-- local misc = line + +local logevent = logline * sep + * (stackline * sep)^0 + -- * (stackatline * sep)^0 + -- * (miscline * sep)^0 + +logevent_grammar = l.Ct(logevent) + +-- local slf4j_severity = l.digit^1 / tonumber +-- local slf4j_log_levels_text = ( +-- (l.P"fatal" + "FATAL") / "6" +-- + (l.P"error" + "ERROR") / "5" +-- + (l.P"warn" + "WARN") / "4" +-- + (l.P"info" + "INFO") / "3" +-- + (l.P"debug" + "DEBUG") / "2" +-- + (l.P"trace" + "TRACE") / "1" +-- ) / tonumber -- TODO: Map slf4j to standard log levels for "Severity" -logevent_grammar = l.Ct(logline) - return M diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index 6a2069e..c7d0593 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -1,27 +1,104 @@ local sfl4j = require("sfl4j") -local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' -local single_line_test_fields = { - Level = "ERROR", - Timestamp = "2014-11-21 16:35:59,501", - Class = "com.domain.client.jobs.OutgoingQueue", - ErrorMessage = "Error handling output file with job job-name" -} - -local function single_logevent() +local function single_line_logevent() + local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' + local single_line_test_fields = { + Level = "ERROR", + Timestamp = "2014-11-21 16:35:59,501", + ErrorClass = "com.domain.client.jobs.OutgoingQueue", + ErrorMessage = "Error handling output file with job job-name" + } + local fields = sfl4j.logevent_grammar:match(single_line_log) if not fields then error("match didn't work " .. single_line_log) end assert(fields.Level == single_line_test_fields.Level, fields.Level) assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) - assert(fields.Class == single_line_test_fields.Class, fields.Class) + assert(fields.ErrorClass == single_line_test_fields.ErrorClass, fields.ErrorClass) assert(fields.ErrorMessage == single_line_test_fields.ErrorMessage, fields.ErrorMessage) end +local function two_line_logevent() + local two_line_log = [[ +ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in user loop +! java.net.SocketTimeoutException: Read timed out + ]] + local two_line_test_fields = { + Level = "ERROR", + Timestamp = "2014-11-20 16:40:51,577", + ErrorClass = "com.domain.substitute.UsersLoop", + ErrorMessage = "Error caught in user loop", + ExceptionClass = "java.net.SocketTimeoutException", + ExceptionMessage = "Read timed out" + } + + local fields = sfl4j.logevent_grammar:match(two_line_log) + if not fields then error("match didn't work " .. two_line_log) end + assert(fields.Level == two_line_test_fields.Level, fields.Level) + assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) + assert(fields.ErrorClass == two_line_test_fields.ErrorClass, fields.ErrorClass) + assert(fields.ErrorMessage == two_line_test_fields.ErrorMessage, fields.ErrorMessage) + assert(fields.ExceptionClass == two_line_test_fields.ExceptionClass, fields.ExceptionClass) + assert(fields.ExceptionMessage == two_line_test_fields.ExceptionMessage, fields.ExceptionMessage) +end + + +local multi_line_log = [[ +ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in user loop +! java.net.SocketTimeoutException: Read timed out +! at java.net.SocketInputStream.socketRead0(Native Method) +! at java.net.SocketInputStream.read(SocketInputStream.java:152) +! at java.net.SocketInputStream.read(SocketInputStream.java:122) +! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442) +! at sun.security.ssl.InputRecord.read(InputRecord.java:480) +! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927) +! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884) +! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102) +! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136) +! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152) +! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270) +! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) +! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) +! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) +! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161) +! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) +! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) +! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) +! at java.lang.reflect.Method.invoke(Method.java:606) +! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138) +! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source) +! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) +! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) +! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253) +! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194) +! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85) +! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108) +! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186) +! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) +! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) +! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) +! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) +! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) +! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) +! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) +! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) +! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) +! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) +! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) +! at java.lang.Thread.run(Thread.java:744) +]] + local function mult_line_logevent() + local multi_line_test_fields = { + Level = "ERROR", + Timestamp = "2014-11-20 16:40:51,577", + ErrorClass = "com.domain.substitute.UsersLoop", + ErrorMessage = "Error caught in user loop" + } end function process() - single_logevent() + single_line_logevent() + two_line_logevent() return 0 end From d700cdb33c6ab4a4b30cab9d3e8afdddfa75fadb Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 10 Dec 2014 19:41:19 +0000 Subject: [PATCH 013/235] Cleanup and break out the full stack --- modules/sfl4j.lua | 17 +++----- src/test/lua/lpeg_sfl4j.lua | 86 +++++++++++++++++++++++++++++-------- 2 files changed, 73 insertions(+), 30 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 4d08d6c..87a3bdb 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -26,26 +26,21 @@ local logline = l.Cg(level, "Level") -- ERROR * space * lbrack * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 * rbrack * space - * l.Cg(class, "ErrorClass") -- com.domain.client.jobs.OutgoingQueue + * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue * colon * space - * l.Cg(msg, "ErrorMessage") -- Error handling output... + * l.Cg(msg, "Message") -- Error handling output... -- TODO: Multiline, need to figure out how to capture the entire stack in a table -- Example: ! java.net.SocketTimeoutException: Read timed out -local stackline = l.P"!" * space - * l.Cg(class, "ExceptionClass") -- java.net.SocketTimeoutException - * colon * space - * l.Cg(msg, "ExceptionMessage") -- Read timed out - +local stackline = l.P"!" * space * class * colon * space * msg * sep -- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) +local stackatline = l.P"!" * space * l.P"at" * space * (l.P(1) - sep)^0 * sep + -- Match line, use this to gather stuff we don't parse properly -- local misc = line -local logevent = logline * sep - * (stackline * sep)^0 - -- * (stackatline * sep)^0 - -- * (miscline * sep)^0 +local logevent = logline * sep * l.Cg(stackline^0 * stackatline^0 * line^0, "Stacktrace") logevent_grammar = l.Ct(logevent) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index c7d0593..84107c3 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -5,16 +5,16 @@ local function single_line_logevent() local single_line_test_fields = { Level = "ERROR", Timestamp = "2014-11-21 16:35:59,501", - ErrorClass = "com.domain.client.jobs.OutgoingQueue", - ErrorMessage = "Error handling output file with job job-name" + Class = "com.domain.client.jobs.OutgoingQueue", + Message = "Error handling output file with job job-name" } local fields = sfl4j.logevent_grammar:match(single_line_log) if not fields then error("match didn't work " .. single_line_log) end assert(fields.Level == single_line_test_fields.Level, fields.Level) assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) - assert(fields.ErrorClass == single_line_test_fields.ErrorClass, fields.ErrorClass) - assert(fields.ErrorMessage == single_line_test_fields.ErrorMessage, fields.ErrorMessage) + assert(fields.Class == single_line_test_fields.Class, fields.Class) + assert(fields.Message == single_line_test_fields.Message, fields.Message) end local function two_line_logevent() @@ -25,20 +25,17 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in local two_line_test_fields = { Level = "ERROR", Timestamp = "2014-11-20 16:40:51,577", - ErrorClass = "com.domain.substitute.UsersLoop", - ErrorMessage = "Error caught in user loop", - ExceptionClass = "java.net.SocketTimeoutException", - ExceptionMessage = "Read timed out" + Class = "com.domain.substitute.UsersLoop", + Message = "Error caught in user loop", + Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" } local fields = sfl4j.logevent_grammar:match(two_line_log) if not fields then error("match didn't work " .. two_line_log) end assert(fields.Level == two_line_test_fields.Level, fields.Level) assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) - assert(fields.ErrorClass == two_line_test_fields.ErrorClass, fields.ErrorClass) - assert(fields.ErrorMessage == two_line_test_fields.ErrorMessage, fields.ErrorMessage) - assert(fields.ExceptionClass == two_line_test_fields.ExceptionClass, fields.ExceptionClass) - assert(fields.ExceptionMessage == two_line_test_fields.ExceptionMessage, fields.ExceptionMessage) + assert(fields.Class == two_line_test_fields.Class, fields.Class) + assert(fields.Message == two_line_test_fields.Message, fields.Message) end @@ -87,18 +84,69 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ! at java.lang.Thread.run(Thread.java:744) ]] -local function mult_line_logevent() - local multi_line_test_fields = { - Level = "ERROR", - Timestamp = "2014-11-20 16:40:51,577", - ErrorClass = "com.domain.substitute.UsersLoop", - ErrorMessage = "Error caught in user loop" - } +local multi_line_test_fields = { + Level = "ERROR", + Timestamp = "2014-11-20 16:40:51,577", + Class = "com.domain.substitute.UsersLoop", + Message = "Error caught in user loop", + Stacktrace = [[ + ! java.net.SocketTimeoutException: Read timed out + ! at java.net.SocketInputStream.socketRead0(Native Method) + ! at java.net.SocketInputStream.read(SocketInputStream.java:152) + ! at java.net.SocketInputStream.read(SocketInputStream.java:122) + ! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442) + ! at sun.security.ssl.InputRecord.read(InputRecord.java:480) + ! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927) + ! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884) + ! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102) + ! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136) + ! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152) + ! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270) + ! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) + ! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) + ! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) + ! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161) + ! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + ! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) + ! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + ! at java.lang.reflect.Method.invoke(Method.java:606) + ! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138) + ! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source) + ! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) + ! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) + ! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253) + ! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194) + ! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85) + ! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108) + ! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186) + ! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) + ! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) + ! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) + ! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) + ! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) + ! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) + ! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) + ! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) + ! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) + ! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) + ! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) + ! at java.lang.Thread.run(Thread.java:744) + ]] +} + +local function multi_line_logevent() + local fields = sfl4j.logevent_grammar:match(multi_line_log) + if not fields then error("match didn't work " .. multi_line_log) end + assert(fields.Level == multi_line_test_fields.Level, fields.Level) + assert(fields.Timestamp == multi_line_test_fields.Timestamp, fields.Timestamp) + assert(fields.Class == multi_line_test_fields.Class, fields.Class) + assert(fields.Message == multi_line_test_fields.Message, fields.Message) end function process() single_line_logevent() two_line_logevent() + multi_line_logevent() return 0 end From e219344f2dd872b4ec658cc8f55efc2bd90a1c32 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 10 Dec 2014 20:03:48 +0000 Subject: [PATCH 014/235] Simplify our stacktrace parsing and fix tests --- modules/sfl4j.lua | 4 ++-- src/test/lua/lpeg_sfl4j.lua | 46 +++---------------------------------- 2 files changed, 5 insertions(+), 45 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 87a3bdb..4386e14 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -32,9 +32,9 @@ local logline = l.Cg(level, "Level") -- ERROR -- TODO: Multiline, need to figure out how to capture the entire stack in a table -- Example: ! java.net.SocketTimeoutException: Read timed out -local stackline = l.P"!" * space * class * colon * space * msg * sep +local stackline = l.P"!" * space * line -- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) -local stackatline = l.P"!" * space * l.P"at" * space * (l.P(1) - sep)^0 * sep +local stackatline = l.P"!" * space * l.P"at" * space * line -- Match line, use this to gather stuff we don't parse properly diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index 84107c3..b25ce50 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -36,6 +36,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == two_line_test_fields.Class, fields.Class) assert(fields.Message == two_line_test_fields.Message, fields.Message) + assert(fields.Stacktrace == two_line_test_fields.Stacktrace, fields.Stacktrace) end @@ -89,49 +90,7 @@ local multi_line_test_fields = { Timestamp = "2014-11-20 16:40:51,577", Class = "com.domain.substitute.UsersLoop", Message = "Error caught in user loop", - Stacktrace = [[ - ! java.net.SocketTimeoutException: Read timed out - ! at java.net.SocketInputStream.socketRead0(Native Method) - ! at java.net.SocketInputStream.read(SocketInputStream.java:152) - ! at java.net.SocketInputStream.read(SocketInputStream.java:122) - ! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442) - ! at sun.security.ssl.InputRecord.read(InputRecord.java:480) - ! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927) - ! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884) - ! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102) - ! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136) - ! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152) - ! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270) - ! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) - ! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) - ! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) - ! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161) - ! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - ! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) - ! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - ! at java.lang.reflect.Method.invoke(Method.java:606) - ! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138) - ! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source) - ! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) - ! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) - ! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253) - ! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194) - ! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85) - ! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108) - ! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186) - ! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) - ! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) - ! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) - ! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) - ! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) - ! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) - ! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) - ! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) - ! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) - ! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) - ! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) - ! at java.lang.Thread.run(Thread.java:744) - ]] + Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" } local function multi_line_logevent() @@ -141,6 +100,7 @@ local function multi_line_logevent() assert(fields.Timestamp == multi_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == multi_line_test_fields.Class, fields.Class) assert(fields.Message == multi_line_test_fields.Message, fields.Message) + assert(fields.Stacktrace == multi_line_test_fields.Stacktrace, fields.Stacktrace) end function process() From 02158b1bb2f1e5ca837791455883ff7fa6334cd1 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 10 Dec 2014 20:05:33 +0000 Subject: [PATCH 015/235] Add some comments --- src/test/lua/lpeg_sfl4j.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index b25ce50..e422105 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -1,5 +1,6 @@ local sfl4j = require("sfl4j") +-- Parse a basic message, these are typically not errors, but whatever. local function single_line_logevent() local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' local single_line_test_fields = { @@ -17,6 +18,7 @@ local function single_line_logevent() assert(fields.Message == single_line_test_fields.Message, fields.Message) end +-- Use this to easily debug multi-line messages local function two_line_logevent() local two_line_log = [[ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in user loop From a1abf5f62bdcae50673659cdcc1ddc296635ab7b Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 10 Dec 2014 20:26:43 +0000 Subject: [PATCH 016/235] Cleanup and simplify --- modules/sfl4j.lua | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 4386e14..e4188d8 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -13,33 +13,25 @@ setfenv(1, M) -- Remove external access to contain everything in the module local space = l.space^1 local sep = l.P"\n" local level = l.alpha^5 -- Log level (5 characters) -local lbrack = l.S("[") -local rbrack = l.S("]") local timestamp = (l.P(1) - rbrack)^0 -local colon = l.S(":") -- : local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue local msg = (l.P(1) - sep)^0 local line = (l.P(1) - sep)^0 * sep -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = l.Cg(level, "Level") -- ERROR - * space * lbrack + * space * l.P("[") * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 - * rbrack * space - * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue - * colon * space - * l.Cg(msg, "Message") -- Error handling output... + * l.P("]") * space + * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue + * l.P(":") * space + * l.Cg(msg, "Message") -- Error handling output... --- TODO: Multiline, need to figure out how to capture the entire stack in a table -- Example: ! java.net.SocketTimeoutException: Read timed out local stackline = l.P"!" * space * line -- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) local stackatline = l.P"!" * space * l.P"at" * space * line - --- Match line, use this to gather stuff we don't parse properly --- local misc = line - local logevent = logline * sep * l.Cg(stackline^0 * stackatline^0 * line^0, "Stacktrace") logevent_grammar = l.Ct(logevent) From 05f6f6783ab7eb318c19b5839faedb8d9c8242e1 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Thu, 11 Dec 2014 15:41:32 +0000 Subject: [PATCH 017/235] Convert levels to severity (syslog) --- modules/sfl4j.lua | 25 ++++++++++++++++++++++--- src/test/lua/lpeg_sfl4j.lua | 33 ++++++++++++++++++--------------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index e4188d8..205b75b 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -13,13 +13,31 @@ setfenv(1, M) -- Remove external access to contain everything in the module local space = l.space^1 local sep = l.P"\n" local level = l.alpha^5 -- Log level (5 characters) -local timestamp = (l.P(1) - rbrack)^0 -local class = (l.P(1) - colon)^1 -- com.domain.client.jobs.OutgoingQueue +local timestamp = (l.P(1) - l.P("]"))^0 +local class = (l.P(1) - l.P(":"))^1 -- com.domain.client.jobs.OutgoingQueue local msg = (l.P(1) - sep)^0 local line = (l.P(1) - sep)^0 * sep +-- slf4j uses the following levels/severity +-- "FATAL" = 6 +-- "ERROR" = 5 +-- "WARN" = 4 +-- "INFO" = 3 +-- "DEBUG" = 2 +-- "TRACE" = 1 + +-- Map sfl4j log levels to syslog severity +local sfl4j_levels = l.Cg(( + l.P"TRACE" / "7" ++ l.P"DEBUG" / "7" ++ l.P"INFO" / "6" ++ l.P"WARN" / "4" ++ l.P"ERROR" / "3" ++ l.P"FATAL" / "0") +/ tonumber, "Severity") + -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name -local logline = l.Cg(level, "Level") -- ERROR +local logline = sfl4j_levels -- ERROR * space * l.P("[") * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 * l.P("]") * space @@ -32,6 +50,7 @@ local stackline = l.P"!" * space * line -- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) local stackatline = l.P"!" * space * l.P"at" * space * line +-- A representation of a full log event local logevent = logline * sep * l.Cg(stackline^0 * stackatline^0 * line^0, "Stacktrace") logevent_grammar = l.Ct(logevent) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index e422105..a9e736c 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -4,15 +4,16 @@ local sfl4j = require("sfl4j") local function single_line_logevent() local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' local single_line_test_fields = { - Level = "ERROR", - Timestamp = "2014-11-21 16:35:59,501", - Class = "com.domain.client.jobs.OutgoingQueue", - Message = "Error handling output file with job job-name" + Severity = 3, + Timestamp = "2014-11-21 16:35:59,501", + Class = "com.domain.client.jobs.OutgoingQueue", + Message = "Error handling output file with job job-name" } local fields = sfl4j.logevent_grammar:match(single_line_log) if not fields then error("match didn't work " .. single_line_log) end - assert(fields.Level == single_line_test_fields.Level, fields.Level) + + assert(fields.Severity == single_line_test_fields.Severity, fields.Severity) assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == single_line_test_fields.Class, fields.Class) assert(fields.Message == single_line_test_fields.Message, fields.Message) @@ -25,16 +26,17 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ! java.net.SocketTimeoutException: Read timed out ]] local two_line_test_fields = { - Level = "ERROR", - Timestamp = "2014-11-20 16:40:51,577", - Class = "com.domain.substitute.UsersLoop", - Message = "Error caught in user loop", + Severity = 3, + Timestamp = "2014-11-20 16:40:51,577", + Class = "com.domain.substitute.UsersLoop", + Message = "Error caught in user loop", Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" } local fields = sfl4j.logevent_grammar:match(two_line_log) if not fields then error("match didn't work " .. two_line_log) end - assert(fields.Level == two_line_test_fields.Level, fields.Level) + + assert(fields.Severity == two_line_test_fields.Severity, fields.Severity) assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == two_line_test_fields.Class, fields.Class) assert(fields.Message == two_line_test_fields.Message, fields.Message) @@ -88,17 +90,18 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ]] local multi_line_test_fields = { - Level = "ERROR", - Timestamp = "2014-11-20 16:40:51,577", - Class = "com.domain.substitute.UsersLoop", - Message = "Error caught in user loop", + Severity = 3, + Timestamp = "2014-11-20 16:40:51,577", + Class = "com.domain.substitute.UsersLoop", + Message = "Error caught in user loop", Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" } local function multi_line_logevent() local fields = sfl4j.logevent_grammar:match(multi_line_log) if not fields then error("match didn't work " .. multi_line_log) end - assert(fields.Level == multi_line_test_fields.Level, fields.Level) + + assert(fields.Severity == multi_line_test_fields.Severity, fields.Severity) assert(fields.Timestamp == multi_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == multi_line_test_fields.Class, fields.Class) assert(fields.Message == multi_line_test_fields.Message, fields.Message) From a64fb07b6d59dd39539d5d3abdf9fa83ddae8431 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Thu, 11 Dec 2014 15:44:01 +0000 Subject: [PATCH 018/235] Support UPPER and lower levels --- modules/sfl4j.lua | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 205b75b..0b344c0 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -28,12 +28,12 @@ local line = (l.P(1) - sep)^0 * sep -- Map sfl4j log levels to syslog severity local sfl4j_levels = l.Cg(( - l.P"TRACE" / "7" -+ l.P"DEBUG" / "7" -+ l.P"INFO" / "6" -+ l.P"WARN" / "4" -+ l.P"ERROR" / "3" -+ l.P"FATAL" / "0") + (l.P"TRACE" + "trace") / "7" ++ (l.P"DEBUG" + "debug") / "7" ++ (l.P"INFO" + "info") / "6" ++ (l.P"WARN" + "warn") / "4" ++ (l.P"ERROR" + "error") / "3" ++ (l.P"FATAL" + "fatal") / "0") / tonumber, "Severity") -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name @@ -55,15 +55,4 @@ local logevent = logline * sep * l.Cg(stackline^0 * stackatline^0 * line^0, "Sta logevent_grammar = l.Ct(logevent) --- local slf4j_severity = l.digit^1 / tonumber --- local slf4j_log_levels_text = ( --- (l.P"fatal" + "FATAL") / "6" --- + (l.P"error" + "ERROR") / "5" --- + (l.P"warn" + "WARN") / "4" --- + (l.P"info" + "INFO") / "3" --- + (l.P"debug" + "DEBUG") / "2" --- + (l.P"trace" + "TRACE") / "1" --- ) / tonumber --- TODO: Map slf4j to standard log levels for "Severity" - return M From 80da6124a41596cbdfe72ab43551a5782a21f2ff Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Thu, 11 Dec 2014 17:55:36 +0000 Subject: [PATCH 019/235] Parse time properly --- modules/sfl4j.lua | 9 +++++++-- src/test/lua/lpeg_sfl4j.lua | 13 +++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 0b344c0..4c250b0 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -3,6 +3,7 @@ -- imports local l = require "lpeg" +local dt = require "date_time" local tonumber = tonumber l.locale(l) @@ -13,7 +14,6 @@ setfenv(1, M) -- Remove external access to contain everything in the module local space = l.space^1 local sep = l.P"\n" local level = l.alpha^5 -- Log level (5 characters) -local timestamp = (l.P(1) - l.P("]"))^0 local class = (l.P(1) - l.P(":"))^1 -- com.domain.client.jobs.OutgoingQueue local msg = (l.P(1) - sep)^0 local line = (l.P(1) - sep)^0 * sep @@ -36,10 +36,15 @@ local sfl4j_levels = l.Cg(( + (l.P"FATAL" + "fatal") / "0") / tonumber, "Severity") +-- Example: 2014-11-21 16:35:59,501 +local time_secfrac = l.Cg(l.P"," * l.digit^1 / tonumber, "sec_frac") +local partial_time = dt.time_hour * l.P":" * dt.time_minute * l.P":" * dt.time_second * time_secfrac^-1 +local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) + -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = sfl4j_levels -- ERROR * space * l.P("[") - * l.Cg(timestamp, "Timestamp") -- 2014-11-21 16:35:59,501 + * l.Cg(sfl4j_datetime, "Timestamp") -- 2014-11-21 16:35:59,501 * l.P("]") * space * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue * l.P(":") * space diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index a9e736c..00fa10f 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -1,11 +1,12 @@ local sfl4j = require("sfl4j") +local dt = require("date_time") -- Parse a basic message, these are typically not errors, but whatever. local function single_line_logevent() local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' local single_line_test_fields = { Severity = 3, - Timestamp = "2014-11-21 16:35:59,501", + Timestamp = 1.416587759e+18, Class = "com.domain.client.jobs.OutgoingQueue", Message = "Error handling output file with job job-name" } @@ -14,7 +15,7 @@ local function single_line_logevent() if not fields then error("match didn't work " .. single_line_log) end assert(fields.Severity == single_line_test_fields.Severity, fields.Severity) - assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) + assert(dt.time_to_ns(fields.Timestamp) == single_line_test_fields.Timestamp, dt.time_to_ns(fields.Timestamp)) assert(fields.Class == single_line_test_fields.Class, fields.Class) assert(fields.Message == single_line_test_fields.Message, fields.Message) end @@ -27,7 +28,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ]] local two_line_test_fields = { Severity = 3, - Timestamp = "2014-11-20 16:40:51,577", + Timestamp = 1.416501651e+18, Class = "com.domain.substitute.UsersLoop", Message = "Error caught in user loop", Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" @@ -37,7 +38,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in if not fields then error("match didn't work " .. two_line_log) end assert(fields.Severity == two_line_test_fields.Severity, fields.Severity) - assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) + assert(dt.time_to_ns(fields.Timestamp) == two_line_test_fields.Timestamp, dt.time_to_ns(fields.Timestamp)) assert(fields.Class == two_line_test_fields.Class, fields.Class) assert(fields.Message == two_line_test_fields.Message, fields.Message) assert(fields.Stacktrace == two_line_test_fields.Stacktrace, fields.Stacktrace) @@ -91,7 +92,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in local multi_line_test_fields = { Severity = 3, - Timestamp = "2014-11-20 16:40:51,577", + Timestamp = 1.416501651e+18, Class = "com.domain.substitute.UsersLoop", Message = "Error caught in user loop", Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" @@ -102,7 +103,7 @@ local function multi_line_logevent() if not fields then error("match didn't work " .. multi_line_log) end assert(fields.Severity == multi_line_test_fields.Severity, fields.Severity) - assert(fields.Timestamp == multi_line_test_fields.Timestamp, fields.Timestamp) + assert(dt.time_to_ns(fields.Timestamp) == multi_line_test_fields.Timestamp, dt.time_to_ns(fields.Timestamp)) assert(fields.Class == multi_line_test_fields.Class, fields.Class) assert(fields.Message == multi_line_test_fields.Message, fields.Message) assert(fields.Stacktrace == multi_line_test_fields.Stacktrace, fields.Stacktrace) From 9ce1e7e484681af7b426af7c1e0509365d171f60 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Thu, 11 Dec 2014 18:00:59 +0000 Subject: [PATCH 020/235] Remove internal gist reference --- modules/sfl4j.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 4c250b0..4191476 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -1,6 +1,3 @@ --- Using this as a reference: https://github.banksimple.com/gist/sjl/92b4b5750739a46d38a7 --- - -- imports local l = require "lpeg" local dt = require "date_time" From d835aa7e0814edf373ccc322adc04c20af0738c8 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Thu, 11 Dec 2014 21:17:11 +0000 Subject: [PATCH 021/235] Remove unneeded variables --- modules/sfl4j.lua | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 4191476..c1d69fb 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -10,7 +10,6 @@ setfenv(1, M) -- Remove external access to contain everything in the module local space = l.space^1 local sep = l.P"\n" -local level = l.alpha^5 -- Log level (5 characters) local class = (l.P(1) - l.P(":"))^1 -- com.domain.client.jobs.OutgoingQueue local msg = (l.P(1) - sep)^0 local line = (l.P(1) - sep)^0 * sep @@ -49,8 +48,6 @@ local logline = sfl4j_levels -- ERROR -- Example: ! java.net.SocketTimeoutException: Read timed out local stackline = l.P"!" * space * line --- Example: ! at com.domain.inet.ftp.TransferMode.upload(Unknown Source) -local stackatline = l.P"!" * space * l.P"at" * space * line -- A representation of a full log event local logevent = logline * sep * l.Cg(stackline^0 * stackatline^0 * line^0, "Stacktrace") From ac657fc5f404ac5d89f8ad6a902966a64fd6153d Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Tue, 16 Dec 2014 15:49:35 +0000 Subject: [PATCH 022/235] Logs of cleanup, fixes and correct time --- modules/sfl4j.lua | 13 +++++-------- src/test/lua/lpeg_sfl4j.lua | 12 ++++++------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index c1d69fb..99ea4a8 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -33,24 +33,21 @@ local sfl4j_levels = l.Cg(( / tonumber, "Severity") -- Example: 2014-11-21 16:35:59,501 -local time_secfrac = l.Cg(l.P"," * l.digit^1 / tonumber, "sec_frac") -local partial_time = dt.time_hour * l.P":" * dt.time_minute * l.P":" * dt.time_second * time_secfrac^-1 -local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) +local time_secfrac = l.Cg(l.digit^3 / tonumber, "sec_frac") +local partial_time = dt.time_hour * l.P":" * dt.time_minute * l.P":" * dt.time_second * l.P"," * time_secfrac^-1 +local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) / dt.time_to_ns -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = sfl4j_levels -- ERROR * space * l.P("[") - * l.Cg(sfl4j_datetime, "Timestamp") -- 2014-11-21 16:35:59,501 + * l.Cg(sfl4j_datetime, "Timestamp") -- 2014-11-21 16:35:59,501 * l.P("]") * space * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue * l.P(":") * space * l.Cg(msg, "Message") -- Error handling output... --- Example: ! java.net.SocketTimeoutException: Read timed out -local stackline = l.P"!" * space * line - -- A representation of a full log event -local logevent = logline * sep * l.Cg(stackline^0 * stackatline^0 * line^0, "Stacktrace") +local logevent = logline * sep * l.Cg(line^0, "Stacktrace") logevent_grammar = l.Ct(logevent) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index 00fa10f..707068a 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -6,7 +6,7 @@ local function single_line_logevent() local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' local single_line_test_fields = { Severity = 3, - Timestamp = 1.416587759e+18, + Timestamp = 1.41658826e+18, Class = "com.domain.client.jobs.OutgoingQueue", Message = "Error handling output file with job job-name" } @@ -15,7 +15,7 @@ local function single_line_logevent() if not fields then error("match didn't work " .. single_line_log) end assert(fields.Severity == single_line_test_fields.Severity, fields.Severity) - assert(dt.time_to_ns(fields.Timestamp) == single_line_test_fields.Timestamp, dt.time_to_ns(fields.Timestamp)) + assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == single_line_test_fields.Class, fields.Class) assert(fields.Message == single_line_test_fields.Message, fields.Message) end @@ -28,7 +28,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ]] local two_line_test_fields = { Severity = 3, - Timestamp = 1.416501651e+18, + Timestamp = 1.416502228e+18, Class = "com.domain.substitute.UsersLoop", Message = "Error caught in user loop", Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" @@ -38,7 +38,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in if not fields then error("match didn't work " .. two_line_log) end assert(fields.Severity == two_line_test_fields.Severity, fields.Severity) - assert(dt.time_to_ns(fields.Timestamp) == two_line_test_fields.Timestamp, dt.time_to_ns(fields.Timestamp)) + assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == two_line_test_fields.Class, fields.Class) assert(fields.Message == two_line_test_fields.Message, fields.Message) assert(fields.Stacktrace == two_line_test_fields.Stacktrace, fields.Stacktrace) @@ -92,7 +92,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in local multi_line_test_fields = { Severity = 3, - Timestamp = 1.416501651e+18, + Timestamp = 1.416502228e+18, Class = "com.domain.substitute.UsersLoop", Message = "Error caught in user loop", Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" @@ -103,7 +103,7 @@ local function multi_line_logevent() if not fields then error("match didn't work " .. multi_line_log) end assert(fields.Severity == multi_line_test_fields.Severity, fields.Severity) - assert(dt.time_to_ns(fields.Timestamp) == multi_line_test_fields.Timestamp, dt.time_to_ns(fields.Timestamp)) + assert(fields.Timestamp == multi_line_test_fields.Timestamp, fields.Timestamp) assert(fields.Class == multi_line_test_fields.Class, fields.Class) assert(fields.Message == multi_line_test_fields.Message, fields.Message) assert(fields.Stacktrace == multi_line_test_fields.Stacktrace, fields.Stacktrace) From 34460d61f9aa89c790858e885c13151120f76d13 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 17 Dec 2014 21:26:21 +0000 Subject: [PATCH 023/235] Move to lower case keys to fit convention --- modules/sfl4j.lua | 10 +++---- src/test/lua/lpeg_sfl4j.lua | 52 ++++++++++++++++++------------------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 99ea4a8..51eb18c 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -30,7 +30,7 @@ local sfl4j_levels = l.Cg(( + (l.P"WARN" + "warn") / "4" + (l.P"ERROR" + "error") / "3" + (l.P"FATAL" + "fatal") / "0") -/ tonumber, "Severity") +/ tonumber, "severity") -- Example: 2014-11-21 16:35:59,501 local time_secfrac = l.Cg(l.digit^3 / tonumber, "sec_frac") @@ -40,14 +40,14 @@ local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) / dt.ti -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = sfl4j_levels -- ERROR * space * l.P("[") - * l.Cg(sfl4j_datetime, "Timestamp") -- 2014-11-21 16:35:59,501 + * l.Cg(sfl4j_datetime, "timestamp") -- 2014-11-21 16:35:59,501 * l.P("]") * space - * l.Cg(class, "Class") -- com.domain.client.jobs.OutgoingQueue + * l.Cg(class, "class") -- com.domain.client.jobs.OutgoingQueue * l.P(":") * space - * l.Cg(msg, "Message") -- Error handling output... + * l.Cg(msg, "message") -- Error handling output... -- A representation of a full log event -local logevent = logline * sep * l.Cg(line^0, "Stacktrace") +local logevent = logline * sep * l.Cg(line^0, "stacktrace") logevent_grammar = l.Ct(logevent) diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index 707068a..bb19d7d 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -5,19 +5,19 @@ local dt = require("date_time") local function single_line_logevent() local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' local single_line_test_fields = { - Severity = 3, - Timestamp = 1.41658826e+18, - Class = "com.domain.client.jobs.OutgoingQueue", - Message = "Error handling output file with job job-name" + severity = 3, + timestamp = 1.41658826e+18, + class = "com.domain.client.jobs.OutgoingQueue", + message = "Error handling output file with job job-name" } local fields = sfl4j.logevent_grammar:match(single_line_log) if not fields then error("match didn't work " .. single_line_log) end - assert(fields.Severity == single_line_test_fields.Severity, fields.Severity) - assert(fields.Timestamp == single_line_test_fields.Timestamp, fields.Timestamp) - assert(fields.Class == single_line_test_fields.Class, fields.Class) - assert(fields.Message == single_line_test_fields.Message, fields.Message) + assert(fields.severity == single_line_test_fields.severity, fields.severity) + assert(fields.timestamp == single_line_test_fields.timestamp, fields.timestamp) + assert(fields.class == single_line_test_fields.class, fields.class) + assert(fields.message == single_line_test_fields.message, fields.message) end -- Use this to easily debug multi-line messages @@ -27,20 +27,20 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ! java.net.SocketTimeoutException: Read timed out ]] local two_line_test_fields = { - Severity = 3, - Timestamp = 1.416502228e+18, - Class = "com.domain.substitute.UsersLoop", - Message = "Error caught in user loop", - Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" + severity = 3, + timestamp = 1.416502228e+18, + class = "com.domain.substitute.UsersLoop", + message = "Error caught in user loop", + stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" } local fields = sfl4j.logevent_grammar:match(two_line_log) if not fields then error("match didn't work " .. two_line_log) end - assert(fields.Severity == two_line_test_fields.Severity, fields.Severity) - assert(fields.Timestamp == two_line_test_fields.Timestamp, fields.Timestamp) - assert(fields.Class == two_line_test_fields.Class, fields.Class) - assert(fields.Message == two_line_test_fields.Message, fields.Message) + assert(fields.severity == two_line_test_fields.severity, fields.severity) + assert(fields.timestamp == two_line_test_fields.timestamp, fields.timestamp) + assert(fields.class == two_line_test_fields.class, fields.class) + assert(fields.message == two_line_test_fields.message, fields.message) assert(fields.Stacktrace == two_line_test_fields.Stacktrace, fields.Stacktrace) end @@ -91,21 +91,21 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ]] local multi_line_test_fields = { - Severity = 3, - Timestamp = 1.416502228e+18, - Class = "com.domain.substitute.UsersLoop", - Message = "Error caught in user loop", - Stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" + severity = 3, + timestamp = 1.416502228e+18, + class = "com.domain.substitute.UsersLoop", + message = "Error caught in user loop", + stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" } local function multi_line_logevent() local fields = sfl4j.logevent_grammar:match(multi_line_log) if not fields then error("match didn't work " .. multi_line_log) end - assert(fields.Severity == multi_line_test_fields.Severity, fields.Severity) - assert(fields.Timestamp == multi_line_test_fields.Timestamp, fields.Timestamp) - assert(fields.Class == multi_line_test_fields.Class, fields.Class) - assert(fields.Message == multi_line_test_fields.Message, fields.Message) + assert(fields.severity == multi_line_test_fields.severity, fields.severity) + assert(fields.timestamp == multi_line_test_fields.timestamp, fields.timestamp) + assert(fields.class == multi_line_test_fields.class, fields.class) + assert(fields.message == multi_line_test_fields.message, fields.message) assert(fields.Stacktrace == multi_line_test_fields.Stacktrace, fields.Stacktrace) end From d521498b2f3566054d42947db96b2efc12eaa19d Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Wed, 17 Dec 2014 21:50:21 +0000 Subject: [PATCH 024/235] Properly calculate sec fractions --- modules/sfl4j.lua | 6 +++++- src/test/lua/lpeg_sfl4j.lua | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index 51eb18c..e9997d7 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -32,8 +32,12 @@ local sfl4j_levels = l.Cg(( + (l.P"FATAL" + "fatal") / "0") / tonumber, "severity") +function convert_to_frac(sec) + return tonumber("0." .. sec) +end + -- Example: 2014-11-21 16:35:59,501 -local time_secfrac = l.Cg(l.digit^3 / tonumber, "sec_frac") +local time_secfrac = l.Cg(l.digit^3 / convert_to_frac, "sec_frac") local partial_time = dt.time_hour * l.P":" * dt.time_minute * l.P":" * dt.time_second * l.P"," * time_secfrac^-1 local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) / dt.time_to_ns diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua index bb19d7d..2fb720d 100644 --- a/src/test/lua/lpeg_sfl4j.lua +++ b/src/test/lua/lpeg_sfl4j.lua @@ -6,7 +6,7 @@ local function single_line_logevent() local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' local single_line_test_fields = { severity = 3, - timestamp = 1.41658826e+18, + timestamp = 1.416587759501e+18, class = "com.domain.client.jobs.OutgoingQueue", message = "Error handling output file with job job-name" } @@ -28,7 +28,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in ]] local two_line_test_fields = { severity = 3, - timestamp = 1.416502228e+18, + timestamp = 1.416501651577e+18, class = "com.domain.substitute.UsersLoop", message = "Error caught in user loop", stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" @@ -92,7 +92,7 @@ ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in local multi_line_test_fields = { severity = 3, - timestamp = 1.416502228e+18, + timestamp = 1.416501651577e+18, class = "com.domain.substitute.UsersLoop", message = "Error caught in user loop", stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" From a6a661e853ee111643160ac62d634b9f264175f3 Mon Sep 17 00:00:00 2001 From: Curt Micol Date: Fri, 19 Dec 2014 19:17:15 +0000 Subject: [PATCH 025/235] Don't use literal function calls --- modules/sfl4j.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua index e9997d7..94bce7f 100644 --- a/modules/sfl4j.lua +++ b/modules/sfl4j.lua @@ -10,7 +10,7 @@ setfenv(1, M) -- Remove external access to contain everything in the module local space = l.space^1 local sep = l.P"\n" -local class = (l.P(1) - l.P(":"))^1 -- com.domain.client.jobs.OutgoingQueue +local class = (l.P(1) - ":")^1 -- com.domain.client.jobs.OutgoingQueue local msg = (l.P(1) - sep)^0 local line = (l.P(1) - sep)^0 * sep @@ -32,22 +32,22 @@ local sfl4j_levels = l.Cg(( + (l.P"FATAL" + "fatal") / "0") / tonumber, "severity") -function convert_to_frac(sec) +local function convert_to_frac(sec) return tonumber("0." .. sec) end -- Example: 2014-11-21 16:35:59,501 local time_secfrac = l.Cg(l.digit^3 / convert_to_frac, "sec_frac") -local partial_time = dt.time_hour * l.P":" * dt.time_minute * l.P":" * dt.time_second * l.P"," * time_secfrac^-1 +local partial_time = dt.time_hour * ":" * dt.time_minute * ":" * dt.time_second * "," * time_secfrac^-1 local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) / dt.time_to_ns -- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name local logline = sfl4j_levels -- ERROR - * space * l.P("[") + * space * "[" * l.Cg(sfl4j_datetime, "timestamp") -- 2014-11-21 16:35:59,501 - * l.P("]") * space + * "]" * space * l.Cg(class, "class") -- com.domain.client.jobs.OutgoingQueue - * l.P(":") * space + * ":" * space * l.Cg(msg, "message") -- Error handling output... -- A representation of a full log event From 47185d4ec3005c45a6865895840316a6e77a7048 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 22 Dec 2014 10:43:58 -0800 Subject: [PATCH 026/235] Allow lsb_output to support strings with NUL characters https://github.com/mozilla-services/heka/issues/1240 --- src/lua_bloom_filter.c | 2 +- src/lua_circular_buffer.c | 8 ++++---- src/lua_hyperloglog.c | 2 +- src/lua_sandbox.c | 10 +++++++--- src/lua_sandbox_private.c | 9 +++++---- src/lua_sandbox_private.h | 3 ++- src/lua_serialize.c | 10 +++++----- 7 files changed, 25 insertions(+), 19 deletions(-) diff --git a/src/lua_bloom_filter.c b/src/lua_bloom_filter.c index 2a36474..fdc32d2 100644 --- a/src/lua_bloom_filter.c +++ b/src/lua_bloom_filter.c @@ -173,7 +173,7 @@ int serialize_bloom_filter(const char* key, bloom_filter* bf, output_data* outpu return 1; } if (serialize_binary(bf->data, bf->bytes, output)) return 1; - if (appends(output, "\")\n")) { + if (appends(output, "\")\n", 3)) { return 1; } return 0; diff --git a/src/lua_circular_buffer.c b/src/lua_circular_buffer.c index 297933d..34673b0 100644 --- a/src/lua_circular_buffer.c +++ b/src/lua_circular_buffer.c @@ -899,7 +899,7 @@ int output_circular_buffer_cbufd(lua_State* lua, circular_buffer* cb, if (appendc(output, '\t')) return 1; lua_rawgeti(lua, -1, column_idx); if (LUA_TNIL == lua_type(lua, -1)) { - if (appends(output, not_a_number)) return 1; + if (appends(output, not_a_number, 3)) return 1; } else { if (serialize_double(output, lua_tonumber(lua, -1))) return 1; } @@ -949,7 +949,7 @@ int output_circular_buffer(lua_State* lua, circular_buffer* cb, return 1; } } - if (appends(output, "]}\n")) return 1; + if (appends(output, "]}\n", 3)) return 1; if (OUTPUT_CBUFD == cb->format) { return output_circular_buffer_cbufd(lua, cb, output); @@ -980,7 +980,7 @@ int serialize_circular_buffer_delta(lua_State* lua, circular_buffer* cb, for (unsigned column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appends(output, " ")) return 1; + if (appendc(output, ' ')) return 1; lua_rawgeti(lua, -1, column_idx); if (serialize_double(output, lua_tonumber(lua, -1))) return 1; lua_pop(lua, 1); // remove the number @@ -1050,7 +1050,7 @@ int serialize_circular_buffer(lua_State* lua, const char* key, if (serialize_circular_buffer_delta(lua, cb, output)) { return 1; } - if (appends(output, "\")\n")) { + if (appends(output, "\")\n", 3)) { return 1; } return 0; diff --git a/src/lua_hyperloglog.c b/src/lua_hyperloglog.c index a00bf26..d11ef47 100644 --- a/src/lua_hyperloglog.c +++ b/src/lua_hyperloglog.c @@ -158,7 +158,7 @@ int serialize_hyperloglog(const char* key, hyperloglog* hll, output_data* output return 1; } if (serialize_binary(hll, sizeof(hyperloglog) - 1, output)) return 1; - if (appends(output, "\")\n")) { + if (appends(output, "\")\n", 3)) { return 1; } return 0; diff --git a/src/lua_sandbox.c b/src/lua_sandbox.c index 07bfc90..cb140d5 100644 --- a/src/lua_sandbox.c +++ b/src/lua_sandbox.c @@ -308,12 +308,16 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) } break; case LUA_TSTRING: - if (appendf(&lsb->output, "%s", lua_tostring(lsb->lua, i))) { - result = 1; + { + size_t len; + const char* s = lua_tolstring(lsb->lua, i, &len); + if (appends(&lsb->output, s, len)) { + result = 1; + } } break; case LUA_TNIL: - if (appends(&lsb->output, "nil")) { + if (appends(&lsb->output, "nil", 3)) { result = 1; } break; diff --git a/src/lua_sandbox_private.c b/src/lua_sandbox_private.c index 4bb1393..39b4000 100644 --- a/src/lua_sandbox_private.c +++ b/src/lua_sandbox_private.c @@ -210,14 +210,15 @@ int realloc_output(output_data* output, size_t needed) } -int appends(output_data* output, const char* str) +int appends(output_data* output, const char* str, size_t len) { - size_t needed = strlen(str) + 1; + size_t needed = len + 1; if (output->size - output->pos < needed) { if (realloc_output(output, needed)) return 1; } - memcpy(output->data + output->pos, str, needed); - output->pos += needed - 1; + memcpy(output->data + output->pos, str, len); + output->pos += len; + output->data[output->pos] = 0; return 0; } diff --git a/src/lua_sandbox_private.h b/src/lua_sandbox_private.h index 2eaaf9a..9d41f06 100644 --- a/src/lua_sandbox_private.h +++ b/src/lua_sandbox_private.h @@ -127,10 +127,11 @@ int realloc_output(output_data* output, size_t needed); * * @param output Pointer the output collector. * @param str String to append to the output. + * @param len Length of the string to append * * @return int Zero on success, non-zero if out of memory. */ -int appends(output_data* output, const char* str); +int appends(output_data* output, const char* str, size_t len); /** * Append a character to the output stream. diff --git a/src/lua_serialize.c b/src/lua_serialize.c index ac779ec..60826b3 100644 --- a/src/lua_serialize.c +++ b/src/lua_serialize.c @@ -147,7 +147,7 @@ int preserve_global_data(lua_sandbox* lsb, const char* data_file) int serialize_double(output_data* output, double d) { if (isnan(d)) { - return appends(output, not_a_number); + return appends(output, not_a_number, 3); } if (d > INT_MAX) { return appendf(output, "%0.17g", d); @@ -557,16 +557,16 @@ int serialize_binary(const void* src, size_t len, output_data* output) for (unsigned i = 0; i < len; ++i) { switch (uc[i]) { case '\n': - if (appends(output, "\\n")) return 1; + if (appends(output, "\\n", 2)) return 1; break; case '\r': - if (appends(output, "\\r")) return 1; + if (appends(output, "\\r", 2)) return 1; break; case '"': - if (appends(output, "\\\"")) return 1; + if (appends(output, "\\\"", 2)) return 1; break; case '\\': - if (appends(output, "\\\\")) return 1; + if (appends(output, "\\\\", 2)) return 1; break; default: if (appendc(output, uc[i])) return 1; From 4ed46bc9b2e7f13cba9d800541afef7e7e90ecd5 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 8 Jan 2015 09:08:43 -0800 Subject: [PATCH 027/235] Fix test results for timestamps with no year since they now default to 2015 --- src/test/lua/lpeg_date_time.lua | 18 +++++++++--------- src/test/lua/lpeg_syslog.lua | 6 +++--- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua index 62eed68..eb1b0b7 100644 --- a/src/test/lua/lpeg_date_time.lua +++ b/src/test/lua/lpeg_date_time.lua @@ -6,7 +6,7 @@ require "string" local dt = require("date_time") -local function test_valid(tc, grammar, tests, results) +local function test_valid(grammar, tests, results) for i,v in ipairs(tests) do t = grammar:match(v) if not t then @@ -14,7 +14,7 @@ local function test_valid(tc, grammar, tests, results) else ns = dt.time_to_ns(t) if ns ~= results[i] then - error(string.format("tc: %d %s expected: %g received: %g", tc, v, results[i], ns)) + error(string.format("test: %d %s expected: %g received: %g", i, v, results[i], ns), 2) end end end @@ -36,7 +36,7 @@ local function rfc3339() if os.date("%c"):find("^%d") then results[6] = 0 -- windows will fail to convert this time end - test_valid(tc, dt.rfc3339, tests, results) + test_valid(dt.rfc3339, tests, results) end local function rfc3339_invalid() @@ -54,25 +54,25 @@ end local function clf() local tests = {"10/Feb/2014:08:46:36 -0800"} local results = {1392050796000000000} - test_valid(tc, dt.clf_timestamp, tests, results) + test_valid(dt.clf_timestamp, tests, results) end local function rfc3164() local tests = {"Feb 10 16:46:36"} - local results = {1392050796000000000} - test_valid(tc, dt.rfc3164_timestamp, tests, results) + local results = {1423586796000000000} + test_valid(dt.rfc3164_timestamp, tests, results) end local function mysql() local tests = {"20140210164636"} local results = {1392050796000000000} - test_valid(tc, dt.mysql_timestamp, tests, results) + test_valid(dt.mysql_timestamp, tests, results) end local function pgsql() local tests = {"2014-02-10 16:46:36"} local results = {1392050796000000000} - test_valid(tc, dt.pgsql_timestamp, tests, results) + test_valid(dt.pgsql_timestamp, tests, results) end local function strftime_all() @@ -125,7 +125,7 @@ local function strftime_invalid() end end -function process(tc) +function process() rfc3339() rfc3339_invalid() clf() diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua index 017aa33..f2b5616 100644 --- a/src/test/lua/lpeg_syslog.lua +++ b/src/test/lua/lpeg_syslog.lua @@ -9,7 +9,7 @@ local function traditional_file_format() local log = 'Feb 9 14:17:01 trink-x230 CRON[20758]: (root) CMD ( cd / && run-parts --report /etc/cron.hourly)\n' local fields = grammar:match(log) assert(fields.msg == "(root) CMD ( cd / && run-parts --report /etc/cron.hourly)", fields.msg) - assert(fields.timestamp == 1391955421000000000, fields.timestamp) + assert(fields.timestamp == 1423491421000000000, fields.timestamp) assert(fields.syslogtag.pid == 20758, fields.syslogtag.pid) assert(fields.syslogtag.programname == "CRON", fields.syslogtag.programname) assert(fields.hostname == "trink-x230", fields.hostname) @@ -42,7 +42,7 @@ local function traditional_forward_format() local log = '<6>Feb 10 08:38:47 trink-x230 kernel: imklog 5.8.6, log source = /proc/kmsg started.' local fields = grammar:match(log) assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) - assert(fields.timestamp == 1392021527000000000, fields.timestamp) + assert(fields.timestamp == 1423557527000000000, fields.timestamp) assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) assert(fields.pri.severity == 6, fields.pri.severity) assert(fields.pri.facility == 0, fields.pri.facility) @@ -67,7 +67,7 @@ local function kitchen_sink() assert(fields["syslogfacility-text"] == 0, fields["syslogfacility-text"]) assert(fields.syslogpriority == 6, fields.syslogpriority) assert(fields["syslogpriority-text"] == 6, fields["syslogpriority-text"]) - assert(fields.timegenerated == 1392024053000000000, fields.timegenerated) + assert(fields.timegenerated == 1423560053000000000, fields.timegenerated) -- time reported is mapped to timestamp assert(fields.timestamp == 1392052853559934000, fields.timestamp) assert(fields["protocol-version"] == "0", fields["protocol-version"]) From 10d5ce4f989b309b21fe537c9653ca8f209de3a4 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 8 Jan 2015 09:26:30 -0800 Subject: [PATCH 028/235] Allow hyphens in the upstream response_time and status arrays https://github.com/mozilla-services/heka/issues/1251 --- modules/common_log_format.lua | 10 ++++++---- src/test/lua/lpeg_clf.lua | 8 +++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index 51be364..5d31c4a 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -47,9 +47,11 @@ local nginx_upstream_sep = ", " local nginx_upstream_gsep = " : " local nginx_upstream_addr = l.C((1 - l.S(nginx_upstream_sep))^1) local nginx_upstream_addrs = nginx_upstream_addr * (nginx_upstream_sep * nginx_upstream_addr)^0 -local nginx_upstream_times = double * (nginx_upstream_sep * double)^0 +local nginx_upstream_time = double + l.P"-" / function () return 0 end +local nginx_upstream_times = nginx_upstream_time * (nginx_upstream_sep * nginx_upstream_time)^0 local nginx_upstream_lengths = integer * (nginx_upstream_sep * integer)^0 -local nginx_upstream_statuses = http_status * (nginx_upstream_sep * http_status)^0 +local nginx_upstream_status = http_status + l.P"-" / function () return 0 end +local nginx_upstream_statuses = nginx_upstream_status * (nginx_upstream_sep * nginx_upstream_status)^0 local nginx_format_variables = { --arg_* @@ -110,8 +112,8 @@ local nginx_format_variables = { , upstream_cache_status = l.P"HIT" + "MISS" + "EXPIRED" + "BYPASS" + "STALE" + "UPDATING" + "REVALIDATED" + "-" , upstream_cache_last_modified = dt.build_strftime_grammar("%a, %d %b %Y %T GMT") / dt.time_to_ns + "-" , upstream_response_length = l.Ct(l.Cg(l.Ct(nginx_upstream_lengths * (nginx_upstream_gsep * nginx_upstream_lengths)^0), "value") * l.Cg(l.Cc"B", "representation")) + "-" - , upstream_response_time = l.Ct(l.Cg(l.Ct(nginx_upstream_times * (nginx_upstream_gsep * nginx_upstream_times)^0), "value") * l.Cg(l.Cc"s", "representation")) + "-" - , upstream_status = l.Ct(nginx_upstream_statuses * (nginx_upstream_gsep * nginx_upstream_statuses)^0) + "-" + , upstream_response_time = l.Ct(l.Cg(l.Ct(nginx_upstream_times * (nginx_upstream_gsep * nginx_upstream_times)^0), "value") * l.Cg(l.Cc"s", "representation")) + , upstream_status = l.Ct(nginx_upstream_statuses * (nginx_upstream_gsep * nginx_upstream_statuses)^0) --upstream_http_* handled by the generic grammar } diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index 6488216..33e1309 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -400,13 +400,13 @@ end local function nginx_upstream() local addrs = {"192.168.1.1:80", "192.168.1.2:80", "unix:/tmp/sock", "192.168.10.1:80", "192.168.10.2:80"} local lengths = {1, 2, 3, 4, 5} - local times = {1.1, 1.2, 1.3, 1.4, 1.5} - local statuses = {200, 201, 202, 203, 204} + local times = {1.1, 1.2, 1.3, 1.4, "-"} + local statuses = {200, 201, 202, 203, "-"} local uscs = "HIT" local header = "header field" local grammar = clf.build_nginx_grammar('$upstream_addr $upstream_cache_status $upstream_response_length $upstream_response_time $upstream_status "$upstream_http_test" $upstream_cache_last_modified') - local log = string.format('%s, %s, %s : %s, %s %s %d, %d, %d : %d, %d %g, %g, %g : %g, %g %d, %d, %d : %d, %d "%s" Mon, 28 Sep 1970 06:00:00 GMT', + local log = string.format('%s, %s, %s : %s, %s %s %d, %d, %d : %d, %d %g, %g, %g : %g, %s %d, %d, %d : %d, %s "%s" Mon, 28 Sep 1970 06:00:00 GMT', addrs[1], addrs[2], addrs[3], addrs[4], addrs[5], uscs, lengths[1], lengths[2], lengths[3], lengths[4], lengths[5], times[1], times[2], times[3], times[4], times[5], @@ -442,6 +442,7 @@ local function nginx_upstream() error(string.format("upstream_response_time representation = '%s'", fields.upstream_response_time.representation)) end for i, v in ipairs(times) do + if v == "-" then v = 0 end if fields.upstream_response_time.value[i] ~= v then error(string.format("expected value: %g received: %g", v, fields.upstream_response_time.value[i])) end @@ -451,6 +452,7 @@ local function nginx_upstream() error(string.format("#upstream_status = %d", #fields.upstream_status)) end for i, v in ipairs(statuses) do + if v == "-" then v = 0 end if fields.upstream_status[i] ~= v then error(string.format("expected value: %d received: %d", v, fields.upstream_status[i])) end From 226d1ad7b770a96b58279c63e48317414af04fab Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 9 Jan 2015 12:04:44 -0800 Subject: [PATCH 029/235] Fix segfault in protobuf varint encoding https://github.com/mozilla-services/heka/issues/1269 --- src/lua_serialize_protobuf.c | 10 +++++----- src/lua_serialize_protobuf.h | 2 +- src/test/lua/output.lua | 3 +++ src/test/test_lua_sandbox.c | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/lua_serialize_protobuf.c b/src/lua_serialize_protobuf.c index de78aa8..ec07a22 100644 --- a/src/lua_serialize_protobuf.c +++ b/src/lua_serialize_protobuf.c @@ -33,11 +33,11 @@ int serialize_table_as_pb(lua_sandbox* lsb, int index) // use existing or create a timestamp lua_getfield(lsb->lua, index, "Timestamp"); - long long ts; + unsigned long long ts; if (lua_isnumber(lsb->lua, -1)) { - ts = (long long)lua_tonumber(lsb->lua, -1); + ts = (unsigned long long)lua_tonumber(lsb->lua, -1); } else { - ts = (long long)(time(NULL) * 1e9); + ts = (unsigned long long)(time(NULL) * 1e9); } lua_pop(lsb->lua, 1); if (pb_write_tag(d, 2, 0)) return 1; @@ -63,7 +63,7 @@ int serialize_table_as_pb(lua_sandbox* lsb, int index) } -int pb_write_varint(output_data* d, long long i) +int pb_write_varint(output_data* d, unsigned long long i) { size_t needed = 10; if (needed > d->size - d->pos) { @@ -167,7 +167,7 @@ int encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, int result = 0; lua_getfield(lsb->lua, index, name); if (lua_isnumber(lsb->lua, -1)) { - long long i = (long long)lua_tonumber(lsb->lua, -1); + unsigned long long i = (unsigned long long)lua_tonumber(lsb->lua, -1); if (!(result = pb_write_tag(d, id, 0))) { result = pb_write_varint(d, i); } diff --git a/src/lua_serialize_protobuf.h b/src/lua_serialize_protobuf.h index ffc6c2d..2de9461 100644 --- a/src/lua_serialize_protobuf.h +++ b/src/lua_serialize_protobuf.h @@ -28,7 +28,7 @@ int serialize_table_as_pb(lua_sandbox* lsb, int index); * * @return int Zero on success, non-zero if out of memory. */ -int pb_write_varint(output_data* d, long long i); +int pb_write_varint(output_data* d, unsigned long long i); /** * Writes a double to the output buffer. diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 5dd6ad6..9ffd331 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -34,6 +34,9 @@ function process(tc) elseif tc == 8 then -- heka message force memmove local hm = {Timestamp = 1e9, Fields = {string="0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"}} write_message(hm) + elseif tc == 9 then -- heka negative values + local hm = {Timestamp = -1, Pid = -1, Severity = -1} + write_message(hm) end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 8d56a51..2094dea 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -431,6 +431,7 @@ static char* test_output() , "\x10\x80\x94\xeb\xdc\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33" #endif , "\x10\x80\x94\xeb\xdc\x03\x52\x8d\x01\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x82\x01\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" + , "\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x40\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" , NULL }; From 194c1008534ee0f34be8d55c030d1597e78f3696 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 9 Jan 2015 13:14:25 -0800 Subject: [PATCH 030/235] Work in progress --- docs/sandbox_api.md | 2 +- include/lua_sandbox.h | 229 ----------------------------- src/CMakeLists.txt | 1 + src/lua_bloom_filter.c | 6 +- src/lua_bloom_filter.h | 2 +- src/lua_circular_buffer.c | 32 ++-- src/lua_hyperloglog.c | 6 +- src/lua_sandbox.c | 6 +- src/lua_sandbox_private.c | 117 +-------------- src/lua_sandbox_private.h | 62 ++------ src/lua_serialize.c | 24 +-- src/lua_serialize.h | 5 + src/lua_serialize_protobuf.c | 275 +++++++++++++++++++++++++---------- src/lua_serialize_protobuf.h | 160 -------------------- src/test/lua/output.lua | 3 + src/test/test_lua_sandbox.c | 4 +- 16 files changed, 267 insertions(+), 667 deletions(-) delete mode 100644 include/lua_sandbox.h diff --git a/docs/sandbox_api.md b/docs/sandbox_api.md index 597cf43..c6f276a 100644 --- a/docs/sandbox_api.md +++ b/docs/sandbox_api.md @@ -59,7 +59,7 @@ By default only the base library is loaded additional libraries must be explicit ____ **output(arg0, arg1, ...argN)** - Appends data to the output buffer, which cannot exceed the output_limit + lsb_appends data to the output buffer, which cannot exceed the output_limit configuration parameter. See lsb_get_output() to connect the output to the host application. diff --git a/include/lua_sandbox.h b/include/lua_sandbox.h deleted file mode 100644 index 28b2a2a..0000000 --- a/include/lua_sandbox.h +++ /dev/null @@ -1,229 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// Generic Lua sandbox for dynamic data analysis @file -#ifndef lua_sandbox_h_ -#define lua_sandbox_h_ - -#include - -typedef enum { - LSB_UNKNOWN = 0, - LSB_RUNNING = 1, - LSB_TERMINATED = 2 -} lsb_state; - -typedef enum { - LSB_US_LIMIT = 0, - LSB_US_CURRENT = 1, - LSB_US_MAXIMUM = 2, - - LSB_US_MAX -} lsb_usage_stat; - -typedef enum { - LSB_UT_MEMORY = 0, - LSB_UT_INSTRUCTION = 1, - LSB_UT_OUTPUT = 2, - - LSB_UT_MAX -} lsb_usage_type; - -#define LSB_ERROR_SIZE 256 - -#ifdef _WIN32 -#if defined(luasandbox_EXPORTS) -#define LSB_EXPORT __declspec(dllexport) -#else -#define LSB_EXPORT __declspec(dllimport) -#endif -#else -#define LSB_EXPORT -#endif - -typedef struct lua_sandbox lua_sandbox; - -/** - * Allocates and initializes the structure around the Lua sandbox. - * - * @param parent Pointer to associate the owner to this sandbox. - * @param lua_file Filename of the Lua script to run in this sandbox. - * @param require_path Location of the common sandbox modules - * @param memory_limit Sets the sandbox memory limit (bytes). - * @param instruction_limit Sets the sandbox Lua instruction limit (count). - * This limit is per call to process_message or timer_event - * @param output_limit Sets the single message payload limit (bytes). This - * limit applies to the in memory output buffer. The buffer is reset back - * to zero when inject_message is called. - * @return lua_sandbox Sandbox pointer or NULL on failure. - */ -LSB_EXPORT lua_sandbox* lsb_create(void* parent, - const char* lua_file, - const char* require_path, - unsigned memory_limit, - unsigned instruction_limit, - unsigned output_limit); - -/** - * Initializes the Lua sandbox and loads/runs the Lua script that was specified - * in lua_create_sandbox. - * - * @param lsb Pointer to the sandbox. - * @param state_file Filename where the global data is read. Use a NULL or empty - * string for no data restoration. The global - * _PRESERVATION_VERSION variable will be examined during - * restoration; if the previous version does not match the - * current version the restoration will be aborted and the - * sandbox will start cleanly. _PRESERVATION_VERSION should be - * incremented any time an incompatible change is made to the - * global data schema. If no version is set the check will - * always succeed and a version of zero is assigned. - * - * @return int Zero on success, non-zero on failure. - */ -LSB_EXPORT int lsb_init(lua_sandbox* lsb, const char* state_file); - -/** - * Frees the memory associated with the sandbox. - * - * @param lsb Sandbox pointer to discard. - * @param state_file Filename where the sandbox global data is saved. Use a - * NULL or empty string for no preservation. - * - * @return NULL on success, pointer to an error message on failure that MUST BE - * FREED by the caller. - */ -LSB_EXPORT char* lsb_destroy(lua_sandbox* lsb, const char* state_file); - -/** - * Retrieve the sandbox usage statistics. - * - * @param lsb Pointer to the sandbox. - * @param lsb_usage_type Type of statistic to retrieve i.e. memory. - * @param lsb_usage_stat Type of statistic to retrieve i.e. current. - * - * @return unsigned Count or number of bytes depending on the statistic. - */ -LSB_EXPORT unsigned -lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, lsb_usage_stat ustat); -/** - * Retrieve the current sandbox status. - * - * @param lsb Pointer to the sandbox. - * - * @return lsb_state code - */ -LSB_EXPORT lsb_state lsb_get_state(lua_sandbox* lsb); - -/** - * Return the last error in human readable form. - * - * @param lsb Pointer to the sandbox. - * - * @return const char* error message - */ -LSB_EXPORT const char* lsb_get_error(lua_sandbox* lsb); - -/** - * Sets the last error string. - * - * @param lsb Pointer to the sandbox. - * @param err Error message. - * - * @return const char* error message - */ -LSB_EXPORT void lsb_set_error(lua_sandbox* lsb, const char* err); - -/** - * Access the Lua pointer. - * - * @param lsb Pointer to the sandbox. - * - * @return lua_State* The lua_State pointer. - */ -LSB_EXPORT lua_State* lsb_get_lua(lua_sandbox* lsb); - -/** - * Access the parent pointer stored in the sandbox. - * - * @param lsb Pointer to the sandbox. - * - * @return void* The parent pointer passed to init. - */ -LSB_EXPORT void* lsb_get_parent(lua_sandbox* lsb); - -/** - * Create a CFunction for use by the Sandbox. The Lua sandbox pointer is pushed - * to upvalue index 1. - * - * @param lsb Pointer to the sandbox. - * @param func Lua CFunction pointer. - * @param func_name Function name exposed to the Lua sandbox. - */ -LSB_EXPORT void lsb_add_function(lua_sandbox* lsb, lua_CFunction func, - const char* func_name); - -/** - * Retrieve the data in the output buffer and reset the buffer. The returned - * output string will remain valid until additional sandbox output is performed. - * The output should be copied if the application needs to hold onto it. - * - * @param lsb Pointer to the sandbox. - * @param len If len is not NULL, it will be set to the length of the string. - * - * @return const char* Pointer to the output buffer. - */ -LSB_EXPORT const char* lsb_get_output(lua_sandbox* lsb, size_t* len); - -/** - * Write an array of variables on the Lua stack to the output buffer. - * - * @param lsb Pointer to the sandbox. - * @param start Lua stack index of first variable. - * @param end Lua stack index of the last variable. - * @param append 0 to overwrite the output buffer, 1 to append the output to it - * - * - */ -LSB_EXPORT void lsb_output(lua_sandbox* lsb, int start, int end, int append); - -/** - * Write a Lua table (in a Heka protobuf structure) to the output buffer. - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index of the table. - * @param append 0 to overwrite the output buffer, 1 to append the output to it - * - * @return int 0 on success - */ -LSB_EXPORT int lsb_output_protobuf(lua_sandbox* lsb, int index, int append); - -/** - * Helper function to load the Lua function and set the instruction limits - * - * @param lsb Pointer to the sandbox. - * @param func_name Name of the function to load - * - * @return int 0 on success - */ -LSB_EXPORT int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name); - -/** - * Helper function to update the statistics after the call - * - * @param lsb Pointer to the sandbox. - */ -LSB_EXPORT void lsb_pcall_teardown(lua_sandbox* lsb); - -/** - * Shutdown the sandbox due to a fatal error. - * - * @param lsb Pointer to the sandbox. - * @param err Reason for termination - */ -LSB_EXPORT void lsb_terminate(lua_sandbox* lsb, const char* err); - -#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0cca751..cde2ce7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ set(LUA_SANDBOX_SRC lua_sandbox.c +lua_output.c lua_sandbox_private.c lua_serialize.c lua_serialize_protobuf.c diff --git a/src/lua_bloom_filter.c b/src/lua_bloom_filter.c index fdc32d2..d1247a9 100644 --- a/src/lua_bloom_filter.c +++ b/src/lua_bloom_filter.c @@ -160,7 +160,7 @@ static int bloom_filter_fromstring(lua_State* lua) int serialize_bloom_filter(const char* key, bloom_filter* bf, output_data* output) { output->pos = 0; - if (appendf(output, + if (lsb_appendf(output, "if %s == nil then %s = bloom_filter.new(%d, %g) end\n", key, key, @@ -169,11 +169,11 @@ int serialize_bloom_filter(const char* key, bloom_filter* bf, output_data* outpu return 1; } - if (appendf(output, "%s:fromstring(\"", key)) { + if (lsb_appendf(output, "%s:fromstring(\"", key)) { return 1; } if (serialize_binary(bf->data, bf->bytes, output)) return 1; - if (appends(output, "\")\n", 3)) { + if (lsb_appends(output, "\")\n", 3)) { return 1; } return 0; diff --git a/src/lua_bloom_filter.h b/src/lua_bloom_filter.h index 9b380c1..302b640 100644 --- a/src/lua_bloom_filter.h +++ b/src/lua_bloom_filter.h @@ -10,7 +10,7 @@ #define lua_bloom_filter_h_ #include -#include "lua_sandbox_private.h" +#include "lsb_output.h" extern const char* lsb_bloom_filter; extern const char* lsb_bloom_filter_table; diff --git a/src/lua_circular_buffer.c b/src/lua_circular_buffer.c index 34673b0..b72718f 100644 --- a/src/lua_circular_buffer.c +++ b/src/lua_circular_buffer.c @@ -864,14 +864,14 @@ int output_circular_buffer_full(circular_buffer* cb, output_data* output) if (row_idx >= cb->rows) row_idx = 0; for (column_idx = 0; column_idx < cb->columns; ++column_idx) { if (column_idx != 0) { - if (appendc(output, '\t')) return 1; + if (lsb_appendc(output, '\t')) return 1; } if (serialize_double(output, cb->values[(row_idx * cb->columns) + column_idx])) { return 1; } } - if (appendc(output, '\n')) return 1; + if (lsb_appendc(output, '\n')) return 1; } return 0; } @@ -896,16 +896,16 @@ int output_circular_buffer_cbufd(lua_State* lua, circular_buffer* cb, if (serialize_double(output, lua_tonumber(lua, -2))) return 1; for (unsigned column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appendc(output, '\t')) return 1; + if (lsb_appendc(output, '\t')) return 1; lua_rawgeti(lua, -1, column_idx); if (LUA_TNIL == lua_type(lua, -1)) { - if (appends(output, not_a_number, 3)) return 1; + if (lsb_appends(output, not_a_number, 3)) return 1; } else { if (serialize_double(output, lua_tonumber(lua, -1))) return 1; } lua_pop(lua, 1); // remove the number } - if (appendc(output, '\n')) return 1; + if (lsb_appendc(output, '\n')) return 1; lua_pop(lua, 1); // remove the value, keep the key } lua_pop(lua, 1); // remove the delta table @@ -928,7 +928,7 @@ int output_circular_buffer(lua_State* lua, circular_buffer* cb, if (cb->ref == LUA_NOREF) return 0; } - if (appendf(output, + if (lsb_appendf(output, "{\"time\":%lld,\"rows\":%d,\"columns\":%d,\"seconds_per_row\":%d,\"column_info\":[", (long long)get_start_time(cb), cb->rows, @@ -940,16 +940,16 @@ int output_circular_buffer(lua_State* lua, circular_buffer* cb, unsigned column_idx; for (column_idx = 0; column_idx < cb->columns; ++column_idx) { if (column_idx != 0) { - if (appendc(output, ',')) return 1; + if (lsb_appendc(output, ',')) return 1; } - if (appendf(output, "{\"name\":\"%s\",\"unit\":\"%s\",\"aggregation\":\"%s\"}", + if (lsb_appendf(output, "{\"name\":\"%s\",\"unit\":\"%s\",\"aggregation\":\"%s\"}", cb->headers[column_idx].name, cb->headers[column_idx].unit, column_aggregation_methods[cb->headers[column_idx].aggregation])) { return 1; } } - if (appends(output, "]}\n", 3)) return 1; + if (lsb_appends(output, "]}\n", 3)) return 1; if (OUTPUT_CBUFD == cb->format) { return output_circular_buffer_cbufd(lua, cb, output); @@ -975,12 +975,12 @@ int serialize_circular_buffer_delta(lua_State* lua, circular_buffer* cb, if (!lua_istable(lua, -1)) { luaL_error(lua, "Invalid delta table structure"); } - if (appendc(output, ' ')) return 1; + if (lsb_appendc(output, ' ')) return 1; if (serialize_double(output, lua_tonumber(lua, -2))) return 1; for (unsigned column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appendc(output, ' ')) return 1; + if (lsb_appendc(output, ' ')) return 1; lua_rawgeti(lua, -1, column_idx); if (serialize_double(output, lua_tonumber(lua, -1))) return 1; lua_pop(lua, 1); // remove the number @@ -1009,7 +1009,7 @@ int serialize_circular_buffer(lua_State* lua, const char* key, if (cb->delta) { delta = ", true"; } - if (appendf(output, + if (lsb_appendf(output, "if %s == nil then %s = circular_buffer.new(%d, %d, %d%s) end\n", key, key, @@ -1022,7 +1022,7 @@ int serialize_circular_buffer(lua_State* lua, const char* key, unsigned column_idx; for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appendf(output, "%s:set_header(%d, \"%s\", \"%s\", \"%s\")\n", + if (lsb_appendf(output, "%s:set_header(%d, \"%s\", \"%s\", \"%s\")\n", key, column_idx + 1, cb->headers[column_idx].name, @@ -1032,7 +1032,7 @@ int serialize_circular_buffer(lua_State* lua, const char* key, } } - if (appendf(output, "%s:fromstring(\"%lld %d", + if (lsb_appendf(output, "%s:fromstring(\"%lld %d", key, (long long)cb->current_time, cb->current_row)) { @@ -1040,7 +1040,7 @@ int serialize_circular_buffer(lua_State* lua, const char* key, } for (unsigned row_idx = 0; row_idx < cb->rows; ++row_idx) { for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appendc(output, ' ')) return 1; + if (lsb_appendc(output, ' ')) return 1; if (serialize_double(output, cb->values[(row_idx * cb->columns) + column_idx])) { return 1; @@ -1050,7 +1050,7 @@ int serialize_circular_buffer(lua_State* lua, const char* key, if (serialize_circular_buffer_delta(lua, cb, output)) { return 1; } - if (appends(output, "\")\n", 3)) { + if (lsb_appends(output, "\")\n", 3)) { return 1; } return 0; diff --git a/src/lua_hyperloglog.c b/src/lua_hyperloglog.c index d11ef47..92e6240 100644 --- a/src/lua_hyperloglog.c +++ b/src/lua_hyperloglog.c @@ -149,16 +149,16 @@ static int hyperloglog_fromstring(lua_State* lua) int serialize_hyperloglog(const char* key, hyperloglog* hll, output_data* output) { output->pos = 0; - if (appendf(output, + if (lsb_appendf(output, "if %s == nil then %s = hyperloglog.new() end\n", key, key)) { return 1; } - if (appendf(output, "%s:fromstring(\"", key)) { + if (lsb_appendf(output, "%s:fromstring(\"", key)) { return 1; } if (serialize_binary(hll, sizeof(hyperloglog) - 1, output)) return 1; - if (appends(output, "\")\n", 3)) { + if (lsb_appends(output, "\")\n", 3)) { return 1; } return 0; diff --git a/src/lua_sandbox.c b/src/lua_sandbox.c index cb140d5..35f9682 100644 --- a/src/lua_sandbox.c +++ b/src/lua_sandbox.c @@ -311,18 +311,18 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) { size_t len; const char* s = lua_tolstring(lsb->lua, i, &len); - if (appends(&lsb->output, s, len)) { + if (lsb_appends(&lsb->output, s, len)) { result = 1; } } break; case LUA_TNIL: - if (appends(&lsb->output, "nil", 3)) { + if (lsb_appends(&lsb->output, "nil", 3)) { result = 1; } break; case LUA_TBOOLEAN: - if (appendf(&lsb->output, "%s", + if (lsb_appendf(&lsb->output, "%s", lua_toboolean(lsb->lua, i) ? "true" : "false")) { result = 1; diff --git a/src/lua_sandbox_private.c b/src/lua_sandbox_private.c index 39b4000..2196c3e 100644 --- a/src/lua_sandbox_private.c +++ b/src/lua_sandbox_private.c @@ -6,6 +6,13 @@ /** @brief Lua sandbox private implementation @file */ +#include "lua_bloom_filter.h" +#include "lua_circular_buffer.h" +#include "lua_hyperloglog.h" +#include "lua_sandbox_private.h" +#include "lua_serialize.h" +#include "lua_serialize_protobuf.h" + #include #include #include @@ -15,13 +22,6 @@ #include #include -#include "lua_bloom_filter.h" -#include "lua_circular_buffer.h" -#include "lua_hyperloglog.h" -#include "lua_sandbox_private.h" -#include "lua_serialize.h" -#include "lua_serialize_protobuf.h" - #ifdef _WIN32 #define PATH_DELIMITER '\\' #else @@ -137,104 +137,6 @@ void update_output_stats(lua_sandbox* lsb) } -int appendf(output_data* output, const char* fmt, ...) -{ - va_list args; - int result = 0; - int remaining = 0; - char* ptr = NULL, *old_ptr = NULL; - do { - ptr = output->data + output->pos; - remaining = (int)(output->size - output->pos); - va_start(args, fmt); - int needed = vsnprintf(ptr, remaining, fmt, args); - va_end(args); - if (needed == -1) { - // Windows and Unix have different return values for this function - // -1 on Unix is a format error - // -1 on Windows means the buffer is too small and the required len - // is not returned - needed = remaining; - } - if (needed >= remaining) { - if (output->maxsize - && (output->size >= output->maxsize - || output->pos + needed >= output->maxsize)) { - return 1; - } - size_t newsize = output->size * 2; - while ((size_t)needed >= newsize - output->pos) { - newsize *= 2; - } - if (output->maxsize && newsize > output->maxsize) { - newsize = output->maxsize; - } - void* p = malloc(newsize); - if (p != NULL) { - memcpy(p, output->data, output->pos); - old_ptr = output->data; - output->data = p; - output->size = newsize; - } else { - return 1; // Out of memory condition. - } - } else { - output->pos += needed; - break; - } - } - while (1); - free(old_ptr); - return result; -} - - -int realloc_output(output_data* output, size_t needed) -{ - if (output->maxsize && needed + output->pos > output->maxsize) { - return 1; - } - size_t newsize = output->size * 2; - while (needed >= newsize - output->pos) { - newsize *= 2; - } - if (output->maxsize && newsize > output->maxsize) { - newsize = output->maxsize; - } - - void* ptr = realloc(output->data, newsize); - if (!ptr) return 1; - output->data = ptr; - output->size = newsize; - return 0; -} - - -int appends(output_data* output, const char* str, size_t len) -{ - size_t needed = len + 1; - if (output->size - output->pos < needed) { - if (realloc_output(output, needed)) return 1; - } - memcpy(output->data + output->pos, str, len); - output->pos += len; - output->data[output->pos] = 0; - return 0; -} - - -int appendc(output_data* output, char ch) -{ - size_t needed = 2; - if (output->size - output->pos < needed) { - if (realloc_output(output, needed)) return 1; - } - output->data[output->pos++] = ch; - output->data[output->pos] = 0; - return 0; -} - - int output(lua_State* lua) { void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); @@ -251,11 +153,6 @@ int output(lua_State* lua) return 0; } -LUALIB_API int luaopen_cjson(lua_State* L); -LUALIB_API int luaopen_lpeg(lua_State* L); -LUALIB_API int luaopen_struct(lua_State* L); -int set_encode_max_buffer(lua_State* L, int index, unsigned maxsize); - int require_library(lua_State* lua) { const char* name = luaL_checkstring(lua, 1); diff --git a/src/lua_sandbox_private.h b/src/lua_sandbox_private.h index 9d41f06..67614c0 100644 --- a/src/lua_sandbox_private.h +++ b/src/lua_sandbox_private.h @@ -8,24 +8,16 @@ #ifndef lua_sandbox_private_h_ #define lua_sandbox_private_h_ -#include -#include -#include "lua_sandbox.h" +#include "lsb_output.h" +#include "lsb.h" -#define OUTPUT_SIZE 1024 +#include +#include #ifdef _WIN32 #define snprintf _snprintf #endif -typedef struct -{ - size_t maxsize; - size_t size; - size_t pos; - char* data; -} output_data; - struct lua_sandbox { lua_State* lua; void* parent; @@ -102,46 +94,6 @@ void sandbox_terminate(lua_sandbox* lsb); */ void update_output_stats(lua_sandbox* lsb); -/** - * Append formatted string to the output stream. - * - * @param output Pointer the output collector. - * @param fmt Printf format specifier. - * - * @return int Zero on success, non-zero if out of memory. - */ -int appendf(output_data* output, const char* fmt, ...); - -/** - * Resize the output buffer when more space is needed. - * - * @param output Output buffer to resize. - * @param needed Number of additional bytes needed. - * - * @return int Zero on success, non-zero on failure. - */ -int realloc_output(output_data* output, size_t needed); - -/** - * Append a fixed string to the output stream. - * - * @param output Pointer the output collector. - * @param str String to append to the output. - * @param len Length of the string to append - * - * @return int Zero on success, non-zero if out of memory. - */ -int appends(output_data* output, const char* str, size_t len); - -/** - * Append a character to the output stream. - * - * @param output Pointer the output collector. - * @param ch Character to append to the output. - * - * @return int Zero on success, non-zero if out of memory. - */ -int appendc(output_data* output, char ch); //////////////////////////////////////////////////////////////////////////////// /// Lua to C function interface @@ -165,4 +117,10 @@ int output(lua_State* lua); */ int require_library(lua_State* lua); +LUALIB_API int luaopen_cjson(lua_State* L); +int set_encode_max_buffer(lua_State* L, int index, unsigned maxsize); + +LUALIB_API int luaopen_lpeg(lua_State* L); +LUALIB_API int luaopen_struct(lua_State* L); + #endif diff --git a/src/lua_serialize.c b/src/lua_serialize.c index 60826b3..eeaded2 100644 --- a/src/lua_serialize.c +++ b/src/lua_serialize.c @@ -97,7 +97,7 @@ int preserve_global_data(lua_sandbox* lsb, const char* data_file) preservation_version, preservation_version, get_preservation_version(lsb->lua)); - appendf(&data.keys, "%s", G); + lsb_appendf(&data.keys, "%s", G); data.keys.pos += 1; data.globals = lua_topointer(lsb->lua, -1); lua_checkstack(lsb->lua, 2); @@ -147,10 +147,10 @@ int preserve_global_data(lua_sandbox* lsb, const char* data_file) int serialize_double(output_data* output, double d) { if (isnan(d)) { - return appends(output, not_a_number, 3); + return lsb_appends(output, not_a_number, 3); } if (d > INT_MAX) { - return appendf(output, "%0.17g", d); + return lsb_appendf(output, "%0.17g", d); } const int precision = 8; @@ -205,7 +205,7 @@ int serialize_double(output_data* output, double d) size_t remaining = output->size - output->pos; size_t needed = (p - buffer) + negative; if (needed >= remaining) { - if (realloc_output(output, needed)) return 1; + if (lsb_realloc_output(output, needed)) return 1; } if (negative) { @@ -261,7 +261,7 @@ int serialize_data(lua_sandbox* lsb, int index, output_data* output) lua_pushstring(lsb->lua, "%q"); lua_pushvalue(lsb->lua, index - 3); if (lua_pcall(lsb->lua, 2, 1, 0) == 0) { - if (appendf(output, "%s", lua_tostring(lsb->lua, -1))) { + if (lsb_appendf(output, "%s", lua_tostring(lsb->lua, -1))) { lua_pop(lsb->lua, 1); // Remove the string table. return 1; } @@ -279,7 +279,7 @@ int serialize_data(lua_sandbox* lsb, int index, output_data* output) lua_pop(lsb->lua, 2); // Remove the pcall result and the string table. break; case LUA_TBOOLEAN: - if (appendf(output, "%s", + if (lsb_appendf(output, "%s", lua_toboolean(lsb->lua, index) ? "true" : "false")) { return 1; @@ -324,7 +324,7 @@ int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) } size_t pos = data->keys.pos; - if (appendf(&data->keys, "%s[%s]", data->keys.data + parent, + if (lsb_appendf(&data->keys, "%s[%s]", data->keys.data + parent, lsb->output.data)) { return 1; } @@ -557,19 +557,19 @@ int serialize_binary(const void* src, size_t len, output_data* output) for (unsigned i = 0; i < len; ++i) { switch (uc[i]) { case '\n': - if (appends(output, "\\n", 2)) return 1; + if (lsb_appends(output, "\\n", 2)) return 1; break; case '\r': - if (appends(output, "\\r", 2)) return 1; + if (lsb_appends(output, "\\r", 2)) return 1; break; case '"': - if (appends(output, "\\\"", 2)) return 1; + if (lsb_appends(output, "\\\"", 2)) return 1; break; case '\\': - if (appends(output, "\\\\", 2)) return 1; + if (lsb_appends(output, "\\\\", 2)) return 1; break; default: - if (appendc(output, uc[i])) return 1; + if (lsb_appendc(output, uc[i])) return 1; break; } } diff --git a/src/lua_serialize.h b/src/lua_serialize.h index ea2f5da..19ececb 100644 --- a/src/lua_serialize.h +++ b/src/lua_serialize.h @@ -14,6 +14,11 @@ extern const char* not_a_number; +typedef int (*lsb_SerializeFunction) (lua_State *L, + const char* key, + void* ud, + output_data* output); + typedef struct { const void* ptr; diff --git a/src/lua_serialize_protobuf.c b/src/lua_serialize_protobuf.c index de78aa8..ebe9445 100644 --- a/src/lua_serialize_protobuf.c +++ b/src/lua_serialize_protobuf.c @@ -12,62 +12,19 @@ #include #include - -int serialize_table_as_pb(lua_sandbox* lsb, int index) -{ - output_data* d = &lsb->output; - d->pos = 0; - size_t needed = 18; - if (needed > d->size) { - if (realloc_output(d, needed)) return 1; - } - - // create a type 4 uuid - d->data[d->pos++] = 2 | (1 << 3); - d->data[d->pos++] = 16; - for (int x = 0; x < 16; ++x) { - d->data[d->pos++] = rand() % 255; - } - d->data[8] = (d->data[8] & 0x0F) | 0x40; - d->data[10] = (d->data[10] & 0x0F) | 0xA0; - - // use existing or create a timestamp - lua_getfield(lsb->lua, index, "Timestamp"); - long long ts; - if (lua_isnumber(lsb->lua, -1)) { - ts = (long long)lua_tonumber(lsb->lua, -1); - } else { - ts = (long long)(time(NULL) * 1e9); - } - lua_pop(lsb->lua, 1); - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, ts)) return 1; - - if (encode_string(lsb, d, 3, "Type", index)) return 1; - if (encode_string(lsb, d, 4, "Logger", index)) return 1; - if (encode_int(lsb, d, 5, "Severity", index)) return 1; - if (encode_string(lsb, d, 6, "Payload", index)) return 1; - if (encode_string(lsb, d, 7, "EnvVersion", index)) return 1; - if (encode_int(lsb, d, 8, "Pid", index)) return 1; - if (encode_string(lsb, d, 9, "Hostname", index)) return 1; - if (encode_fields(lsb, d, 10, "Fields", index)) return 1; - // if we go above 15 pb_write_tag will need to start varint encoding - needed = 1; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - d->data[d->pos] = 0; // NULL terminate incase someone tries to treat this - // as a string - - return 0; -} - - -int pb_write_varint(output_data* d, long long i) +/** + * Writes a varint encoded number to the output buffer. + * + * @param d Pointer to the output data buffer. + * @param i Number to be encoded. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int pb_write_varint(output_data* d, unsigned long long i) { size_t needed = 10; if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; + if (lsb_realloc_output(d, needed)) return 1; } if (i == 0) { @@ -84,11 +41,19 @@ int pb_write_varint(output_data* d, long long i) } -int pb_write_double(output_data* d, double i) +/** + * Writes a double to the output buffer. + * + * @param d Pointer to the output data buffer. + * @param i Double to be encoded. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int pb_write_double(output_data* d, double i) { size_t needed = sizeof(double); if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; + if (lsb_realloc_output(d, needed)) return 1; } // todo add big endian support if necessary @@ -98,11 +63,19 @@ int pb_write_double(output_data* d, double i) } -int pb_write_bool(output_data* d, int i) +/** + * Writes a bool to the output buffer. + * + * @param d Pointer to the output data buffer. + * @param i Number to be encoded. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int pb_write_bool(output_data* d, int i) { size_t needed = 1; if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; + if (lsb_realloc_output(d, needed)) return 1; } if (i) { @@ -114,11 +87,20 @@ int pb_write_bool(output_data* d, int i) } -int pb_write_tag(output_data* d, char id, char wire_type) +/** + * Writes a field tag (tag id/wire type) to the output buffer. + * + * @param d Pointer to the output data buffer. + * @param id Field identifier. + * @param wire_type Field wire type. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int pb_write_tag(output_data* d, char id, char wire_type) { size_t needed = 1; if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; + if (lsb_realloc_output(d, needed)) return 1; } d->data[d->pos++] = wire_type | (id << 3); @@ -126,7 +108,17 @@ int pb_write_tag(output_data* d, char id, char wire_type) } -int pb_write_string(output_data* d, char id, const char* s, size_t len) +/** + * Writes a string to the output buffer. + * + * @param d Pointer to the output data buffer. + * @param id Field identifier. + * @param s String to output. + * @param len Length of s. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int pb_write_string(output_data* d, char id, const char* s, size_t len) { if (pb_write_tag(d, id, 2)) { @@ -138,7 +130,7 @@ int pb_write_string(output_data* d, char id, const char* s, size_t len) size_t needed = len; if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; + if (lsb_realloc_output(d, needed)) return 1; } memcpy(&d->data[d->pos], s, len); d->pos += len; @@ -146,8 +138,21 @@ int pb_write_string(output_data* d, char id, const char* s, size_t len) } -int encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index) +/** + * Retrieve the string value for a Lua table entry (the table should be on top + * of the stack). If the entry is not found or not a string nothing is encoded. + * + * @param lsb Pointer to the sandbox. + * @param d Pointer to the output data buffer. + * @param id Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int +encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, + int index) { int result = 0; lua_getfield(lsb->lua, index, name); @@ -161,13 +166,27 @@ int encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, } -int encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index) +/** + * Retrieve the numeric value for a Lua table entry (the table should be on top + * of the stack). If the entry is not found or not a number nothing is encoded, + * otherwise the number is varint encoded. + * + * @param lsb Pointer to the sandbox. + * @param d Pointer to the output data buffer. + * @param id Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int +encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, + int index) { int result = 0; lua_getfield(lsb->lua, index, name); if (lua_isnumber(lsb->lua, -1)) { - long long i = (long long)lua_tonumber(lsb->lua, -1); + unsigned long long i = (unsigned long long)lua_tonumber(lsb->lua, -1); if (!(result = pb_write_tag(d, id, 0))) { result = pb_write_varint(d, i); } @@ -177,8 +196,33 @@ int encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, } -int encode_field_array(lua_sandbox* lsb, output_data* d, int t, - const char* representation) +/** + * Encodes the field value. + * + * @param lsb Pointer to the sandbox. + * @param d Pointer to the output data buffer. + * @param first Boolean indicator used to add addition protobuf data in the + * correct order. + * @param representation String representation of the field i.e., "ms" + * + * @return int Zero on success, non-zero if out of memory. + */ +static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, + const char* representation); + + +/** + * Encodes a field that has an array of values. + * + * @param lsb Pointer to the sandbox. + * @param d Pointer to the output data buffer. + * @param ltype Lua type of the array values. + * @param representation String representation of the field i.e., "ms" + * + * @return int Zero on success, non-zero if out of memory. + */ +static int encode_field_array(lua_sandbox* lsb, output_data* d, int t, + const char* representation) { int result = 0, first = (int)lua_objlen(lsb->lua, -1); lua_checkstack(lsb->lua, 2); @@ -197,7 +241,15 @@ int encode_field_array(lua_sandbox* lsb, output_data* d, int t, } -int encode_field_object(lua_sandbox* lsb, output_data* d) +/** + * Encodes a field that contains metadata in addition to its value. + * + * @param lsb Pointer to the sandbox. + * @param d Pointer to the output data buffer. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int encode_field_object(lua_sandbox* lsb, output_data* d) { int result = 0; const char* representation = NULL; @@ -212,8 +264,8 @@ int encode_field_object(lua_sandbox* lsb, output_data* d) } -int encode_field_value(lua_sandbox* lsb, output_data* d, int first, - const char* representation) +static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, + const char* representation) { int result = 1; size_t len; @@ -301,7 +353,17 @@ int encode_field_value(lua_sandbox* lsb, output_data* d, int first, } -int update_field_length(output_data* d, size_t len_pos) +/** + * Updates the field length in the output buffer once the size is known, this + * allows for single pass encoding. + * + * @param d Pointer to the output data buffer. + * @param len_pos Position in the output buffer where the length should be + * written. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int update_field_length(output_data* d, size_t len_pos) { size_t len = d->pos - len_pos - 1; if (len < 128) { @@ -315,7 +377,7 @@ int update_field_length(output_data* d, size_t len_pos) } size_t needed = cnt - 1; if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; + if (lsb_realloc_output(d, needed)) return 1; } size_t end_pos = d->pos + needed; memmove(&d->data[len_pos + cnt], &d->data[len_pos + 1], len); @@ -326,8 +388,21 @@ int update_field_length(output_data* d, size_t len_pos) } -int encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index) +/** + * Iterates over the specified Lua table encoding the contents as user defined + * message fields. + * + * @param lsb Pointer to the sandbox. + * @param d Pointer to the output data buffer. + * @param id Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int +encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, + int index) { int result = 0; lua_getfield(lsb->lua, index, name); @@ -358,3 +433,53 @@ int encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, lua_pop(lsb->lua, 1); // remove the fields table return result; } + + +int serialize_table_as_pb(lua_sandbox* lsb, int index) +{ + output_data* d = &lsb->output; + d->pos = 0; + size_t needed = 18; + if (needed > d->size) { + if (lsb_realloc_output(d, needed)) return 1; + } + + // create a type 4 uuid + d->data[d->pos++] = 2 | (1 << 3); + d->data[d->pos++] = 16; + for (int x = 0; x < 16; ++x) { + d->data[d->pos++] = rand() % 255; + } + d->data[8] = (d->data[8] & 0x0F) | 0x40; + d->data[10] = (d->data[10] & 0x0F) | 0xA0; + + // use existing or create a timestamp + lua_getfield(lsb->lua, index, "Timestamp"); + unsigned long long ts; + if (lua_isnumber(lsb->lua, -1)) { + ts = (unsigned long long)lua_tonumber(lsb->lua, -1); + } else { + ts = (unsigned long long)(time(NULL) * 1e9); + } + lua_pop(lsb->lua, 1); + if (pb_write_tag(d, 2, 0)) return 1; + if (pb_write_varint(d, ts)) return 1; + + if (encode_string(lsb, d, 3, "Type", index)) return 1; + if (encode_string(lsb, d, 4, "Logger", index)) return 1; + if (encode_int(lsb, d, 5, "Severity", index)) return 1; + if (encode_string(lsb, d, 6, "Payload", index)) return 1; + if (encode_string(lsb, d, 7, "EnvVersion", index)) return 1; + if (encode_int(lsb, d, 8, "Pid", index)) return 1; + if (encode_string(lsb, d, 9, "Hostname", index)) return 1; + if (encode_fields(lsb, d, 10, "Fields", index)) return 1; + // if we go above 15 pb_write_tag will need to start varint encoding + needed = 1; + if (needed > d->size - d->pos) { + if (lsb_realloc_output(d, needed)) return 1; + } + d->data[d->pos] = 0; // NULL terminate incase someone tries to treat this + // as a string + + return 0; +} diff --git a/src/lua_serialize_protobuf.h b/src/lua_serialize_protobuf.h index ffc6c2d..4f6d739 100644 --- a/src/lua_serialize_protobuf.h +++ b/src/lua_serialize_protobuf.h @@ -20,164 +20,4 @@ */ int serialize_table_as_pb(lua_sandbox* lsb, int index); -/** - * Writes a varint encoded number to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Number to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_varint(output_data* d, long long i); - -/** - * Writes a double to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Double to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_double(output_data* d, double i); - -/** - * Writes a bool to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Number to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_bool(output_data* d, int i); - -/** - * Writes a field tag (tag id/wire type) to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param wire_type Field wire type. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_tag(output_data* d, char id, char wire_type); - -/** - * Writes a string to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param s String to output. - * @param len Length of s. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_string(output_data* d, char id, const char* s, size_t len); - -/** - * Retrieve the string value for a Lua table entry (the table should be on top - * of the stack). If the entry is not found or not a string nothing is encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index); - -/** - * Retrieve the numeric value for a Lua table entry (the table should be on top - * of the stack). If the entry is not found or not a number nothing is encoded, - * otherwise the number is varint encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index); - -/** - * Encodes the entry on top of the Lua stack as a double. If the entry is not a - * number nothing is encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_double(lua_sandbox* lsb, output_data* d, char id); - -/** - * Encodes a field that has an array of values. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param ltype Lua type of the array values. - * @param representation String representation of the field i.e., "ms" - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_field_array(lua_sandbox* lsb, output_data* d, int ltype, - const char* representation); - -/** - * Encodes a field that contains metadata in addition to its value. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_field_object(lua_sandbox* lsb, output_data* d); - -/** - * Encodes the field value. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param first Boolean indicator used to add addition protobuf data in the - * correct order. - * @param representation String representation of the field i.e., "ms" - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_field_value(lua_sandbox* lsb, output_data* d, int first, - const char* representation); - -/** - * Updates the field length in the output buffer once the size is known, this - * allows for single pass encoding. - * - * @param d Pointer to the output data buffer. - * @param len_pos Position in the output buffer where the length should be - * written. - * - * @return int Zero on success, non-zero if out of memory. - */ -int update_field_length(output_data* d, size_t len_pos); - -/** - * Iterates over the specified Lua table encoding the contents as user defined - * message fields. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index); - #endif diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 5dd6ad6..9ffd331 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -34,6 +34,9 @@ function process(tc) elseif tc == 8 then -- heka message force memmove local hm = {Timestamp = 1e9, Fields = {string="0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"}} write_message(hm) + elseif tc == 9 then -- heka negative values + local hm = {Timestamp = -1, Pid = -1, Severity = -1} + write_message(hm) end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 8d56a51..6446777 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -6,7 +6,7 @@ /** @brief Lua sandbox unit tests @file */ -#include "lua_sandbox.h" +#include "lsb.h" #include #include @@ -431,6 +431,7 @@ static char* test_output() , "\x10\x80\x94\xeb\xdc\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33" #endif , "\x10\x80\x94\xeb\xdc\x03\x52\x8d\x01\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x82\x01\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" + , "\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x40\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" , NULL }; @@ -1508,7 +1509,6 @@ static char* benchmark_hyperloglog_add() static char* all_tests() { - mu_run_test(test_create_error); mu_run_test(test_init_error); mu_run_test(test_destroy_error); From 6a4ef43ef5ac55a9bded07d5b8fb28c85a7fea6a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 15 Jan 2015 08:58:49 -0800 Subject: [PATCH 031/235] Refactor the sandbox - improve encapsulation with more static functions - add lsb prefix to any exposed defines/functions/types - break out lua modules so they can be used outside the sandbox - add an interface to allow modules to interact with sandbox serialization and output --- cmake/externals.cmake | 39 +- docs/bloom_filter.md | 48 - docs/circular_buffer.md | 207 ---- docs/hyperloglog.md | 49 - docs/sandbox_api.md | 6 +- include/lsb.h | 208 ++++ include/lsb_output.h | 96 ++ include/lsb_serialize.h | 77 ++ src/CMakeLists.txt | 27 +- src/cephes.c | 33 - src/cephes.h | 8 - src/{lua_sandbox.c => lsb.c} | 228 ++-- src/lsb_modules.c | 156 +++ src/lsb_modules.h | 50 + src/lsb_output.c | 234 ++++ src/lsb_private.h | 41 + src/{lua_serialize.c => lsb_serialize.c} | 742 +++++------ ...ze_protobuf.c => lsb_serialize_protobuf.c} | 49 +- ...ze_protobuf.h => lsb_serialize_protobuf.h} | 11 +- src/lua_bloom_filter.c | 207 ---- src/lua_bloom_filter.h | 43 - src/lua_circular_buffer.c | 1091 ----------------- src/lua_circular_buffer.h | 63 - src/lua_hyperloglog.c | 193 --- src/lua_hyperloglog.h | 84 -- src/lua_sandbox_private.c | 243 ---- src/lua_sandbox_private.h | 126 -- src/lua_serialize.h | 173 --- src/redis_hyperloglog.c | 503 -------- src/test/lua/bloom_filter_errors.lua | 43 - src/test/lua/circular_buffer.lua | 230 +--- src/test/lua/circular_buffer_errors.lua | 108 +- src/test/lua/hyperloglog_errors.lua | 30 - src/test/lua/output_errors.lua | 2 + src/test/test_lua_sandbox.c | 146 +-- src/xxhash.c | 476 ------- src/xxhash.h | 165 --- 37 files changed, 1473 insertions(+), 4762 deletions(-) delete mode 100644 docs/bloom_filter.md delete mode 100644 docs/circular_buffer.md delete mode 100644 docs/hyperloglog.md create mode 100644 include/lsb.h create mode 100644 include/lsb_output.h create mode 100644 include/lsb_serialize.h delete mode 100644 src/cephes.c delete mode 100644 src/cephes.h rename src/{lua_sandbox.c => lsb.c} (72%) create mode 100644 src/lsb_modules.c create mode 100644 src/lsb_modules.h create mode 100644 src/lsb_output.c create mode 100644 src/lsb_private.h rename src/{lua_serialize.c => lsb_serialize.c} (64%) rename src/{lua_serialize_protobuf.c => lsb_serialize_protobuf.c} (90%) rename src/{lua_serialize_protobuf.h => lsb_serialize_protobuf.h} (70%) delete mode 100644 src/lua_bloom_filter.c delete mode 100644 src/lua_bloom_filter.h delete mode 100644 src/lua_circular_buffer.c delete mode 100644 src/lua_circular_buffer.h delete mode 100644 src/lua_hyperloglog.c delete mode 100644 src/lua_hyperloglog.h delete mode 100644 src/lua_sandbox_private.c delete mode 100644 src/lua_sandbox_private.h delete mode 100644 src/lua_serialize.h delete mode 100644 src/redis_hyperloglog.c delete mode 100644 src/test/lua/bloom_filter_errors.lua delete mode 100644 src/test/lua/hyperloglog_errors.lua delete mode 100644 src/xxhash.c delete mode 100644 src/xxhash.h diff --git a/cmake/externals.cmake b/cmake/externals.cmake index e44c4e2..1d36246 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -12,8 +12,7 @@ endif() set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) -set(CJSON_PATCH_FILE "lua-cjson-2_1_0.patch") -set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DADDRESS_MODEL=${ADDRESS_MODEL} --no-warn-unused-cli) +set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DADDRESS_MODEL=${ADDRESS_MODEL} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include --no-warn-unused-cli) set(LUA_INCLUDE_DIR "${EP_BASE}/include") if (LUA_JIT) @@ -68,21 +67,49 @@ endif() include_directories(${LUA_INCLUDE_DIR}) externalproject_add( - lpeg-0_12 + lua_lpeg URL http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-0.12.tar.gz URL_MD5 4abb3c28cd8b6565c6a65e88f06c9162 PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/lpeg-0_12.patch CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) -add_dependencies(lpeg-0_12 ${LUA_PROJECT}) +add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( - lua-cjson-2_1_0 + lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git GIT_TAG be85986dc481ad2a0d3abc06ac57e1d1241d9d4c CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) -add_dependencies(lua-cjson-2_1_0 ${LUA_PROJECT}) +add_dependencies(lua_cjson ${LUA_PROJECT}) +externalproject_add( + lua_bloom_filter + GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git + GIT_TAG 9d195247363052bed9d6de7457dfee25a86abf06 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_DIR ${EP_BASE} +) +add_dependencies(lua_bloom_filter ${LUA_PROJECT}) + +externalproject_add( + lua_circular_buffer + URL /work/git/lua_circular_buffer1.tgz + GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git + GIT_TAG 8e8b224a9e64e9d0f95c0c7acdf55fdce34cfa11 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_DIR ${EP_BASE} +) +add_dependencies(lua_circular_buffer ${LUA_PROJECT}) + +externalproject_add( + lua_hyperloglog + URL /work/git/lua_hyperloglog1.tgz + GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git + GIT_TAG 50780c5ce2c83be0510cf27ebfa742c4e74683aa + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_DIR ${EP_BASE} +) +add_dependencies(lua_hyperloglog ${LUA_PROJECT}) diff --git a/docs/bloom_filter.md b/docs/bloom_filter.md deleted file mode 100644 index 8680c48..0000000 --- a/docs/bloom_filter.md +++ /dev/null @@ -1,48 +0,0 @@ -Lua Bloom Filter Library -======================== - -The library is implemented in the _bloom_filter_ table. - -Constructor ------------ -**bloom_filter.new** (items, probability) - -*Arguments* -- items (unsigned) The maximum number of items to be inserted into the filter (must be > 1) -- probability (double) The probability of false positives (must be between 0 and 1) - -*Return* - -A bloom filter object. - -Methods -------- -bool **add** (key) - -*Arguments* -- key (string/number) The key to add to the bloom filter. - -*Return* - -True if the key was added, false if it already existed. - -____ -bool **query** (key) - -*Arguments* -- key (string/number) The key to lookup in the bloom filter. - -*Return* - -True if the key exists, false if it doesn't. - -____ -void **clear** () - -*Arguments* - -none - -*Return* - -none diff --git a/docs/circular_buffer.md b/docs/circular_buffer.md deleted file mode 100644 index db4505b..0000000 --- a/docs/circular_buffer.md +++ /dev/null @@ -1,207 +0,0 @@ -Lua Circular Buffer Library -=========================== - -The library is a sliding window time series data store and is implemented in the _circular_buffer_ table. - -Constructor ------------ -**circular_buffer.new** (rows, columns, seconds_per_row, enable_delta) - -*Arguments* -- rows (unsigned) The number of rows in the buffer (must be > 1) -- columns (unsigned)The number of columns in the buffer (must be > 0) -- seconds_per_row (unsigned) The number of seconds each row represents (must be > 0). -- enable_delta (**optional, default false** bool) When true the changes made to the - circular buffer between delta outputs are tracked. - -*Return* - -A circular buffer object. - -Methods -------- -**Note:** All column arguments are 1 based. If the column is out of range for the configured circular buffer a fatal error is generated. - -double **add** (nanoseconds, column, value) - -*Arguments* -- nanosecond (unsigned) The number of nanosecond since the UNIX epoch. The value is - used to determine which row is being operated on. -- column (unsigned) The column within the specified row to perform an add operation on. -- value (double) The value to be added to the specified row/column. - -*Return* - -The value of the updated row/column or nil if the time was outside the range of the buffer. - -____ -double **set** (nanoseconds, column, value) - -*Arguments* -- nanosecond (unsigned) The number of nanosecond since the UNIX epoch. The value is - used to determine which row is being operated on. -- column (unsigned) The column within the specified row to perform a set operation on. -- value (double) The value to be overwritten at the specified row/column. - For aggregation methods "min" and "max" the value is only overwritten if it is smaller/larger than the current value. - -*Return* - -The resulting value of the row/column or nil if the time was outside the range of the buffer. - -____ -double **get** (nanoseconds, column) - -*Arguments* -- nanosecond (unsigned) The number of nanosecond since the UNIX epoch. The value is used - to determine which row is being operated on. -- column (unsigned) The column within the specified row to retrieve the data from. - -*Return* - -The value at the specifed row/column or nil if the time was outside the range of the buffer. - -____ -double, double, double **get_configuration** () - -*Arguments* -- none - -*Return* - -The circular buffer dimension values specified in the constructor. -- rows -- columns -- seconds_per_row - -____ -int **set_header** (column, name, unit, aggregation_method) - -*Arguments* -- column (unsigned) The column number where the header information is applied. -- name (string) Descriptive name of the column (maximum 15 characters). Any non alpha - numeric characters will be converted to underscores. (default: Column_N) -- unit (string - optional) The unit of measure (maximum 7 characters). Alpha numeric, - '/', and '*' characters are allowed everything else will be converted to underscores. - i.e. KiB, Hz, m/s (default: count) -- aggregation_method (string - optional) Controls how the column data is aggregated - when combining multiple circular buffers. - - **sum** The total is computed for the time/column (default). - - **min** The smallest value is retained for the time/column. - - **max** The largest value is retained for the time/column. - - **none** No aggregation will be performed the column. - -*Return* - -The column number passed into the function. - -____ -string, string, string **get_header** (column) - -*Arguments* -- column (unsigned) The column number of the header information to be retrieved. - -*Return* - -The current values of specified header column. -- name -- unit -- aggregation_method - -____ -double, int **compute** (function, column, start, end) - -*Arguments* -- function (string) The name of the compute function (sum|avg|sd|min|max|variance). -- column (unsigned) The column that the computation is performed against. -- start (optional - unsigned) The number of nanosecond since the UNIX epoch. Sets the - start time of the computation range; if nil the buffer's start time is used. -- end (optional- unsigned) The number of nanosecond since the UNIX epoch. Sets the - end time of the computation range (inclusive); if nil the buffer's end time is used. - The end time must be greater than or equal to the start time. - -*Returns* - -- The result of the computation for the specifed column over the given range or nil if the range fell outside of the buffer. -- The number of rows that contained a valid numeric value. - -____ -double, double **mannwhitneyu** (column, start_x, end_x, start_y, end_y, use_continuity) - -Computes the Mann-Whitney rank test on samples x and y. - -*Arguments* -- column (unsigned) The column that the computation is performed against. -- start_1 (unsigned) The number of nanosecond since the UNIX epoch. -- end_1 (unsigned) The number of nanosecond since the UNIX epoch. The end time must be greater than or equal to the start time. -- start_2 (unsigned). -- end_2 (unsigned). -- use_continuity (optional - bool) Whether a continuity correction (1/2) should be taken into account (default: true). - -*Returns* (nil if the range fell outside the buffer) - -- U_1 Mann-Whitney statistic. -- One-sided p-value assuming a asymptotic normal distribution. - -**Note:** Use only when the number of observation in each sample is > 20 and you have 2 independent samples of ranks. -Mann-Whitney U is significant if the u-obtained is LESS THAN or equal to the critical value of U. - -This test corrects for ties and by default uses a continuity correction. The reported p-value is for a one-sided -hypothesis, to get the two-sided p-value multiply the returned p-value by 2. - -____ -double **current_time** () - -*Arguments* -- none - -*Return* - -The time of the most current row in the circular buffer. - -____ -cbuf **format** (format) - Sets an internal flag to control the output format of the circular buffer data structure; if deltas are not enabled or there haven't been any modifications, nothing is output. - -*Arguments* -- format (string) - - **cbuf** The circular buffer full data set format. - - **cbufd** The circular buffer delta data set format. - -*Return* - -The circular buffer object. - -Output ------- -The circular buffer can be passed to the output() function. The output format -can be selected using the format() function. - -The cbuf (full data set) output format consists of newline delimited rows -starting with a json header row followed by the data rows with tab delimited -columns. The time in the header corresponds to the time of the first data row, -the time for the other rows is calculated using the seconds_per_row header value. - - {json header} - row1_col1\trow1_col2\n - . - . - . - rowN_col1\trowN_col2\n - -The cbufd (delta) output format consists of newline delimited rows starting with -a json header row followed by the data rows with tab delimited columns. The -first column is the timestamp for the row (time_t). The cbufd output will only -contain the rows that have changed and the corresponding delta values for each -column. - - {json header} - row14_timestamp\trow14_col1\trow14_col2\n - row10_timestamp\trow10_col1\trow10_col2\n - -Sample Cbuf Output ------------------- - - {"time":2,"rows":3,"columns":3,"seconds_per_row":60,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"}]} - 10002 0 0 - 11323 0 0 - 10685 0 0 diff --git a/docs/hyperloglog.md b/docs/hyperloglog.md deleted file mode 100644 index 184cf50..0000000 --- a/docs/hyperloglog.md +++ /dev/null @@ -1,49 +0,0 @@ -Lua HyperLogLog Library -======================= - -The library is implemented in the _hyperloglog_ table. - -Constructor ------------ -**hyperloglog.new** () - -*Arguments* - -none - -*Return* - -A HyperLogLog object. - -Methods -------- -bool **add** (key) - -*Arguments* -- key (string/number) The key to add to the HyperLogLog. - -*Return* - -True if add altered the estimate, false if it has not. - -____ -double **count** () - -*Arguments* - -none - -*Return* - -The approximated cardinality of the set. - -____ -void **clear** () - -*Arguments* - -none - -*Return* - -none diff --git a/docs/sandbox_api.md b/docs/sandbox_api.md index c6f276a..74ab361 100644 --- a/docs/sandbox_api.md +++ b/docs/sandbox_api.md @@ -30,8 +30,8 @@ By default only the base library is loaded additional libraries must be explicit - **base library** - The standard require() function is overridden by this version. - Disabled functions: collectgarbage, coroutine, dofile, load, loadfile, loadstring, module, print. - - [bloom_filter](bloom_filter.md) - - [circular_buffer](circular_buffer.md) + - **bloom_filter** https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md + - **circular_buffer** https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md - **cjson** http://www.kyne.com.au/~mark/software/lua-cjson-manual.html. With the following modifications: - Loads the cjson module in a global cjson table - The encode buffer is limited to the sandbox output_limit. @@ -40,7 +40,7 @@ By default only the base library is loaded additional libraries must be explicit If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - The new() function has been disabled so only a single cjson parser can be created. - The encode_keep_buffer() function has been disabled (the buffer is always reused). - - [hyperloglog](hyperloglog.md) + - **hyperloglog** https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md - **lpeg** loads the Lua Parsing Expression Grammar Library http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html - **math** - **os** diff --git a/include/lsb.h b/include/lsb.h new file mode 100644 index 0000000..e995c50 --- /dev/null +++ b/include/lsb.h @@ -0,0 +1,208 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Generic Lua sandbox for dynamic data analysis @file */ + +#ifndef lsb_h_ +#define lsb_h_ + +#include + +#ifdef _WIN32 +#ifdef luasandbox_EXPORTS +#define LSB_EXPORT __declspec(dllexport) +#else +#define LSB_EXPORT __declspec(dllimport) +#endif +#else +#define LSB_EXPORT +#endif + +#define LSB_ERROR_SIZE 256 + +typedef enum { + LSB_UNKNOWN = 0, + LSB_RUNNING = 1, + LSB_TERMINATED = 2 +} lsb_state; + +typedef enum { + LSB_US_LIMIT = 0, + LSB_US_CURRENT = 1, + LSB_US_MAXIMUM = 2, + + LSB_US_MAX +} lsb_usage_stat; + +typedef enum { + LSB_UT_MEMORY = 0, + LSB_UT_INSTRUCTION = 1, + LSB_UT_OUTPUT = 2, + + LSB_UT_MAX +} lsb_usage_type; + +typedef struct lua_sandbox lua_sandbox; +typedef struct lsb_output_data lsb_output_data; + +/** + * Allocates and initializes the structure around the Lua sandbox. + * + * @param parent Pointer to associate the owner to this sandbox. + * @param lua_file Filename of the Lua script to run in this sandbox. + * @param require_path Location of the common sandbox modules + * @param memory_limit Sets the sandbox memory limit (bytes). + * @param instruction_limit Sets the sandbox Lua instruction limit (count). + * This limit is per call to process_message or timer_event + * @param output_limit Sets the single message payload limit (bytes). This + * limit applies to the in memory output buffer. The buffer is reset back + * to zero when inject_message is called. + * @return lua_sandbox Sandbox pointer or NULL on failure. + */ +LSB_EXPORT lua_sandbox* lsb_create(void* parent, + const char* lua_file, + const char* require_path, + unsigned memory_limit, + unsigned instruction_limit, + unsigned output_limit); + +/** + * Initializes the Lua sandbox and loads/runs the Lua script that was specified + * in lua_create_sandbox. + * + * @param lsb Pointer to the sandbox. + * @param state_file Filename where the global data is read. Use a NULL or empty + * string for no data restoration. The global + * _PRESERVATION_VERSION variable will be examined during + * restoration; if the previous version does not match the + * current version the restoration will be aborted and the + * sandbox will start cleanly. _PRESERVATION_VERSION should be + * incremented any time an incompatible change is made to the + * global data schema. If no version is set the check will + * always succeed and a version of zero is assigned. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT int lsb_init(lua_sandbox* lsb, const char* state_file); + +/** + * Frees the memory associated with the sandbox. + * + * @param lsb Sandbox pointer to discard. + * @param state_file Filename where the sandbox global data is saved. Use a + * NULL or empty string for no preservation. + * + * @return NULL on success, pointer to an error message on failure that MUST BE + * FREED by the caller. + */ +LSB_EXPORT char* lsb_destroy(lua_sandbox* lsb, const char* state_file); + +/** + * Retrieve the sandbox usage statistics. + * + * @param lsb Pointer to the sandbox. + * @param lsb_usage_type Type of statistic to retrieve i.e. memory. + * @param lsb_usage_stat Type of statistic to retrieve i.e. current. + * + * @return unsigned Count or number of bytes depending on the statistic. + */ +LSB_EXPORT unsigned +lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, lsb_usage_stat ustat); +/** + * Retrieve the current sandbox status. + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_state code + */ +LSB_EXPORT lsb_state lsb_get_state(lua_sandbox* lsb); + +/** + * Return the last error in human readable form. + * + * @param lsb Pointer to the sandbox. + * + * @return const char* error message + */ +LSB_EXPORT const char* lsb_get_error(lua_sandbox* lsb); + +/** + * Sets the last error string. + * + * @param lsb Pointer to the sandbox. + * @param err Error message. + * + * @return const char* error message + */ +LSB_EXPORT void lsb_set_error(lua_sandbox* lsb, const char* err); + +/** + * Access the Lua pointer. + * + * @param lsb Pointer to the sandbox. + * + * @return lua_State* The lua_State pointer. + */ +LSB_EXPORT lua_State* lsb_get_lua(lua_sandbox* lsb); + +/** + * Access the parent pointer stored in the sandbox. + * + * @param lsb Pointer to the sandbox. + * + * @return void* The parent pointer passed to init. + */ +LSB_EXPORT void* lsb_get_parent(lua_sandbox* lsb); + +/** + * Create a CFunction for use by the Sandbox. The Lua sandbox pointer is pushed + * to upvalue index 1. + * + * @param lsb Pointer to the sandbox. + * @param func Lua CFunction pointer. + * @param func_name Function name exposed to the Lua sandbox. + */ +LSB_EXPORT void lsb_add_function(lua_sandbox* lsb, lua_CFunction func, + const char* func_name); + +/** + * Retrieve the data in the output buffer and reset the buffer. The returned + * output string will remain valid until additional sandbox output is performed. + * The output should be copied if the application needs to hold onto it. + * + * @param lsb Pointer to the sandbox. + * @param len If len is not NULL, it will be set to the length of the string. + * + * @return const char* Pointer to the output buffer. + */ +LSB_EXPORT const char* lsb_get_output(lua_sandbox* lsb, size_t* len); + +/** + * Helper function to load the Lua function and set the instruction limits + * + * @param lsb Pointer to the sandbox. + * @param func_name Name of the function to load + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name); + +/** + * Helper function to update the statistics after the call + * + * @param lsb Pointer to the sandbox. + */ +LSB_EXPORT void lsb_pcall_teardown(lua_sandbox* lsb); + +/** + * Shutdown the sandbox due to a fatal error. + * + * @param lsb Pointer to the sandbox. + * @param err Reason for termination + */ +LSB_EXPORT void lsb_terminate(lua_sandbox* lsb, const char* err); + +#endif diff --git a/include/lsb_output.h b/include/lsb_output.h new file mode 100644 index 0000000..de13e17 --- /dev/null +++ b/include/lsb_output.h @@ -0,0 +1,96 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox output buffer functions @file */ + +#ifndef lsb_output_h_ +#define lsb_output_h_ + +#include +#include + +#include "lsb.h" + +/** + * Add a output function to the environment table. The environment table must be + * on the top of the stack. This function will receive the userdata and + * lsb_output_data struct as pointers on the Lua stack. + * + * lsb_output_data* output = (output_data*)lua_touserdata(lua, -1); + * ud_object* ud = (ud_object*)lua_touserdata(lua, -2); + * + * @param lua Pointer the Lua state. + * @param fp Function pointer to the outputter. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT void lsb_add_output_function(lua_State* lua, lua_CFunction fp); + +/** + * Append a character to the output stream. + * + * @param output Pointer the output collector. + * @param ch Character to append to the output. + * + * @return int Zero on success, non-zero if out of memory. + */ +LSB_EXPORT int lsb_appendc(lsb_output_data* output, char ch); + +/** + * Append formatted string to the output stream. + * + * @param output Pointer the output collector. + * @param fmt Printf format specifier. + * + * @return int Zero on success, non-zero if out of memory. + */ +LSB_EXPORT int lsb_appendf(lsb_output_data* output, const char* fmt, ...); + +/** + * Append a fixed string to the output stream. + * + * @param output Pointer the output collector. + * @param str String to append to the output. + * @param len Length of the string to append + * + * @return int Zero on success, non-zero if out of memory. + */ +LSB_EXPORT +int lsb_appends(lsb_output_data* output, const char* str, size_t len); + +/** + * Resize the output buffer when more space is needed. + * + * @param output Output buffer to resize. + * @param needed Number of additional bytes needed. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT int lsb_realloc_output(lsb_output_data* output, size_t needed); + +/** + * Write an array of variables on the Lua stack to the output buffer. + * + * @param lsb Pointer to the sandbox. + * @param start Lua stack index of first variable. + * @param end Lua stack index of the last variable. + * @param append 0 to overwrite the output buffer, 1 to append the output to it + * + */ +LSB_EXPORT void lsb_output(lua_sandbox* lsb, int start, int end, int append); + +/** + * Write a Lua table (in a Heka protobuf structure) to the output buffer. + * + * @param lsb Pointer to the sandbox. + * @param index Lua stack index of the table. + * @param append 0 to overwrite the output buffer, 1 to append the output to it + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_output_protobuf(lua_sandbox* lsb, int index, int append); + +#endif diff --git a/include/lsb_serialize.h b/include/lsb_serialize.h new file mode 100644 index 0000000..4ceff6b --- /dev/null +++ b/include/lsb_serialize.h @@ -0,0 +1,77 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua sandbox serialization @file */ + +#ifndef lsb_serialize_h_ +#define lsb_serialize_h_ + +#include +#include + +#include "lsb_output.h" + +/** + * Serialize all user global data to disk. + * + * @param lsb Pointer to the sandbox. + * @param data_file Filename where the data will be written (create/overwrite) + * + * @return int Zero on success, non-zero on failure. + */ +int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file); + +/** + * Restores previously serialized data from disk. + * + * @param lsb Pointer to the sandbox. + * @param data_file Filename from where the data will be read. + * + * @return int Zero on success, non-zero on failure. + */ +int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file); + +/** + * Add a serialization function to the environment table. The environment table + * must be on the top of the stack. This function will receive the userdata, + * fully qualified variable name, and lsb_output_data struct as pointers on the + * Lua stack. + * + * lsb_output_data* output = (output_data*)lua_touserdata(lua, -1); + * const char *key = (const char*)lua_touserdata(lua, -2); + * ud_object* ud = (ud_object*)lua_touserdata(lua, -3); + * + * @param lua Pointer the Lua state. + * @param fp Function pointer to the serializer. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT void +lsb_add_serialize_function(lua_State *lua, lua_CFunction fp); + +/** + * Serializes a binary data to a Lua string. + * + * @param src Pointer to the binary data. + * @param len Length in bytes of the data to output. + * @param output Pointer the output collector. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT int +lsb_serialize_binary(const void *src, size_t len, lsb_output_data* output); + +/** + * More efficient serialization of a double to a string + * + * @param output Pointer the output collector. + * @param d Double value to convert to a string. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT int lsb_serialize_double(lsb_output_data* output, double d); + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cde2ce7..6fa60d4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,17 +3,11 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. set(LUA_SANDBOX_SRC -lua_sandbox.c -lua_output.c -lua_sandbox_private.c -lua_serialize.c -lua_serialize_protobuf.c -lua_circular_buffer.c -lua_bloom_filter.c -lua_hyperloglog.c -cephes.c -xxhash.c -redis_hyperloglog.c +lsb.c +lsb_modules.c +lsb_output.c +lsb_serialize.c +lsb_serialize_protobuf.c struct.c ) @@ -23,6 +17,9 @@ if(MSVC) "${EP_BASE}/lib/lua.lib" "${EP_BASE}/lib/lpeg.lib" "${EP_BASE}/lib/cjson.lib" + "${EP_BASE}/lib/bloom_filter.lib" + "${EP_BASE}/lib/circular_buffer.lib" + "${EP_BASE}/lib/hyperloglog.lib" ) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.dll") @@ -33,6 +30,9 @@ elseif(MINGW) "${EP_BASE}/lib/liblua.dll" "${EP_BASE}/lib/liblpeg.dll" "${EP_BASE}/lib/libcjson.dll" + "${EP_BASE}/lib/bloom_filter.dll" + "${EP_BASE}/lib/circular_buffer.dll" + "${EP_BASE}/lib/hyperloglog.dll" ) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) @@ -52,6 +52,9 @@ else() "${EP_BASE}/lib/liblua.a" "${EP_BASE}/lib/liblpeg.a" "${EP_BASE}/lib/libcjson.a" + "${EP_BASE}/lib/bloom_filter.a" + "${EP_BASE}/lib/circular_buffer.a" + "${EP_BASE}/lib/hyperloglog.a" ${LINK_DL} -lm ) add_library(luasandbox STATIC ${LUA_SANDBOX_SRC}) @@ -61,7 +64,7 @@ endif() target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) -add_dependencies(luasandbox ${LUA_PROJECT} lpeg-0_12 lua-cjson-2_1_0) +add_dependencies(luasandbox ${LUA_PROJECT} lua_lpeg lua_cjson lua_bloom_filter lua_circular_buffer lua_hyperloglog) install(TARGETS luasandbox DESTINATION lib) install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) install(DIRECTORY "${LUA_INCLUDE_DIR}/" DESTINATION include) diff --git a/src/cephes.c b/src/cephes.c deleted file mode 100644 index 9cfabdc..0000000 --- a/src/cephes.c +++ /dev/null @@ -1,33 +0,0 @@ -/* -* Cephes Math Library Release 2.2: June, 1992 -* Copyright 1984, 1987, 1988, 1992 by Stephen L. Moshier -* Direct inquiries to 30 Frost Street, Cambridge, MA 02140 -*/ - -/* extracted the needed function from ntdr.c */ - -#include - -const double SQRTH = 0.70710678118654752440; /* sqrt(2)/2 */ - -double ndtr(double a) -{ - double x, y, z; - - if (isnan(a)) { - return a; - } - - x = a * SQRTH; - z = fabs(x); - - if (z < SQRTH) y = 0.5 + 0.5 * erf(x); - - else { - y = 0.5 * erfc(z); - - if (x > 0) y = 1.0 - y; - } - - return (y); -} diff --git a/src/cephes.h b/src/cephes.h deleted file mode 100644 index 5eb340e..0000000 --- a/src/cephes.h +++ /dev/null @@ -1,8 +0,0 @@ -/* -* Cephes Math Library Release 2.2: June, 1992 -* Copyright 1984, 1987, 1988, 1992 by Stephen L. Moshier -* Direct inquiries to 30 Frost Street, Cambridge, MA 02140 -*/ - -/* Normal Distribution Function */ -double ndtr(double a); diff --git a/src/lua_sandbox.c b/src/lsb.c similarity index 72% rename from src/lua_sandbox.c rename to src/lsb.c index 35f9682..ba2b946 100644 --- a/src/lua_sandbox.c +++ b/src/lsb.c @@ -6,24 +6,112 @@ /** @brief Lua sandboxed implementation @file */ +#include "lsb.h" + +#include +#include +#include +#include #include -#include #include +#include #include #include -#include -#include -#include -#include "lua_sandbox_private.h" -#include "lua_serialize.h" -#include "lua_serialize_protobuf.h" -#include "lua_circular_buffer.h" + +#include "lsb_modules.h" +#include "lsb_output.h" +#include "lsb_private.h" +#include "lsb_serialize.h" +#include "lsb_serialize_protobuf.h" static const char* disable_base_functions[] = { "collectgarbage", "coroutine", - "dofile", "load", "loadfile", "loadstring", "module", "print", "require", NULL }; + "dofile", "load", "loadfile", "loadstring", "module", "print", "require", + NULL }; static jmp_buf g_jbuf; +#ifndef LUA_JIT +/** +* Implementation of the memory allocator for the Lua state. +* +* See: http://www.lua.org/manual/5.1/manual.html#lua_Alloc +* +* @param ud Pointer to the lua_sandbox +* @param ptr Pointer to the memory block being allocated/reallocated/freed. +* @param osize The original size of the memory block. +* @param nsize The new size of the memory block. +* +* @return void* A pointer to the memory block. +*/ +static void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize) +{ + lua_sandbox* lsb = (lua_sandbox*)ud; + + void* nptr = NULL; + if (nsize == 0) { + free(ptr); + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= (unsigned)osize; + } else { + unsigned new_state_memory = + (unsigned)(lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize); + if (0 == lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] + || new_state_memory + <= lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]) { + nptr = realloc(ptr, nsize); + if (nptr != NULL) { + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = + new_state_memory; + if (lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + > lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; + } + } + } + } + return nptr; +} +#endif + + +static size_t instruction_usage(lua_sandbox* lsb) +{ + return lua_gethookcount(lsb->lua) - lua_gethookcountremaining(lsb->lua); +} + + +static void instruction_manager(lua_State* lua, lua_Debug* ar) +{ + if (LUA_HOOKCOUNT == ar->event) { + luaL_error(lua, "instruction_limit exceeded"); + } +} + + +static int output(lua_State* lua) +{ + void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == luserdata) { + return luaL_error(lua, "output() invalid lightuserdata"); + } + lua_sandbox* lsb = (lua_sandbox*)luserdata; + + int n = lua_gettop(lua); + if (n == 0) { + return luaL_argerror(lsb->lua, 0, "must have at least one argument"); + } + lsb_output(lsb, 1, n, 1); + return 0; +} + + +static int unprotected_panic(lua_State* lua) +{ + (void)lua; + longjmp(g_jbuf, 1); + return 0; +} + lua_sandbox* lsb_create(void* parent, const char* lua_file, @@ -70,10 +158,10 @@ lua_sandbox* lsb_create(void* parent, lsb->error_message[0] = 0; lsb->output.pos = 0; lsb->output.maxsize = output_limit; - if (output_limit && output_limit < OUTPUT_SIZE) { + if (output_limit && output_limit < LSB_OUTPUT_SIZE) { lsb->output.size = output_limit; } else { - lsb->output.size = OUTPUT_SIZE; + lsb->output.size = LSB_OUTPUT_SIZE; } lsb->output.data = malloc(lsb->output.size); size_t len = strlen(lua_file); @@ -100,15 +188,6 @@ lua_sandbox* lsb_create(void* parent, return lsb; } - -int unprotected_panic(lua_State* lua) -{ - (void)lua; - longjmp(g_jbuf, 1); - return 0; -} - - int lsb_init(lua_sandbox* lsb, const char* data_file) { if (!lsb) { @@ -119,23 +198,23 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif - load_library(lsb->lua, "", luaopen_base, disable_base_functions); + lsb_load_library(lsb->lua, "", luaopen_base, disable_base_functions); lua_pop(lsb->lua, 1); // Create a simple package cache lua_createtable(lsb->lua, 0, 1); lua_pushvalue(lsb->lua, -1); - lua_setglobal(lsb->lua, package_table); + lua_setglobal(lsb->lua, lsb_package_table); // Add empty metatable to prevent serialization lua_newtable(lsb->lua); lua_setmetatable(lsb->lua, -2); // add the loaded table lua_newtable(lsb->lua); - lua_setfield(lsb->lua, -2, loaded_table); + lua_setfield(lsb->lua, -2, lsb_loaded_table); lua_pop(lsb->lua, 1); // remove the package table lua_pushlightuserdata(lsb->lua, (void*)lsb); - lua_pushcclosure(lsb->lua, &require_library, 1); + lua_pushcclosure(lsb->lua, &lsb_require_library, 1); lua_setglobal(lsb->lua, "require"); lua_pushlightuserdata(lsb->lua, (void*)lsb); @@ -158,7 +237,7 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) if (len >= LSB_ERROR_SIZE || len < 0) { lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } - sandbox_terminate(lsb); + lsb_terminate(lsb, NULL); return 2; } else { lua_gc(lsb->lua, LUA_GCCOLLECT, 0); @@ -171,7 +250,7 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) } lsb->state = LSB_RUNNING; if (data_file != NULL && strlen(data_file) > 0) { - if (restore_global_data(lsb, data_file)) return 3; + if (lsb_restore_global_data(lsb, data_file)) return 3; } } lua_atpanic(lsb->lua, pf); @@ -187,7 +266,7 @@ char* lsb_destroy(lua_sandbox* lsb, const char* data_file) } if (lsb->lua && data_file && strlen(data_file) > 0) { - if (preserve_global_data(lsb, data_file) != 0) { + if (lsb_preserve_global_data(lsb, data_file) != 0) { size_t len = strlen(lsb->error_message); err = malloc(len + 1); if (err != NULL) { @@ -195,7 +274,7 @@ char* lsb_destroy(lua_sandbox* lsb, const char* data_file) } } } - sandbox_terminate(lsb); + lsb_terminate(lsb, NULL); free(lsb->output.data); free(lsb->lua_file); free(lsb->require_path); @@ -292,85 +371,6 @@ const char* lsb_get_output(lua_sandbox* lsb, size_t* len) } -void lsb_output(lua_sandbox* lsb, int start, int end, int append) -{ - if (!append) { - lsb->output.pos = 0; - } - - int result = 0; - void* ud = NULL; - for (int i = start; result == 0 && i <= end; ++i) { - switch (lua_type(lsb->lua, i)) { - case LUA_TNUMBER: - if (serialize_double(&lsb->output, lua_tonumber(lsb->lua, i))) { - result = 1; - } - break; - case LUA_TSTRING: - { - size_t len; - const char* s = lua_tolstring(lsb->lua, i, &len); - if (lsb_appends(&lsb->output, s, len)) { - result = 1; - } - } - break; - case LUA_TNIL: - if (lsb_appends(&lsb->output, "nil", 3)) { - result = 1; - } - break; - case LUA_TBOOLEAN: - if (lsb_appendf(&lsb->output, "%s", - lua_toboolean(lsb->lua, i) - ? "true" : "false")) { - result = 1; - } - break; - case LUA_TUSERDATA: - ud = userdata_type(lsb->lua, i, lsb_circular_buffer); - if (ud) { - if (output_circular_buffer(lsb->lua, (circular_buffer*)ud, - &lsb->output)) { - result = 1; - } - } else { - luaL_argerror(lsb->lua, i, "unknown userdata type"); - } - break; - default: - luaL_argerror(lsb->lua, i, "unsuported type"); - break; - } - } - update_output_stats(lsb); - if (result != 0) { - if (lsb->error_message[0] == 0) { - luaL_error(lsb->lua, "output_limit exceeded"); - } - luaL_error(lsb->lua, lsb->error_message); - } -} - - -int lsb_output_protobuf(lua_sandbox* lsb, int index, int append) -{ - if (!append) { - lsb->output.pos = 0; - } - - size_t last_pos = lsb->output.pos; - if (serialize_table_as_pb(lsb, index) != 0) { - lsb->output.pos = last_pos; - return 1; - } - - update_output_stats(lsb); - return 0; -} - - int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name) { @@ -404,7 +404,15 @@ void lsb_pcall_teardown(lua_sandbox* lsb) void lsb_terminate(lua_sandbox* lsb, const char* err) { - strncpy(lsb->error_message, err, LSB_ERROR_SIZE); - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - sandbox_terminate(lsb); + if (err) { + strncpy(lsb->error_message, err, LSB_ERROR_SIZE); + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + + if (lsb->lua) { + lua_close(lsb->lua); + lsb->lua = NULL; + } + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = 0; + lsb->state = LSB_TERMINATED; } diff --git a/src/lsb_modules.c b/src/lsb_modules.c new file mode 100644 index 0000000..4cd8804 --- /dev/null +++ b/src/lsb_modules.c @@ -0,0 +1,156 @@ + +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox modules implementation @file */ + +#include "lsb_modules.h" + +#include +#include +#include +#include +#include + +#include "lsb_private.h" + +#ifndef MAX_PATH +#define MAX_PATH 260 +#endif + +#ifdef _WIN32 +#define PATH_DELIMITER '\\' +#else +#define PATH_DELIMITER '/' +#endif + +const char* lsb_disable_none[] = { NULL }; +const char* lsb_package_table = "package"; +const char* lsb_loaded_table = "loaded"; + +// If necessary, add an empty metatable to flag the table as non-data +// during preservation. +static void add_empty_metatable(lua_State* lua) +{ + if (lua_getmetatable(lua, -1) == 0) { + lua_newtable(lua); + lua_setmetatable(lua, -2); + } else { + lua_pop(lua, 1); + } +} + + +void lsb_load_library(lua_State* lua, const char* table, lua_CFunction f, + const char** disable) +{ + lua_pushcfunction(lua, f); + lua_call(lua, 0, 1); + + if (strlen(table) == 0) { // Handle the special "" base table. + for (int i = 0; disable[i]; ++i) { + lua_pushnil(lua); + lua_setfield(lua, LUA_GLOBALSINDEX, disable[i]); + } + } else { + for (int i = 0; disable[i]; ++i) { + lua_pushnil(lua); + lua_setfield(lua, -2, disable[i]); + } + add_empty_metatable(lua); + } +} + + +int lsb_require_library(lua_State* lua) +{ + const char* name = luaL_checkstring(lua, 1); + lua_getglobal(lua, lsb_package_table); + if (!lua_istable(lua, -1)) { + return luaL_error(lua, "%s table is missing", lsb_package_table); + } + lua_getfield(lua, -1, lsb_loaded_table); + if (!lua_istable(lua, -1)) { + return luaL_error(lua, "%s.%s table is missing", lsb_package_table, + lsb_loaded_table); + } + lua_getfield(lua, -1, name); + if (!lua_isnil(lua, -1)) { + return 1; // returned the cache copy + } + lua_pop(lua, 1); // remove the nil + int pos = lua_gettop(lua); + lua_pushboolean(lua, 1); + lua_setfield(lua, pos, name); // mark it as loaded to prevent a dependency loop + + if (strcmp(name, LUA_STRLIBNAME) == 0) { + lsb_load_library(lua, name, luaopen_string, lsb_disable_none); + } else if (strcmp(name, LUA_MATHLIBNAME) == 0) { + lsb_load_library(lua, name, luaopen_math, lsb_disable_none); + } else if (strcmp(name, LUA_TABLIBNAME) == 0) { + lsb_load_library(lua, name, luaopen_table, lsb_disable_none); + } else if (strcmp(name, LUA_OSLIBNAME) == 0) { + const char* disable[] = { "execute", "exit", "remove", "rename", + "setlocale", "tmpname", NULL }; + lsb_load_library(lua, name, luaopen_os, disable); + } else if (strcmp(name, mozsvc_circular_buffer_table) == 0) { + lsb_load_library(lua, name, luaopen_circular_buffer, lsb_disable_none); + } else if (strcmp(name, mozsvc_bloom_filter_table) == 0) { + lsb_load_library(lua, name, luaopen_bloom_filter, lsb_disable_none); + } else if (strcmp(name, mozsvc_hyperloglog_table) == 0) { + lsb_load_library(lua, name, luaopen_hyperloglog, lsb_disable_none); + } else if (strcmp(name, "lpeg") == 0) { + lsb_load_library(lua, name, luaopen_lpeg, lsb_disable_none); + } else if (strcmp(name, "cjson") == 0) { + void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == luserdata) { + return luaL_error(lua, "require_library() invalid lightuserdata"); + } + lua_sandbox* lsb = (lua_sandbox*)luserdata; + + const char* disable[] = { "new", "encode_keep_buffer", NULL }; + lsb_load_library(lua, name, luaopen_cjson, disable); + if (set_encode_max_buffer(lua, -1, (unsigned)lsb->output.maxsize)) { + return luaL_error(lua, "cjson encode buffer could not be configured"); + } + lua_pushvalue(lua, -1); + lua_setglobal(lua, name); + } else if (strcmp(name, "struct") == 0) { + lsb_load_library(lua, name, luaopen_struct, lsb_disable_none); + } else { + void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == luserdata) { + return luaL_error(lua, "require_library() invalid lightuserdata"); + } + lua_sandbox* lsb = (lua_sandbox*)luserdata; + + if (!lsb->require_path) { + return luaL_error(lua, "require_library() external modules are disabled"); + } + + int i = 0; + while (name[i]) { + if (!isalnum(name[i]) && name[i] != '_') { + return luaL_error(lua, "invalid module name '%s'", name); + } + ++i; + } + char fn[MAX_PATH]; + i = snprintf(fn, MAX_PATH, "%s%c%s.lua", lsb->require_path, PATH_DELIMITER, + name); + if (i < 0 || i >= MAX_PATH) { + return luaL_error(lua, "require_path exceeded %d", MAX_PATH); + } + + if (luaL_dofile(lua, fn) != 0) { + return luaL_error(lua, "%s", lua_tostring(lua, -1)); + } + add_empty_metatable(lua); + } + lua_pushvalue(lua, -1); + lua_setfield(lua, pos, name); + return 1; +} diff --git a/src/lsb_modules.h b/src/lsb_modules.h new file mode 100644 index 0000000..a62b3bc --- /dev/null +++ b/src/lsb_modules.h @@ -0,0 +1,50 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua sandbox module management @file */ + +#ifndef lsb_modules_h_ +#define lsb_modules_h_ + +#include + +#include "lua_bloom_filter.h" +#include "lua_circular_buffer.h" +#include "lua_hyperloglog.h" + +extern const char* lsb_disable_none[]; +extern const char* lsb_package_table; +extern const char* lsb_loaded_table; + +/** + * Performs the library load and secures the sandbox environment for use. + * + * @param lua Pointer to the Lua state. + * @param table Name of the table being loaded. + * @param f Pointer to the table load function. + * @param disable Array of function names to disable in the loaded table. + */ +void lsb_load_library(lua_State* lua, const char* table, lua_CFunction f, + const char** disable); + +/** + * Overridden 'require' used to load optional sandbox libraries in global space. + * + * @param lua Pointer to the Lua state. + * + * @return int Returns 1 value on the stack (for the standard modules a table + * for the LPEG grammars, userdata). + */ +int lsb_require_library(lua_State* lua); + +/* declarations for modules without headers */ +LUALIB_API int luaopen_cjson(lua_State* L); +int set_encode_max_buffer(lua_State* L, int index, unsigned maxsize); + +LUALIB_API int luaopen_lpeg(lua_State* L); +LUALIB_API int luaopen_struct(lua_State* L); + +#endif diff --git a/src/lsb_output.c b/src/lsb_output.c new file mode 100644 index 0000000..0738fee --- /dev/null +++ b/src/lsb_output.c @@ -0,0 +1,234 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox output buffer implementation @file */ + +#include "lsb_output.h" + +#include +#include +#include +#include +#include + +#include "lsb_private.h" +#include "lsb_serialize.h" +#include "lsb_serialize_protobuf.h" + +static const char* output_function = "lsb_output"; + +static lua_CFunction get_output_function(lua_State* lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, output_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + + +static void update_output_stats(lua_sandbox* lsb) +{ + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = (unsigned)lsb->output.pos; + if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] + > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + } +} + + +void lsb_add_output_function(lua_State* lua, lua_CFunction fp) +{ + lua_pushstring(lua, output_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +int lsb_appendc(lsb_output_data* output, char ch) +{ + size_t needed = 2; + if (output->size - output->pos < needed) { + if (lsb_realloc_output(output, needed)) return 1; + } + output->data[output->pos++] = ch; + output->data[output->pos] = 0; + return 0; +} + + +int lsb_appendf(lsb_output_data* output, const char* fmt, ...) +{ + va_list args; + int result = 0; + int remaining = 0; + char* ptr = NULL, *old_ptr = NULL; + do { + ptr = output->data + output->pos; + remaining = (int)(output->size - output->pos); + va_start(args, fmt); + int needed = vsnprintf(ptr, remaining, fmt, args); + va_end(args); + if (needed == -1) { + // Windows and Unix have different return values for this function + // -1 on Unix is a format error + // -1 on Windows means the buffer is too small and the required len + // is not returned + needed = remaining; + } + if (needed >= remaining) { + if (output->maxsize + && (output->size >= output->maxsize + || output->pos + needed >= output->maxsize)) { + return 1; + } + size_t newsize = output->size * 2; + while ((size_t)needed >= newsize - output->pos) { + newsize *= 2; + } + if (output->maxsize && newsize > output->maxsize) { + newsize = output->maxsize; + } + void* p = malloc(newsize); + if (p != NULL) { + memcpy(p, output->data, output->pos); + old_ptr = output->data; + output->data = p; + output->size = newsize; + } else { + return 1; // Out of memory condition. + } + } else { + output->pos += needed; + break; + } + } + while (1); + free(old_ptr); + return result; +} + + +int lsb_appends(lsb_output_data* output, const char* str, size_t len) +{ + size_t needed = len + 1; + if (output->size - output->pos < needed) { + if (lsb_realloc_output(output, needed)) return 1; + } + memcpy(output->data + output->pos, str, len); + output->pos += len; + output->data[output->pos] = 0; + return 0; +} + + +int lsb_realloc_output(lsb_output_data* output, size_t needed) +{ + if (output->maxsize && needed + output->pos > output->maxsize) { + return 1; + } + size_t newsize = output->size * 2; + while (needed >= newsize - output->pos) { + newsize *= 2; + } + if (output->maxsize && newsize > output->maxsize) { + newsize = output->maxsize; + } + + void* ptr = realloc(output->data, newsize); + if (!ptr) return 1; + output->data = ptr; + output->size = newsize; + return 0; +} + + +void lsb_output(lua_sandbox* lsb, int start, int end, int append) +{ + if (!append) { + lsb->output.pos = 0; + } + + int result = 0; + for (int i = start; result == 0 && i <= end; ++i) { + switch (lua_type(lsb->lua, i)) { + case LUA_TNUMBER: + if (lsb_serialize_double(&lsb->output, lua_tonumber(lsb->lua, i))) { + result = 1; + } + break; + case LUA_TSTRING: + { + size_t len; + const char* s = lua_tolstring(lsb->lua, i, &len); + if (lsb_appends(&lsb->output, s, len)) { + result = 1; + } + } + break; + case LUA_TNIL: + if (lsb_appends(&lsb->output, "nil", 3)) { + result = 1; + } + break; + case LUA_TBOOLEAN: + if (lsb_appendf(&lsb->output, "%s", + lua_toboolean(lsb->lua, i) + ? "true" : "false")) { + result = 1; + } + break; + case LUA_TUSERDATA: + { + lua_CFunction fp = get_output_function(lsb->lua, i); + if (!fp) { + luaL_argerror(lsb->lua, i, "unknown userdata type"); + return; // never reaches here but the compiler doesn't know it + } + lua_pushlightuserdata(lsb->lua, &lsb->output); + result = fp(lsb->lua); + lua_pop(lsb->lua, 1); // remove output + } + break; + default: + luaL_argerror(lsb->lua, i, "unsuported type"); + break; + } + } + update_output_stats(lsb); + if (result != 0) { + if (lsb->error_message[0] == 0) { + luaL_error(lsb->lua, "output_limit exceeded"); + } + luaL_error(lsb->lua, lsb->error_message); + } +} + + +int lsb_output_protobuf(lua_sandbox* lsb, int index, int append) +{ + if (lua_gettop(lsb->lua) != 1 || lua_type(lsb->lua, 1) != LUA_TTABLE) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "takes a single table argument"); + return 1; + } + + if (!append) { + lsb->output.pos = 0; + } + + size_t last_pos = lsb->output.pos; + if (lsb_serialize_table_as_pb(lsb, index) != 0) { + lsb->output.pos = last_pos; + return 1; + } + + update_output_stats(lsb); + return 0; +} diff --git a/src/lsb_private.h b/src/lsb_private.h new file mode 100644 index 0000000..31fa2e7 --- /dev/null +++ b/src/lsb_private.h @@ -0,0 +1,41 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua sandbox private functions @file */ + +#ifndef lsb_private_h_ +#define lsb_private_h_ + +#include + +#include "lsb.h" + +#ifdef _WIN32 +#define snprintf _snprintf +#endif + +#define LSB_OUTPUT_SIZE 1024 + +struct lsb_output_data +{ + size_t maxsize; + size_t size; + size_t pos; + char* data; +}; + +struct lua_sandbox { + lua_State* lua; + void* parent; + lsb_state state; + lsb_output_data output; + char* lua_file; + char* require_path; + unsigned usage[LSB_UT_MAX][LSB_US_MAX]; + char error_message[LSB_ERROR_SIZE]; +}; + +#endif diff --git a/src/lua_serialize.c b/src/lsb_serialize.c similarity index 64% rename from src/lua_serialize.c rename to src/lsb_serialize.c index eeaded2..489daca 100644 --- a/src/lua_serialize.c +++ b/src/lsb_serialize.c @@ -6,223 +6,211 @@ /** @brief Sandbox serialization implementation @file */ +#include "lsb_serialize.h" + #include #include #include #include #include -#include "lua_serialize.h" -#include "lua_circular_buffer.h" -#include "lua_bloom_filter.h" -#include "lua_hyperloglog.h" -const char* not_a_number = "nan"; -static const char* preservation_version = "_PRESERVATION_VERSION"; +#include "lsb_modules.h" +#include "lsb_private.h" -static int get_preservation_version(lua_State* lua) +typedef struct { - int ver = 0; - lua_getglobal(lua, preservation_version); - int t = lua_type(lua, -1); - if (t == LUA_TNUMBER) { - ver = (int)lua_tointeger(lua, -1); - } - lua_pop(lua, 1); // remove the version from the stack + const void* ptr; + size_t name_pos; +} table_ref; - if (t != LUA_TNIL) { // remove the version from the data preservation - lua_pushnil(lua); - lua_setglobal(lua, preservation_version); - } - return ver; -} - -int preserve_global_data(lua_sandbox* lsb, const char* data_file) +typedef struct { - static const char* G = "_G"; + size_t size; + size_t pos; + table_ref* array; +} table_ref_array; - // make sure the string library is loaded before we start - lua_getglobal(lsb->lua, LUA_STRLIBNAME); - if (!lua_istable(lsb->lua, -1)) { - load_library(lsb->lua, LUA_STRLIBNAME, luaopen_string, disable_none); - } - lua_pop(lsb->lua, 1); // Remove string table. +typedef struct +{ + FILE* fh; + lsb_output_data keys; + table_ref_array tables; + const void* globals; +} serialization_data; - lua_getglobal(lsb->lua, G); - if (!lua_istable(lsb->lua, -1)) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve_global_data cannot access the global table"); - return 1; - } +static const char* not_a_number = "nan"; +static const char* preservation_version = "_PRESERVATION_VERSION"; +static const char* serialize_function = "lsb_serialize"; + +/** + * Serializes a Lua table structure. + * + * @param lsb Pointer to the sandbox. + * @param data Pointer to the serialization state data. + * @param parent Index pointing to the parent's name in the table array. + * + * @return int Zero on success, non-zero on failure. + */ +static int +serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent); + +/** + * Serializes a Lua data value. + * + * @param lsb Pointer to the sandbox. + * @param index Lua stack index where the data resides. + * @param output Pointer the output collector. + * + * @return int + */ +static int serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output); + +/** + * Serializes a table key value pair. + * + * @param lsb Pointer to the sandbox. + * @param data Pointer to the serialization state data. + * @param parent Index pointing to the parent's name in the table array. + * + * @return int Zero on success, non-zero on failure. + */ +static int +serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent); + +/** + * Returns the serialization function if the userdata implemented one. + * + * @param lua Lua State. + * @param index Position on the stack where the userdata pointer resides. + * + * @return lua_CFunction NULL if not found + */ +static lua_CFunction get_serialize_function(lua_State* lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, serialize_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} - FILE* fh = fopen(data_file, "wb"); - if (fh == NULL) { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve_global_data could not open: %s", data_file); - if (len >= LSB_ERROR_SIZE || len < 0) { - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; +/** + * Helper function to determine what data should not be serialized. + * + * @param lsb Pointer to the sandbox. + * @param data Pointer to the serialization state data. + * @param index Lua stack index where the data resides. + * @param fp Function pointer to be set for userdata serialization + * + * @return int + */ +static int +ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index, + lua_CFunction* fp) +{ + switch (lua_type(lsb->lua, index)) { + case LUA_TTABLE: + if (lua_getmetatable(lsb->lua, index) != 0) { + lua_pop(lsb->lua, 1); // Remove the metatable. + return 1; } + if (lua_topointer(lsb->lua, index) == data->globals) { + return 1; + } + break; + case LUA_TUSERDATA: + *fp = get_serialize_function(lsb->lua, index); + if (!*fp) { + return 1; + } + break; + case LUA_TNONE: + case LUA_TFUNCTION: + case LUA_TTHREAD: + case LUA_TLIGHTUSERDATA: + case LUA_TNIL: return 1; } + return 0; +} - int result = 0; - serialization_data data; - data.fh = fh; - data.keys.maxsize = 0; - -// Clear the sandbox limits during preservation. -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); -#else -// unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif - lua_sethook(lsb->lua, instruction_manager, 0, 0); -// size_t cur_output_size = lsb->output.size; -// size_t max_output_size = lsb->output.maxsize; - lsb->output.maxsize = 0; -// end clear - data.keys.size = OUTPUT_SIZE; - data.keys.pos = 0; - data.keys.data = malloc(data.keys.size); - data.tables.size = 64; - data.tables.pos = 0; - data.tables.array = malloc(data.tables.size * sizeof(table_ref)); - if (data.tables.array == NULL || data.keys.data == NULL) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve_global_data out of memory"); - result = 1; - } else { - fprintf(data.fh, "if %s and %s ~= %d then return end\n", - preservation_version, - preservation_version, - get_preservation_version(lsb->lua)); - lsb_appendf(&data.keys, "%s", G); - data.keys.pos += 1; - data.globals = lua_topointer(lsb->lua, -1); - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - result = serialize_kvp(lsb, &data, 0); - lua_pop(lsb->lua, 1); +/** + * Looks for a table to see if it has already been processed. + * + * @param tra Pointer to the table references. + * @param ptr Pointer value of the table. + * + * @return table_ref* NULL if not found. + */ +static table_ref* find_table_ref(table_ref_array* tra, const void* ptr) +{ + for (size_t i = 0; i < tra->pos; ++i) { + if (ptr == tra->array[i].ptr) { + return &tra->array[i]; } - lua_pop(lsb->lua, lua_gettop(lsb->lua)); - // Wipe the entire Lua stack. Since incremental cleanup on failure - // was added the stack should only contain table _G. - } - free(data.tables.array); - free(data.keys.data); - fclose(fh); - if (result != 0) { - remove(data_file); - } - -// Uncomment if we start preserving state when not destroying the sandbox -/* -// Restore the sandbox limits after preservation -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); -#else - lua_gc(lsb->lua, LUA_GCCOLLECT, 0); - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; - lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; -#endif - lsb->output.maxsize = max_output_size; - lsb->output.pos = 0; - if (lsb->output.size > cur_output_size) { - void* ptr = realloc(lsb->output.data, cur_output_size); - if (!ptr) return 1; - lsb->output.data = ptr; - lsb->output.size = cur_output_size; } -// end restore -*/ - - return result; + return NULL; } -int serialize_double(output_data* output, double d) +/** + * Adds a table to the processed array. + * + * @param tra Pointer to the table references. + * @param ptr Pointer value of the table. + * @param name_pos Index pointing to name in the table array. + * + * @return table_ref* Pointer to the table reference or NULL if out of memory. + */ +static table_ref* +add_table_ref(table_ref_array* tra, const void* ptr, size_t name_pos) { - if (isnan(d)) { - return lsb_appends(output, not_a_number, 3); - } - if (d > INT_MAX) { - return lsb_appendf(output, "%0.17g", d); - } - - const int precision = 8; - const unsigned magnitude = 100000000; - char buffer[20]; - char* p = buffer; - int negative = 0; - - if (d < 0) { - negative = 1; - d = -d; - } - - int number = (int)d; - double tmp = (d - number) * magnitude; - unsigned fraction = (unsigned)tmp; - double diff = tmp - fraction; - - if (diff > 0.5) { - ++fraction; - if (fraction >= magnitude) { - fraction = 0; - ++number; - } - } else if (diff == 0.5 && ((fraction == 0) || (fraction & 1))) { - // bankers rounding - ++fraction; - } - - // output decimal fraction - if (fraction != 0) { - int nodigits = 1; - char c = 0; - for (int x = 0; x < precision; ++x) { - c = fraction % 10; - if (!(c == 0 && nodigits)) { - *p++ = c + '0'; - nodigits = 0; - } - fraction /= 10; + if (tra->pos == tra->size) { + size_t newsize = tra->size * 2; + void* p = realloc(tra->array, newsize * sizeof(table_ref)); + if (p != NULL) { + tra->array = p; + tra->size = newsize; + } else { + return NULL; } - *p++ = '.'; } + tra->array[tra->pos].ptr = ptr; + tra->array[tra->pos].name_pos = name_pos; + return &tra->array[tra->pos++]; +} - // output number - do { - *p++ = (number % 10) + '0'; - number /= 10; - } - while (number > 0); - size_t remaining = output->size - output->pos; - size_t needed = (p - buffer) + negative; - if (needed >= remaining) { - if (lsb_realloc_output(output, needed)) return 1; +/** + * Extracts the current preservation from the sandbox. + * + * @param lua Lua state + * + * @return int Version numebr + */ +static int get_preservation_version(lua_State* lua) +{ + int ver = 0; + lua_getglobal(lua, preservation_version); + int t = lua_type(lua, -1); + if (t == LUA_TNUMBER) { + ver = (int)lua_tointeger(lua, -1); } + lua_pop(lua, 1); // remove the version from the stack - if (negative) { - output->data[output->pos++] = '-'; - } - do { - --p; - output->data[output->pos++] = *p; + if (t != LUA_TNIL) { // remove the version from the data preservation + lua_pushnil(lua); + lua_setglobal(lua, preservation_version); } - while (p != buffer); - output->data[output->pos] = 0; - - return 0; + return ver; } -int serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent) +static int +serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent) { int result = 0; lua_checkstack(lsb->lua, 2); @@ -236,17 +224,18 @@ int serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent) } -int serialize_data(lua_sandbox* lsb, int index, output_data* output) +static int +serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output) { output->pos = 0; switch (lua_type(lsb->lua, index)) { case LUA_TNUMBER: - if (serialize_double(output, lua_tonumber(lsb->lua, index))) { + if (lsb_serialize_double(output, lua_tonumber(lsb->lua, index))) { return 1; } break; case LUA_TSTRING: - // The stack is cleaned up on failure by preserve_global_data + // The stack is cleaned up on failure by lsb_preserve_global_data // but for clarity it is incrementally cleaned up anyway. lua_checkstack(lsb->lua, 4); lua_getglobal(lsb->lua, LUA_STRLIBNAME); @@ -280,8 +269,8 @@ int serialize_data(lua_sandbox* lsb, int index, output_data* output) break; case LUA_TBOOLEAN: if (lsb_appendf(output, "%s", - lua_toboolean(lsb->lua, index) - ? "true" : "false")) { + lua_toboolean(lsb->lua, index) + ? "true" : "false")) { return 1; } break; @@ -295,37 +284,21 @@ int serialize_data(lua_sandbox* lsb, int index, output_data* output) } -void* userdata_type(lua_State* lua, int index, const char* tname) -{ - void* ud = lua_touserdata(lua, index); - if (ud != NULL) { - if (lua_getmetatable(lua, index)) { - lua_getfield(lua, LUA_REGISTRYINDEX, tname); - if (!lua_rawequal(lua, -1, -2)) { - ud = NULL; - } - } - } - lua_pop(lua, 2); // metatable and field - return ud; -} - - -int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) +static int +serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) { int kindex = -2, vindex = -1, result = 0; - - if (ignore_value_type(lsb, data, vindex)) { + lua_CFunction fp = NULL; + if (ignore_value_type(lsb, data, vindex, &fp)) { return 0; } - if (serialize_data(lsb, kindex, &lsb->output)) { return 1; } size_t pos = data->keys.pos; if (lsb_appendf(&data->keys, "%s[%s]", data->keys.data + parent, - lsb->output.data)) { + lsb->output.data)) { return 1; } @@ -349,87 +322,35 @@ int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) fprintf(data->fh, "%s\n", data->keys.data + seen->name_pos); } } else if (lua_type(lsb->lua, vindex) == LUA_TUSERDATA) { - void* ud = NULL; - if ((ud = userdata_type(lsb->lua, vindex, lsb_circular_buffer))) { - table_ref* seen = find_table_ref(&data->tables, ud); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ud, pos); - if (seen != NULL) { - data->keys.pos += 1; - result = serialize_circular_buffer(lsb->lua, - data->keys.data + pos, - (circular_buffer*)ud, - &lsb->output); - if (result == 0) { - fprintf(data->fh, "%s", lsb->output.data); - } - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_circular_buffer out of memory"); - return 1; - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + - seen->name_pos); - } - } else if ((ud = userdata_type(lsb->lua, vindex, lsb_bloom_filter))) { - table_ref* seen = find_table_ref(&data->tables, ud); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ud, pos); - if (seen != NULL) { - data->keys.pos += 1; - result = serialize_bloom_filter(data->keys.data + pos, - (bloom_filter*)ud, - &lsb->output); - if (result == 0) { - size_t n = fwrite(lsb->output.data, 1, lsb->output.pos, data->fh); - if (n != lsb->output.pos) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_bloom_filter failed"); - return 1; - } - } - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_bloom_filter out of memory"); - return 1; - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + - seen->name_pos); - } - } else if ((ud = userdata_type(lsb->lua, vindex, lsb_hyperloglog))) { - table_ref* seen = find_table_ref(&data->tables, ud); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ud, pos); - if (seen != NULL) { - data->keys.pos += 1; - result = serialize_hyperloglog(data->keys.data + pos, - (hyperloglog*)ud, - &lsb->output); - if (result == 0) { - size_t n = fwrite(lsb->output.data, 1, lsb->output.pos, data->fh); - if (n != lsb->output.pos) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_hyperloglog failed"); - return 1; - } + void* ud = lua_touserdata(lsb->lua, vindex); + table_ref* seen = find_table_ref(&data->tables, ud); + if (seen == NULL) { + seen = add_table_ref(&data->tables, ud, pos); + if (seen != NULL) { + data->keys.pos += 1; + lua_pushlightuserdata(lsb->lua, data->keys.data + pos); + lua_pushlightuserdata(lsb->lua, &lsb->output); + lsb->output.pos = 0; + result = fp(lsb->lua); + lua_pop(lsb->lua, 2); // remove the key and the output + if (result == 0) { + size_t n = fwrite(lsb->output.data, 1, lsb->output.pos, data->fh); + if (n != lsb->output.pos) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_serialize failed %s", data->keys.data + pos); + return 1; } - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_hyperloglog out of memory"); - return 1; } } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + - seen->name_pos); + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_serialize out of memory %s", data->keys.data + pos); + return 1; } + } else { + fprintf(data->fh, "%s = ", data->keys.data + pos); + data->keys.pos = pos; + fprintf(data->fh, "%s\n", data->keys.data + + seen->name_pos); } } else { fprintf(data->fh, "%s = ", data->keys.data + pos); @@ -443,78 +364,119 @@ int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) } -int ignore_key(lua_sandbox* lsb, int index) +int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) { - if (lua_type(lsb->lua, index) == LUA_TSTRING) { - const char* key = lua_tostring(lsb->lua, index); - if (key[0] == '_') { - return 1; - } + static const char* G = "_G"; + + // make sure the string library is loaded before we start + lua_getglobal(lsb->lua, LUA_STRLIBNAME); + if (!lua_istable(lsb->lua, -1)) { + lsb_load_library(lsb->lua, LUA_STRLIBNAME, luaopen_string, + lsb_disable_none); } - return 0; -} + lua_pop(lsb->lua, 1); // Remove string table. + lua_getglobal(lsb->lua, G); + if (!lua_istable(lsb->lua, -1)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_preserve_global_data cannot access the global table"); + return 1; + } -table_ref* find_table_ref(table_ref_array* tra, const void* ptr) -{ - for (size_t i = 0; i < tra->pos; ++i) { - if (ptr == tra->array[i].ptr) { - return &tra->array[i]; + FILE* fh = fopen(data_file, "wb"); + if (fh == NULL) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_preserve_global_data could not open: %s", + data_file); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } + return 1; } - return NULL; -} + int result = 0; + serialization_data data; + data.fh = fh; + data.keys.maxsize = 0; -table_ref* add_table_ref(table_ref_array* tra, const void* ptr, size_t name_pos) -{ - if (tra->pos == tra->size) { - size_t newsize = tra->size * 2; - void* p = realloc(tra->array, newsize * sizeof(table_ref)); - if (p != NULL) { - tra->array = p; - tra->size = newsize; - } else { - return NULL; +// Clear the sandbox limits during preservation. +#ifdef LUA_JIT + lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); +#else +// unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; +#endif + lua_sethook(lsb->lua, NULL, 0, 0); +// size_t cur_output_size = lsb->output.size; +// size_t max_output_size = lsb->output.maxsize; + lsb->output.maxsize = 0; +// end clear + + data.keys.size = LSB_OUTPUT_SIZE; + data.keys.pos = 0; + data.keys.data = malloc(data.keys.size); + data.tables.size = 64; + data.tables.pos = 0; + data.tables.array = malloc(data.tables.size * sizeof(table_ref)); + if (data.tables.array == NULL || data.keys.data == NULL) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_preserve_global_data out of memory"); + result = 1; + } else { + fprintf(data.fh, "if %s and %s ~= %d then return end\n", + preservation_version, + preservation_version, + get_preservation_version(lsb->lua)); + lsb_appendf(&data.keys, "%s", G); + data.keys.pos += 1; + data.globals = lua_topointer(lsb->lua, -1); + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (result == 0 && lua_next(lsb->lua, -2) != 0) { + result = serialize_kvp(lsb, &data, 0); + lua_pop(lsb->lua, 1); } + lua_pop(lsb->lua, lua_gettop(lsb->lua)); + // Wipe the entire Lua stack. Since incremental cleanup on failure + // was added the stack should only contain table _G. + } + free(data.tables.array); + free(data.keys.data); + fclose(fh); + if (result != 0) { + remove(data_file); } - tra->array[tra->pos].ptr = ptr; - tra->array[tra->pos].name_pos = name_pos; - return &tra->array[tra->pos++]; -} - -int ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index) -{ - switch (lua_type(lsb->lua, index)) { - case LUA_TTABLE: - if (lua_getmetatable(lsb->lua, index) != 0) { - lua_pop(lsb->lua, 1); // Remove the metatable. - return 1; - } - if (lua_topointer(lsb->lua, index) == data->globals) { - return 1; - } - break; - case LUA_TUSERDATA: - if (!userdata_type(lsb->lua, index, lsb_circular_buffer) && - !userdata_type(lsb->lua, index, lsb_bloom_filter) && - !userdata_type(lsb->lua, index, lsb_hyperloglog)) { - return 1; - } - break; - case LUA_TNONE: - case LUA_TFUNCTION: - case LUA_TTHREAD: - case LUA_TLIGHTUSERDATA: - case LUA_TNIL: - return 1; +// Uncomment if we start preserving state when not destroying the sandbox +// Note: serialization uses the output buffer, inprogress output can be +// destroyed if the user was collecting output between calls. +/* +// Restore the sandbox limits after preservation +#ifdef LUA_JIT + lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); +#else + lua_gc(lsb->lua, LUA_GCCOLLECT, 0); + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; + lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; +#endif + lsb->output.maxsize = max_output_size; + lsb->output.pos = 0; + if (lsb->output.size > cur_output_size) { + void* ptr = realloc(lsb->output.data, cur_output_size); + if (!ptr) return 1; + lsb->output.data = ptr; + lsb->output.size = cur_output_size; } - return 0; +// end restore +*/ + + return result; } -int restore_global_data(lua_sandbox* lsb, const char* data_file) +int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) { // Clear the sandbox limits during restoration. #ifdef LUA_JIT @@ -523,18 +485,18 @@ int restore_global_data(lua_sandbox* lsb, const char* data_file) unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif - lua_sethook(lsb->lua, instruction_manager, 0, 0); + lua_sethook(lsb->lua, NULL, 0, 0); int err = 0; if ((err = luaL_dofile(lsb->lua, data_file)) != 0) { if (LUA_ERRFILE != err) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "restore_global_data %s", + "lsb_restore_global_data %s", lua_tostring(lsb->lua, -1)); if (len >= LSB_ERROR_SIZE || len < 0) { lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } - sandbox_terminate(lsb); + lsb_terminate(lsb, NULL); return 1; } } @@ -551,7 +513,15 @@ int restore_global_data(lua_sandbox* lsb, const char* data_file) } -int serialize_binary(const void* src, size_t len, output_data* output) +void lsb_add_serialize_function(lua_State* lua, lua_CFunction fp) +{ + lua_pushstring(lua, serialize_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +int lsb_serialize_binary(const void* src, size_t len, lsb_output_data* output) { const char* uc = (const char*)src; for (unsigned i = 0; i < len; ++i) { @@ -575,3 +545,81 @@ int serialize_binary(const void* src, size_t len, output_data* output) } return 0; } + + +int lsb_serialize_double(lsb_output_data* output, double d) +{ + if (isnan(d)) { + return lsb_appends(output, not_a_number, 3); + } + if (d > INT_MAX) { + return lsb_appendf(output, "%0.17g", d); + } + + const int precision = 8; + const unsigned magnitude = 100000000; + char buffer[20]; + char* p = buffer; + int negative = 0; + + if (d < 0) { + negative = 1; + d = -d; + } + + int number = (int)d; + double tmp = (d - number) * magnitude; + unsigned fraction = (unsigned)tmp; + double diff = tmp - fraction; + + if (diff > 0.5) { + ++fraction; + if (fraction >= magnitude) { + fraction = 0; + ++number; + } + } else if (diff == 0.5 && ((fraction == 0) || (fraction & 1))) { + // bankers rounding + ++fraction; + } + + // output decimal fraction + if (fraction != 0) { + int nodigits = 1; + char c = 0; + for (int x = 0; x < precision; ++x) { + c = fraction % 10; + if (!(c == 0 && nodigits)) { + *p++ = c + '0'; + nodigits = 0; + } + fraction /= 10; + } + *p++ = '.'; + } + + // output number + do { + *p++ = (number % 10) + '0'; + number /= 10; + } + while (number > 0); + + size_t remaining = output->size - output->pos; + size_t needed = (p - buffer) + negative; + if (needed >= remaining) { + if (lsb_realloc_output(output, needed)) return 1; + } + + if (negative) { + output->data[output->pos++] = '-'; + } + do { + --p; + output->data[output->pos++] = *p; + } + while (p != buffer); + output->data[output->pos] = 0; + + return 0; +} diff --git a/src/lua_serialize_protobuf.c b/src/lsb_serialize_protobuf.c similarity index 90% rename from src/lua_serialize_protobuf.c rename to src/lsb_serialize_protobuf.c index ebe9445..d857615 100644 --- a/src/lua_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -4,14 +4,17 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/// @brief Lua sandbox Heka protobuf serialization @file +/** @brief Lua sandbox Heka protobuf serialization @file */ -#include "lua_serialize_protobuf.h" +#include "lsb_serialize_protobuf.h" #include #include #include +#include "lsb_output.h" +#include "lsb_private.h" + /** * Writes a varint encoded number to the output buffer. * @@ -20,7 +23,7 @@ * * @return int Zero on success, non-zero if out of memory. */ -static int pb_write_varint(output_data* d, unsigned long long i) +static int pb_write_varint(lsb_output_data* d, unsigned long long i) { size_t needed = 10; if (needed > d->size - d->pos) { @@ -49,7 +52,7 @@ static int pb_write_varint(output_data* d, unsigned long long i) * * @return int Zero on success, non-zero if out of memory. */ -static int pb_write_double(output_data* d, double i) +static int pb_write_double(lsb_output_data* d, double i) { size_t needed = sizeof(double); if (needed > d->size - d->pos) { @@ -71,7 +74,7 @@ static int pb_write_double(output_data* d, double i) * * @return int Zero on success, non-zero if out of memory. */ -static int pb_write_bool(output_data* d, int i) +static int pb_write_bool(lsb_output_data* d, int i) { size_t needed = 1; if (needed > d->size - d->pos) { @@ -96,7 +99,7 @@ static int pb_write_bool(output_data* d, int i) * * @return int Zero on success, non-zero if out of memory. */ -static int pb_write_tag(output_data* d, char id, char wire_type) +static int pb_write_tag(lsb_output_data* d, char id, char wire_type) { size_t needed = 1; if (needed > d->size - d->pos) { @@ -118,9 +121,9 @@ static int pb_write_tag(output_data* d, char id, char wire_type) * * @return int Zero on success, non-zero if out of memory. */ -static int pb_write_string(output_data* d, char id, const char* s, size_t len) +static int +pb_write_string(lsb_output_data* d, char id, const char* s, size_t len) { - if (pb_write_tag(d, id, 2)) { return 1; } @@ -151,7 +154,7 @@ static int pb_write_string(output_data* d, char id, const char* s, size_t len) * @return int Zero on success, non-zero if out of memory. */ static int -encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, +encode_string(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, int index) { int result = 0; @@ -180,7 +183,7 @@ encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, * @return int Zero on success, non-zero if out of memory. */ static int -encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, +encode_int(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, int index) { int result = 0; @@ -207,8 +210,9 @@ encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, * * @return int Zero on success, non-zero if out of memory. */ -static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, - const char* representation); +static int +encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, + const char* representation); /** @@ -221,8 +225,9 @@ static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, * * @return int Zero on success, non-zero if out of memory. */ -static int encode_field_array(lua_sandbox* lsb, output_data* d, int t, - const char* representation) +static int +encode_field_array(lua_sandbox* lsb, lsb_output_data* d, int t, + const char* representation) { int result = 0, first = (int)lua_objlen(lsb->lua, -1); lua_checkstack(lsb->lua, 2); @@ -249,7 +254,7 @@ static int encode_field_array(lua_sandbox* lsb, output_data* d, int t, * * @return int Zero on success, non-zero if out of memory. */ -static int encode_field_object(lua_sandbox* lsb, output_data* d) +static int encode_field_object(lua_sandbox* lsb, lsb_output_data* d) { int result = 0; const char* representation = NULL; @@ -264,8 +269,9 @@ static int encode_field_object(lua_sandbox* lsb, output_data* d) } -static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, - const char* representation) +static int +encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, + const char* representation) { int result = 1; size_t len; @@ -348,6 +354,7 @@ static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", lua_typename(lsb->lua, t)); result = 1; + break; } return result; } @@ -363,7 +370,7 @@ static int encode_field_value(lua_sandbox* lsb, output_data* d, int first, * * @return int Zero on success, non-zero if out of memory. */ -static int update_field_length(output_data* d, size_t len_pos) +static int update_field_length(lsb_output_data* d, size_t len_pos) { size_t len = d->pos - len_pos - 1; if (len < 128) { @@ -401,7 +408,7 @@ static int update_field_length(output_data* d, size_t len_pos) * @return int Zero on success, non-zero if out of memory. */ static int -encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, +encode_fields(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, int index) { int result = 0; @@ -435,9 +442,9 @@ encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, } -int serialize_table_as_pb(lua_sandbox* lsb, int index) +int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index) { - output_data* d = &lsb->output; + lsb_output_data* d = &lsb->output; d->pos = 0; size_t needed = 18; if (needed > d->size) { diff --git a/src/lua_serialize_protobuf.h b/src/lsb_serialize_protobuf.h similarity index 70% rename from src/lua_serialize_protobuf.h rename to src/lsb_serialize_protobuf.h index 4f6d739..d2f27d9 100644 --- a/src/lua_serialize_protobuf.h +++ b/src/lsb_serialize_protobuf.h @@ -4,11 +4,12 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/// Lua sandbox Heka protobuf serialization implementation @file -#ifndef lua_serialize_protobuf_h_ -#define lua_serialize_protobuf_h_ +/** Lua sandbox Heka protobuf serialization implementation @file */ -#include "lua_sandbox_private.h" +#ifndef lsb_serialize_protobuf_h_ +#define lsb_serialize_protobuf_h_ + +#include "lsb.h" /** * Serialize a specific Lua table structure as Protobuf @@ -18,6 +19,6 @@ * * @return int Zero on success, non-zero on failure. */ -int serialize_table_as_pb(lua_sandbox* lsb, int index); +int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index); #endif diff --git a/src/lua_bloom_filter.c b/src/lua_bloom_filter.c deleted file mode 100644 index d1247a9..0000000 --- a/src/lua_bloom_filter.c +++ /dev/null @@ -1,207 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua bloom_filter implementation @file */ - -#include -#include -#include -#include -#include - -#include "lua_bloom_filter.h" -#include "lua_serialize.h" -#include "xxhash.h" - -const char* lsb_bloom_filter = "lsb.bloom_filter"; -const char* lsb_bloom_filter_table = "bloom_filter"; - -struct bloom_filter -{ - size_t items; - size_t bytes; - size_t bits; - unsigned int hashes; - double probability; - char data[]; -}; - - -static int bloom_filter_new(lua_State* lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n == 2, 0, "incorrect number of arguments"); - int items = luaL_checkint(lua, 1); - luaL_argcheck(lua, 1 < items, 1, "items must be > 1"); - double probability = luaL_checknumber(lua, 2); - luaL_argcheck(lua, 0 < probability && 1 > probability, 2, "probability must be between 0 and 1"); - - size_t bits = (size_t)ceil(items * log(probability) / log(1 / pow(2, log(2)))); - size_t bytes = (size_t)ceil((double)bits / CHAR_BIT); - unsigned int hashes = (unsigned int)round(log(2) * bits/items); - - // subtract 1 for the byte already included in the struct - size_t nbytes = sizeof(bloom_filter) + bytes; - bloom_filter* bf = (bloom_filter*)lua_newuserdata(lua, nbytes); - bf->items = items; - bf->bits = bits; - bf->bytes = bytes; - bf->hashes = hashes; - bf->probability = probability; - memset(bf->data, 0, bf->bytes); - - luaL_getmetatable(lua, lsb_bloom_filter); - lua_setmetatable(lua, -2); - - return 1; -} - - -static bloom_filter* check_bloom_filter(lua_State* lua, int args) -{ - void* ud = luaL_checkudata(lua, 1, lsb_bloom_filter); - luaL_argcheck(lua, ud != NULL, 1, "invalid userdata type"); - luaL_argcheck(lua, args == lua_gettop(lua), 0, - "incorrect number of arguments"); - return (bloom_filter*)ud; -} - - -static int bloom_filter_add(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 2); - size_t len = 0; - double val = 0; - void *key = NULL; - switch (lua_type(lua, 2)) { - case LUA_TSTRING: - key = (void*) lua_tolstring(lua, 2, &len); - break; - case LUA_TNUMBER: - val = lua_tonumber(lua, 2); - len = sizeof(double); - key = &val; - break; - default: - luaL_argerror(lua, 2, "must be a string or number"); - break; - } - unsigned int bit = 0; - int added = 0; - - for (unsigned int i = 0; i < bf->hashes; ++i) { - bit = XXH32(key, (int)len, i) % bf->bits; - if (!(bf->data[bit/CHAR_BIT] & 1<<(bit%CHAR_BIT))) { - bf->data[bit/CHAR_BIT] |= 1<<(bit%CHAR_BIT); - added = 1; - } - } - - lua_pushboolean(lua, added); - return 1; -} - - -static int bloom_filter_query(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 2); - size_t len = 0; - double val = 0; - void *key = NULL; - switch (lua_type(lua, 2)) { - case LUA_TSTRING: - key = (void*) lua_tolstring(lua, 2, &len); - break; - case LUA_TNUMBER: - val = lua_tonumber(lua, 2); - len = sizeof(double); - key = &val; - break; - default: - luaL_argerror(lua, 2, "must be a string or number"); - break; - } - unsigned int bit = 0; - int found = 1; - - for (unsigned int i = 0; i < bf->hashes && found; ++i) { - bit = XXH32(key, (int)len, i) % bf->bits; - found = bf->data[bit/CHAR_BIT] & 1<<(bit%CHAR_BIT); - } - - lua_pushboolean(lua, found); - return 1; -} - - -static int bloom_filter_clear(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 1); - memset(bf->data, 0, bf->bytes); - return 0; -} - -static int bloom_filter_fromstring(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 2); - size_t len = 0; - const char* values = luaL_checklstring(lua, 2, &len); - if (len != bf->bytes) { - luaL_error(lua, "fromstring() bytes found: %d, expected %d", len, bf->bytes); - } - memcpy(bf->data, values, len); - return 0; -} - - -int serialize_bloom_filter(const char* key, bloom_filter* bf, output_data* output) -{ - output->pos = 0; - if (lsb_appendf(output, - "if %s == nil then %s = bloom_filter.new(%d, %g) end\n", - key, - key, - bf->items, - bf->probability)) { - return 1; - } - - if (lsb_appendf(output, "%s:fromstring(\"", key)) { - return 1; - } - if (serialize_binary(bf->data, bf->bytes, output)) return 1; - if (lsb_appends(output, "\")\n", 3)) { - return 1; - } - return 0; -} - - -static const struct luaL_reg bloom_filterlib_f[] = -{ - { "new", bloom_filter_new } - , { NULL, NULL } -}; - -static const struct luaL_reg bloom_filterlib_m[] = -{ - { "add", bloom_filter_add } - , { "query", bloom_filter_query } - , { "clear", bloom_filter_clear } - , { "fromstring", bloom_filter_fromstring } // used for data restoration - , { NULL, NULL } -}; - - -int luaopen_bloom_filter(lua_State* lua) -{ - luaL_newmetatable(lua, lsb_bloom_filter); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, bloom_filterlib_m); - luaL_register(lua, lsb_bloom_filter_table, bloom_filterlib_f); - return 1; -} diff --git a/src/lua_bloom_filter.h b/src/lua_bloom_filter.h deleted file mode 100644 index 302b640..0000000 --- a/src/lua_bloom_filter.h +++ /dev/null @@ -1,43 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua bloom_filter filter @file */ - -#ifndef lua_bloom_filter_h_ -#define lua_bloom_filter_h_ - -#include -#include "lsb_output.h" - -extern const char* lsb_bloom_filter; -extern const char* lsb_bloom_filter_table; -typedef struct bloom_filter bloom_filter; - -/** - * Serialize bloom_filter data - * - * @param key Lua variable name. - * @param b Bloom userdata object. - * @param output Output stream where the data is written. - * @return Zero on success - * - */ -int serialize_bloom_filter(const char* key, bloom_filter* bf, - output_data* output); - - -/** - * Bloom library loader - * - * @param lua Lua state. - * - * @return 1 on success - * - */ -int luaopen_bloom_filter(lua_State* lua); - - -#endif diff --git a/src/lua_circular_buffer.c b/src/lua_circular_buffer.c deleted file mode 100644 index b72718f..0000000 --- a/src/lua_circular_buffer.c +++ /dev/null @@ -1,1091 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// @brief Lua circular buffer implementation @file - -#include "cephes.h" -#include "lua_circular_buffer.h" -#include "lua_serialize.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -const char* lsb_circular_buffer = "lsb.circular_buffer"; -const char* lsb_circular_buffer_table = "circular_buffer"; - -#define COLUMN_NAME_SIZE 16 -#define UNIT_LABEL_SIZE 8 - -static const char* column_aggregation_methods[] = { "sum", "min", "max", "none", - "none", NULL }; -static const char* default_unit = "count"; - -typedef enum { - AGGREGATION_SUM = 0, - AGGREGATION_MIN = 1, - AGGREGATION_MAX = 2, - AGGREGATION_UNUSED = 3, - AGGREGATION_NONE = 4, - - MAX_AGGREGATION -} COLUMN_AGGREGATION; - -typedef enum { - OUTPUT_CBUF = 0, - OUTPUT_CBUFD = 1, - - LSB_OUTPUT_FORMAT -} OUTPUT_FORMAT; - -typedef struct -{ - char name[COLUMN_NAME_SIZE]; - char unit[UNIT_LABEL_SIZE]; - COLUMN_AGGREGATION aggregation; -} header_info; - -struct circular_buffer -{ - time_t current_time; - unsigned seconds_per_row; - unsigned current_row; - unsigned rows; - unsigned columns; - header_info* headers; - double* values; - int delta; - OUTPUT_FORMAT format; - int ref; - char bytes[1]; -}; - - -static time_t get_start_time(circular_buffer* cb) -{ - return cb->current_time - (cb->seconds_per_row * (cb->rows - 1)); -} - - -static void copy_cleared_row(circular_buffer* cb, double* cleared, size_t rows) -{ - size_t pool = 1; - size_t ask; - - while (rows > 0) { - if (rows >= pool) { - ask = pool; - } else { - ask = rows; - } - memcpy(cleared + (pool * cb->columns), cleared, sizeof(double) * cb->columns * ask); - rows -= ask; - pool += ask; - } -} - - -static void clear_rows(circular_buffer* cb, unsigned num_rows) -{ - if (num_rows >= cb->rows) { - num_rows = cb->rows; - } - unsigned row = cb->current_row; - ++row; - if (row >= cb->rows) {row = 0;} - for (unsigned c = 0; c < cb->columns; ++c) { - cb->values[(row * cb->columns) + c] = NAN; - } - double* cleared = &cb->values[row * cb->columns]; - if (row + num_rows - 1 >= cb->rows) { - copy_cleared_row(cb, cleared, cb->rows - row - 1); - for (unsigned c = 0; c < cb->columns; ++c) { - cb->values[c] = NAN; - } - copy_cleared_row(cb, cb->values, row + num_rows - 1 - cb->rows); - } else { - copy_cleared_row(cb, cleared, num_rows - 1); - } -} - - -static int circular_buffer_new(lua_State* lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n >= 3 && n <= 4, 0, "incorrect number of arguments"); - int rows = luaL_checkint(lua, 1); - luaL_argcheck(lua, 1 < rows, 1, "rows must be > 1"); - int columns = luaL_checkint(lua, 2); - luaL_argcheck(lua, 0 < columns, 2, "columns must be > 0"); - int seconds_per_row = luaL_checkint(lua, 3); - luaL_argcheck(lua, 0 < seconds_per_row, 3, "seconds_per_row is out of range"); - int delta = 0; - if (4 == n) { - delta = lua_toboolean(lua, 4); - } - - size_t header_bytes = sizeof(header_info) * columns; - size_t buffer_bytes = sizeof(double) * rows * columns; - size_t struct_bytes = sizeof(circular_buffer) - 1; // subtract 1 for the - // byte already included - // in the struct - - size_t nbytes = header_bytes + buffer_bytes + struct_bytes; - circular_buffer* cb = (circular_buffer*)lua_newuserdata(lua, nbytes); - cb->ref = LUA_NOREF; - cb->delta = delta; - cb->format = OUTPUT_CBUF; - cb->headers = (header_info*)&cb->bytes[0]; - cb->values = (double*)&cb->bytes[header_bytes]; - - luaL_getmetatable(lua, lsb_circular_buffer); - lua_setmetatable(lua, -2); - - cb->current_time = seconds_per_row * (rows - 1); - cb->current_row = rows - 1; - cb->rows = rows; - cb->columns = columns; - cb->seconds_per_row = seconds_per_row; - memset(cb->bytes, 0, header_bytes); - for (unsigned column_idx = 0; column_idx < cb->columns; ++column_idx) { - snprintf(cb->headers[column_idx].name, COLUMN_NAME_SIZE, - "Column_%d", column_idx + 1); - strncpy(cb->headers[column_idx].unit, default_unit, - UNIT_LABEL_SIZE - 1); - } - clear_rows(cb, rows); - return 1; -} - - -static circular_buffer* check_circular_buffer(lua_State* lua, int min_args) -{ - void* ud = luaL_checkudata(lua, 1, lsb_circular_buffer); - luaL_argcheck(lua, ud != NULL, 1, "invalid userdata type"); - luaL_argcheck(lua, min_args <= lua_gettop(lua), 0, - "incorrect number of arguments"); - return (circular_buffer*)ud; -} - - -static int check_row(circular_buffer* cb, double ns, int advance) -{ - time_t t = (time_t)(ns / 1e9); - t = t - (t % cb->seconds_per_row); - - int current_row = (int)(cb->current_time / cb->seconds_per_row); - int requested_row = (int)(t / cb->seconds_per_row); - int row_delta = requested_row - current_row; - int row = requested_row % cb->rows; - - if (row_delta > 0 && advance) { - clear_rows(cb, row_delta); - cb->current_time = t; - cb->current_row = row; - } else if (requested_row > current_row - || abs(row_delta) >= (int)cb->rows) { - return -1; - } - return row; -} - - -static int check_column(lua_State* lua, circular_buffer* cb, int arg) -{ - unsigned column = luaL_checkint(lua, arg); - luaL_argcheck(lua, 1 <= column && column <= cb->columns, arg, - "column out of range"); - --column; // make zero based - return column; -} - - -static void circular_buffer_add_delta(lua_State* lua, circular_buffer* cb, - double ns, int column, double value) -{ - // Storing the deltas in a Lua table allows the sandbox to account for the - // memory usage. todo: if too inefficient use a C data struct and report - // memory usage back to the sandbox - time_t t = (time_t)(ns / 1e9); - t = t - (t % cb->seconds_per_row); - lua_getglobal(lua, lsb_circular_buffer_table); - if (lua_istable(lua, -1)) { - if (cb->ref == LUA_NOREF) { - lua_newtable(lua); - cb->ref = luaL_ref(lua, -2); - } - // get the delta table for this cbuf - lua_rawgeti(lua, -1, cb->ref); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 2); // remove bogus table and cbuf table - return; - } - - // get the delta row using the timestamp - lua_rawgeti(lua, -1, (int)t); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 1); // remove non table entry - lua_newtable(lua); - lua_rawseti(lua, -2, (int)t); - lua_rawgeti(lua, -1, (int)t); - } - - // get the previous delta value - lua_rawgeti(lua, -1, column); - value += lua_tonumber(lua, -1); - lua_pop(lua, 1); // remove the old value - - // push the new delta - lua_pushnumber(lua, value); - lua_rawseti(lua, -2, column); - - lua_pop(lua, 2); // remove ref table, timestamped row - } else { - luaL_error(lua, "Could not find table %s", lsb_circular_buffer_table); - } - lua_pop(lua, 1); // remove the circular buffer table or failed nil - return; -} - - -static int circular_buffer_add(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 4); - double ns = luaL_checknumber(lua, 2); - int row = check_row(cb, - ns, - 1); // advance the buffer forward if - // necessary - int column = check_column(lua, cb, 3); - double value = luaL_checknumber(lua, 4); - if (row != -1) { - int i = (row * cb->columns) + column; - if (isnan(cb->values[i])) { - cb->values[i] = value; - } else { - cb->values[i] += value; - } - lua_pushnumber(lua, cb->values[i]); - if (cb->delta && value != 0) { - if (cb->headers[column].aggregation != AGGREGATION_SUM) { - value = cb->values[i]; - } - circular_buffer_add_delta(lua, cb, ns, column, value); - } - } else { - lua_pushnil(lua); - } - return 1; -} - - -static int circular_buffer_get(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 3); - int row = check_row(cb, - luaL_checknumber(lua, 2), - 0); - int column = check_column(lua, cb, 3); - - if (row != -1) { - lua_pushnumber(lua, cb->values[(row * cb->columns) + column]); - } else { - lua_pushnil(lua); - } - return 1; -} - - -static int circular_buffer_get_configuration(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 1); - - lua_pushnumber(lua, cb->rows); - lua_pushnumber(lua, cb->columns); - lua_pushnumber(lua, cb->seconds_per_row); - return 3; -} - - -static int circular_buffer_set(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 4); - double ns = luaL_checknumber(lua, 2); - int row = check_row(cb, ns, 1); // advance the buffer forward if - // necessary - int column = check_column(lua, cb, 3); - double value = luaL_checknumber(lua, 4); - - if (row != -1) { - int i = (row * cb->columns) + column; - double old = cb->values[i]; - switch (cb->headers[column].aggregation) { - case AGGREGATION_MIN: - if (isnan(cb->values[i]) || value < cb->values[i]) { - cb->values[i] = value; - if (cb->delta) { - circular_buffer_add_delta(lua, cb, ns, column, value); - } - } - break; - case AGGREGATION_MAX: - if (isnan(cb->values[i]) || value > cb->values[i]) { - cb->values[i] = value; - if (cb->delta) { - circular_buffer_add_delta(lua, cb, ns, column, value); - } - } - break; - default: - cb->values[i] = value; - if (cb->delta) { - if (!isnan(old)) { - value -= old; - } - circular_buffer_add_delta(lua, cb, ns, column, value); - } - break; - } - lua_pushnumber(lua, cb->values[i]); - } else { - lua_pushnil(lua); - } - return 1; -} - - -static int circular_buffer_set_header(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 3); - int column = check_column(lua, cb, 2); - const char* name = luaL_checkstring(lua, 3); - const char* unit = luaL_optstring(lua, 4, default_unit); - cb->headers[column].aggregation = luaL_checkoption(lua, 5, "sum", - column_aggregation_methods); - - strncpy(cb->headers[column].name, name, COLUMN_NAME_SIZE - 1); - char* n = cb->headers[column].name; - for (int j = 0; n[j] != 0; ++j) { - if (!isalnum(n[j])) { - n[j] = '_'; - } - } - strncpy(cb->headers[column].unit, unit, UNIT_LABEL_SIZE - 1); - n = cb->headers[column].unit; - for (int j = 0; n[j] != 0; ++j) { - if (n[j] != '/' && n[j] != '*' && !isalnum(n[j])) { - n[j] = '_'; - } - } - - lua_pushinteger(lua, column + 1); // return the 1 based Lua column - return 1; -} - - -static int circular_buffer_get_header(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 2); - int column = check_column(lua, cb, 2); - - lua_pushstring(lua, cb->headers[column].name); - lua_pushstring(lua, cb->headers[column].unit); - lua_pushstring(lua, - column_aggregation_methods[cb->headers[column].aggregation]); - return 3; -} - - -static double compute_sum(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double value = 0; - double result = 0; - unsigned row = start_row; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (isnan(value)) { - continue; - } - ++(*active_rows); - result += value; - } - while (row++ != end_row); - return result; -} - - -static double compute_avg(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double value = 0; - double result = 0; - unsigned row = start_row; - unsigned row_count = 0; - - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - result += value; - ++row_count; - } - } - while (row++ != end_row); - *active_rows = row_count; - return result / row_count; -} - - -static double compute_variance(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double avg = compute_avg(cb, column, start_row, end_row, active_rows); - if (isnan(avg)) return avg; - - double sum_squares = 0; - double value = 0; - unsigned row = start_row; - unsigned row_count = 0; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - value -= avg; - sum_squares += value * value; - ++row_count; - } - } - while (row++ != end_row); - return sum_squares / row_count; -} - - -static double compute_sd(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - return sqrt(compute_variance(cb, column, start_row, end_row, active_rows)); -} - - -static double compute_min(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double result = DBL_MAX; - double value = 0; - unsigned row = start_row; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - ++(*active_rows); - if (value < result) { - result = value; - } - } - } - while (row++ != end_row); - if (result == DBL_MAX) { - result = NAN; - } - return result; -} - - -static double compute_max(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double result = DBL_MIN; - double value = 0; - unsigned row = start_row; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - ++(*active_rows); - if (value > result) { - result = value; - } - } - } - while (row++ != end_row); - if (result == DBL_MIN) { - result = NAN; - } - return result; -} - - -static int circular_buffer_compute(lua_State* lua) -{ - static const char* functions[] = { "sum", "avg", "sd", "min", "max", - "variance", NULL }; - circular_buffer* cb = check_circular_buffer(lua, 3); - int function = luaL_checkoption(lua, 2, NULL, functions); - int column = check_column(lua, cb, 3); - - // optional range arguments - double start_ns = luaL_optnumber(lua, 4, get_start_time(cb) * 1e9); - double end_ns = luaL_optnumber(lua, 5, cb->current_time * 1e9); - luaL_argcheck(lua, end_ns >= start_ns, 5, "end must be >= start"); - - unsigned active_rows = 0; - int start_row = check_row(cb, start_ns, 0); - int end_row = check_row(cb, end_ns, 0); - if (-1 == start_row || -1 == end_row) { - lua_pushnil(lua); - lua_pushinteger(lua, active_rows); - return 2; - } - - double result = 0; - switch (function) { - case 0: - result = compute_sum(cb, column, start_row, end_row, &active_rows); - break; - case 1: - result = compute_avg(cb, column, start_row, end_row, &active_rows); - break; - case 2: - result = compute_sd(cb, column, start_row, end_row, &active_rows); - break; - case 3: - result = compute_min(cb, column, start_row, end_row, &active_rows); - break; - case 4: - result = compute_max(cb, column, start_row, end_row, &active_rows); - break; - case 5: - result = compute_variance(cb, column, start_row, end_row, &active_rows); - break; - } - - lua_pushnumber(lua, result); - lua_pushinteger(lua, active_rows); - return 2; -} - - -static void append_values(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - double ranked[]) -{ - unsigned row = start_row; - unsigned x = 0; - do { - if (row == cb->rows) { - row = 0; - } - ranked[x++] = cb->values[(row * cb->columns) + column]; - } - while (row++ != end_row); -} - - -static double rank_data(double* sorted[], size_t ranked_size) -{ - size_t next = 0, dupe_count = 0; - double tie_correction = 0; - for (size_t i = 0; i < ranked_size; ++i) { - next = i + 1; - if (i == ranked_size - 1 || (!(isnan(*sorted[i]) && isnan(*sorted[next])) - && *sorted[i] != *sorted[next])) { - if (dupe_count) { - double tie_rank = next - 0.5 * dupe_count; - for (size_t j = i - dupe_count; j < next; ++j) { - *sorted[j] = tie_rank; - } - ++dupe_count; - tie_correction += pow((double)dupe_count, 3.0) - dupe_count; - dupe_count = 0; - } else { - *sorted[i] = (double)next; - } - } else { - ++dupe_count; - } - } - tie_correction = 1.0 - tie_correction / (pow((double)ranked_size, 3.0) - ranked_size); - return tie_correction; -} - - -static int double_pp_compare(const void* a, const void* b) -{ - double* d1 = *(double**)a; - double* d2 = *(double**)b; - - if (isnan(*d1) && isnan(*d2)) return 0; - if (isnan(*d1)) return -1; - if (isnan(*d2)) return 1; - - if (*d1 < *d2) return -1; - if (*d1 == *d2) return 0; - return 1; -} - -// http://en.wikipedia.org/wiki/Mann-Whitney_U_test -static int circular_buffer_mannwhitneyu(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 6); - int n = lua_gettop(lua); - luaL_argcheck(lua, n <= 7, 0, "too many arguments"); - int column = check_column(lua, cb, 2); - - double start_1 = luaL_checknumber(lua, 3); - double end_1 = luaL_checknumber(lua, 4); - luaL_argcheck(lua, end_1 >= start_1, 4, "end_1 must be >= start_1"); - - double start_2 = luaL_checknumber(lua, 5); - double end_2 = luaL_checknumber(lua, 6); - luaL_argcheck(lua, end_2 >= start_2, 6, "end_2 must be >= start_2"); - luaL_argcheck(lua, end_1 < start_2 || end_2 < start_1, 4, "ranges must not overlap"); - - int use_continuity = 1; // optional argument - if (7 == n) { - luaL_argcheck(lua, lua_isboolean(lua, n), n, "use_continuity must be a boolean"); - use_continuity = lua_toboolean(lua, n); - } - - int start_1_row = check_row(cb, start_1, 0); - int end_1_row = check_row(cb, end_1, 0); - if (-1 == start_1_row || -1 == end_1_row) { - return 0; - } - int start_2_row = check_row(cb, start_2, 0); - int end_2_row = check_row(cb, end_2, 0); - if (-1 == start_2_row || -1 == end_2_row) { - return 0; - } - - size_t n1 = (size_t)((end_1 - start_1) / 1e9 / cb->seconds_per_row) + 1; - size_t n2 = (size_t)((end_2 - start_2) / 1e9 / cb->seconds_per_row) + 1; - size_t ranked_size = n1 + n2; - // note: user could temporarily exceed the sandbox memory limit here without - // detection - double* ranked = (double*)malloc(sizeof(double) * ranked_size); - if (!ranked) { - return 0; - } - append_values(cb, column, start_1_row, end_1_row, ranked); - append_values(cb, column, start_2_row, end_2_row, ranked + n1); - - double** sorted = (double**)malloc(sizeof(double*) * ranked_size); - if (!sorted) { - free(ranked); - return 0; - } - for (size_t i = 0; i < ranked_size; ++i) { - sorted[i] = ranked + i; - } - - qsort(sorted, ranked_size, sizeof(double*), double_pp_compare); - double tie_correction = rank_data(sorted, ranked_size); - free(sorted); - if (!tie_correction) { // data set values are all identical - free(ranked); - return 0; - } - - double sum = 0; - for (size_t i = 0; i < n1; ++i) { - if (!isnan(ranked[i])) { - sum += ranked[i]; - } - } - free(ranked); - - double u1 = sum - (n1 * (n1 + 1)) / 2.0; - double u2 = n1 * n2 - u1; - double lu = u1 > u2 ? u1 : u2; - - double z = 0; - double sd = sqrt(tie_correction * n1 * n2 * (n1 + n2 + 1) / 12.0); - if (use_continuity) { - // normal approximation for prob calc with continuity correction - z = fabs((lu - 0.5 - n1 * n2 / 2.0) / sd); - } else { - // normal approximation for prob calc - z = fabs((lu - n1 * n2 / 2.0) / sd); - } - - lua_pushnumber(lua, u1); - lua_pushnumber(lua, ndtr(-z)); - return 2; -} - - -static int circular_buffer_current_time(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 0); - lua_pushnumber(lua, cb->current_time * 1e9); - return 1; // return the current time -} - -static int circular_buffer_format(lua_State* lua) -{ - static const char* output_types[] = { "cbuf", "cbufd", NULL }; - circular_buffer* cb = check_circular_buffer(lua, 2); - luaL_argcheck(lua, 2 == lua_gettop(lua), 0, - "incorrect number of arguments"); - - cb->format = luaL_checkoption(lua, 2, NULL, output_types); - lua_pop(lua, 1); // remove the format - return 1; // return the circular buffer object -} - - -const char* get_output_format(circular_buffer* cb) -{ - switch (cb->format) { - case OUTPUT_CBUFD: - return "cbufd"; - default: - return "cbuf"; - } -} - - -static void read_time_row(char** p, circular_buffer* cb) -{ - cb->current_time = (time_t)strtoll(*p, &*p, 10); - cb->current_row = strtoul(*p, &*p, 10); -} - - -static int read_double(char** p, double* value) -{ - while (**p != 0 && isspace(**p)) { - ++*p; - } - if (0 == **p) return 0; - - if (**p == not_a_number[0]) { - ++*p; - if (0 == **p || **p != not_a_number[1]) return 0; - - ++*p; - if (0 == **p || **p != not_a_number[2]) return 0; - - ++*p; - *value = NAN; - } else { - *value = strtod(*p, &*p); - } - return 1; -} - - -static void circular_buffer_delta_fromstring(lua_State* lua, - circular_buffer* cb, - char** p) -{ - double value, ns = 0; - size_t pos = 0; - while (read_double(&*p, &value)) { - if (pos == 0) { // new row, starts with a time_t - ns = value * 1e9; - } else { - circular_buffer_add_delta(lua, cb, ns, (int)(pos - 1), value); - } - if (pos == cb->columns) { - pos = 0; - } else { - ++pos; - } - } - if (pos != 0) { - lua_pushstring(lua, "fromstring() invalid delta"); - lua_error(lua); - } - return; -} - - -static int circular_buffer_fromstring(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 2); - const char* values = luaL_checkstring(lua, 2); - - char* p = (char*)values; - read_time_row(&p, cb); - - size_t pos = 0; - size_t len = cb->rows * cb->columns; - double value; - while (pos < len && read_double(&p, &value)) { - cb->values[pos++] = value; - } - if (pos == len) { - if (cb->delta) { - circular_buffer_delta_fromstring(lua, cb, &p); - } - } else { - luaL_error(lua, "fromstring() too few values: %d, expected %d", pos, len); - } - if (read_double(&p, &value)) { - luaL_error(lua, "fromstring() too many values, more than: %d", len); - } - return 0; -} - - -int output_circular_buffer_full(circular_buffer* cb, output_data* output) -{ - unsigned column_idx; - unsigned row_idx = cb->current_row + 1; - for (unsigned i = 0; i < cb->rows; ++i, ++row_idx) { - if (row_idx >= cb->rows) row_idx = 0; - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (column_idx != 0) { - if (lsb_appendc(output, '\t')) return 1; - } - if (serialize_double(output, - cb->values[(row_idx * cb->columns) + column_idx])) { - return 1; - } - } - if (lsb_appendc(output, '\n')) return 1; - } - return 0; -} - - -int output_circular_buffer_cbufd(lua_State* lua, circular_buffer* cb, - output_data* output) -{ - lua_getglobal(lua, lsb_circular_buffer_table); - if (lua_istable(lua, -1)) { - // get the delta table for this cbuf - lua_rawgeti(lua, -1, cb->ref); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 2); // remove bogus table and cbuf table - luaL_error(lua, "Could not find the delta table"); - } - lua_pushnil(lua); - while (lua_next(lua, -2) != 0) { - if (!lua_istable(lua, -1)) { - luaL_error(lua, "Invalid delta table structure"); - } - if (serialize_double(output, lua_tonumber(lua, -2))) return 1; - for (unsigned column_idx = 0; column_idx < cb->columns; - ++column_idx) { - if (lsb_appendc(output, '\t')) return 1; - lua_rawgeti(lua, -1, column_idx); - if (LUA_TNIL == lua_type(lua, -1)) { - if (lsb_appends(output, not_a_number, 3)) return 1; - } else { - if (serialize_double(output, lua_tonumber(lua, -1))) return 1; - } - lua_pop(lua, 1); // remove the number - } - if (lsb_appendc(output, '\n')) return 1; - lua_pop(lua, 1); // remove the value, keep the key - } - lua_pop(lua, 1); // remove the delta table - - // delete the delta table - luaL_unref(lua, -1, cb->ref); - cb->ref = LUA_NOREF; - } else { - luaL_error(lua, "Could not find table %s", lsb_circular_buffer_table); - } - lua_pop(lua, 1); // remove the circular buffer table or failed nil - return 0; -} - - -int output_circular_buffer(lua_State* lua, circular_buffer* cb, - output_data* output) -{ - if (OUTPUT_CBUFD == cb->format) { - if (cb->ref == LUA_NOREF) return 0; - } - - if (lsb_appendf(output, - "{\"time\":%lld,\"rows\":%d,\"columns\":%d,\"seconds_per_row\":%d,\"column_info\":[", - (long long)get_start_time(cb), - cb->rows, - cb->columns, - cb->seconds_per_row)) { - return 1; - } - - unsigned column_idx; - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (column_idx != 0) { - if (lsb_appendc(output, ',')) return 1; - } - if (lsb_appendf(output, "{\"name\":\"%s\",\"unit\":\"%s\",\"aggregation\":\"%s\"}", - cb->headers[column_idx].name, - cb->headers[column_idx].unit, - column_aggregation_methods[cb->headers[column_idx].aggregation])) { - return 1; - } - } - if (lsb_appends(output, "]}\n", 3)) return 1; - - if (OUTPUT_CBUFD == cb->format) { - return output_circular_buffer_cbufd(lua, cb, output); - } - return output_circular_buffer_full(cb, output); -} - - -int serialize_circular_buffer_delta(lua_State* lua, circular_buffer* cb, - output_data* output) -{ - if (cb->ref == LUA_NOREF) return 0; - lua_getglobal(lua, lsb_circular_buffer_table); - if (lua_istable(lua, -1)) { - // get the delta table for this cbuf - lua_rawgeti(lua, -1, cb->ref); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 2); // remove bogus table and cbuf table - luaL_error(lua, "Could not find the delta table"); - } - lua_pushnil(lua); - while (lua_next(lua, -2) != 0) { - if (!lua_istable(lua, -1)) { - luaL_error(lua, "Invalid delta table structure"); - } - if (lsb_appendc(output, ' ')) return 1; - if (serialize_double(output, lua_tonumber(lua, -2))) return 1; - - for (unsigned column_idx = 0; column_idx < cb->columns; - ++column_idx) { - if (lsb_appendc(output, ' ')) return 1; - lua_rawgeti(lua, -1, column_idx); - if (serialize_double(output, lua_tonumber(lua, -1))) return 1; - lua_pop(lua, 1); // remove the number - } - lua_pop(lua, 1); // remove the value, keep the key - } - lua_pop(lua, 1); // remove the delta table - - // delete the delta table - lua_pushnil(lua); - lua_rawseti(lua, -2, cb->ref); - cb->ref = LUA_NOREF; - } else { - luaL_error(lua, "Could not find table %s", lsb_circular_buffer_table); - } - lua_pop(lua, 1); // remove the circular buffer table or failed nil - return 0; -} - - -int serialize_circular_buffer(lua_State* lua, const char* key, - circular_buffer* cb, output_data* output) -{ - output->pos = 0; - char* delta = ""; - if (cb->delta) { - delta = ", true"; - } - if (lsb_appendf(output, - "if %s == nil then %s = circular_buffer.new(%d, %d, %d%s) end\n", - key, - key, - cb->rows, - cb->columns, - cb->seconds_per_row, - delta)) { - return 1; - } - - unsigned column_idx; - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (lsb_appendf(output, "%s:set_header(%d, \"%s\", \"%s\", \"%s\")\n", - key, - column_idx + 1, - cb->headers[column_idx].name, - cb->headers[column_idx].unit, - column_aggregation_methods[cb->headers[column_idx].aggregation])) { - return 1; - } - } - - if (lsb_appendf(output, "%s:fromstring(\"%lld %d", - key, - (long long)cb->current_time, - cb->current_row)) { - return 1; - } - for (unsigned row_idx = 0; row_idx < cb->rows; ++row_idx) { - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (lsb_appendc(output, ' ')) return 1; - if (serialize_double(output, - cb->values[(row_idx * cb->columns) + column_idx])) { - return 1; - } - } - } - if (serialize_circular_buffer_delta(lua, cb, output)) { - return 1; - } - if (lsb_appends(output, "\")\n", 3)) { - return 1; - } - return 0; -} - - -static const struct luaL_reg circular_bufferlib_f[] = -{ - { "new", circular_buffer_new } - , { NULL, NULL } -}; - -static const struct luaL_reg circular_bufferlib_m[] = -{ - { "add", circular_buffer_add } - , { "get", circular_buffer_get } - , { "get_configuration", circular_buffer_get_configuration } - , { "set", circular_buffer_set } - , { "set_header", circular_buffer_set_header } - , { "get_header", circular_buffer_get_header } - , { "compute", circular_buffer_compute } - , { "mannwhitneyu", circular_buffer_mannwhitneyu } - , { "current_time", circular_buffer_current_time } - , { "format", circular_buffer_format } - , { "fromstring", circular_buffer_fromstring } // used for data restoration - , { NULL, NULL } -}; - - -int luaopen_circular_buffer(lua_State* lua) -{ - luaL_newmetatable(lua, lsb_circular_buffer); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, circular_bufferlib_m); - luaL_register(lua, lsb_circular_buffer_table, circular_bufferlib_f); - return 1; -} diff --git a/src/lua_circular_buffer.h b/src/lua_circular_buffer.h deleted file mode 100644 index 21beed9..0000000 --- a/src/lua_circular_buffer.h +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// @brief Lua circular buffer - time series data store for sandboxes @file -#ifndef lua_circular_buffer_h_ -#define lua_circular_buffer_h_ - -#include -#include "lua_sandbox_private.h" - -extern const char* lsb_circular_buffer; -extern const char* lsb_circular_buffer_table; -typedef struct circular_buffer circular_buffer; - -/** - * Output the circular buffer user data - * - * @param lua Lua state. - * @param cb Circular buffer userdata object. - * @param output Output stream where the data is written. - * @return Zero on success - * - */ -int output_circular_buffer(lua_State* lua, circular_buffer* cb, - output_data* output); - -/** - * Get the current output format string. - * - * @param cb Circular buffer userdata object. - * - * @return const char* - */ -const char* get_output_format(circular_buffer* cb); - -/** - * Serialize the circular buffer user data - * - * @param lua Lua state. - * @param key Lua variable name. - * @param cb Circular buffer userdata object. - * @param output Output stream where the data is written. - * @return Zero on success - * - */ -int serialize_circular_buffer(lua_State* lua, const char* key, - circular_buffer* cb, output_data* output); - -/** - * Circular buffer library loader - * - * @param lua Lua state. - * - * @return 1 on success - * - */ -int luaopen_circular_buffer(lua_State* lua); - - -#endif diff --git a/src/lua_hyperloglog.c b/src/lua_hyperloglog.c deleted file mode 100644 index 92e6240..0000000 --- a/src/lua_hyperloglog.c +++ /dev/null @@ -1,193 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua hyperloglog implementation @file */ - -#include -#include -#include -#include - -#include "lua_hyperloglog.h" -#include "lua_serialize.h" - -const char* lsb_hyperloglog = "lsb.hyperloglog"; -const char* lsb_hyperloglog_table = "hyperloglog"; - -static const char* hll_magic = "HYLL"; - -/* The cached cardinality MSB is used to signal validity of the cached value. */ -#define HLL_INVALIDATE_CACHE(hll) (hll)->card[7] |= (1<<7) -#define HLL_VALID_CACHE(hll) (((hll)->card[7] & (1<<7)) == 0) - -static int hyperloglog_new(lua_State* lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n == 0, 0, "incorrect number of arguments"); - - size_t nbytes = sizeof(hyperloglog); - hyperloglog* hll = (hyperloglog*)lua_newuserdata(lua, nbytes); - memcpy(hll->magic, hll_magic, sizeof(hll->magic)); - hll->encoding = HLL_DENSE; - HLL_INVALIDATE_CACHE(hll); - memset(hll->notused, 0, sizeof(hll->notused)); - memset(hll->registers, 0, HLL_REGISTERS_SIZE); - - luaL_getmetatable(lua, lsb_hyperloglog); - lua_setmetatable(lua, -2); - - return 1; -} - - -static hyperloglog* check_hyperloglog(lua_State* lua, int args) -{ - void* ud = luaL_checkudata(lua, 1, lsb_hyperloglog); - luaL_argcheck(lua, ud != NULL, 1, "invalid userdata type"); - luaL_argcheck(lua, args == lua_gettop(lua), 0, - "incorrect number of arguments"); - return (hyperloglog*)ud; -} - - -static int hyperloglog_add(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 2); - size_t len = 0; - double val = 0; - void* key = NULL; - switch (lua_type(lua, 2)) { - case LUA_TSTRING: - key = (void*)lua_tolstring(lua, 2, &len); - break; - case LUA_TNUMBER: - val = lua_tonumber(lua, 2); - len = sizeof(double); - key = &val; - break; - default: - luaL_argerror(lua, 2, "must be a string or number"); - break; - } - - int altered = 0; - if (1 == hllDenseAdd(hll->registers, (unsigned char*)key, len)) { - HLL_INVALIDATE_CACHE(hll); - altered = 1; - } - - lua_pushboolean(lua, altered); - return 1; -} - - -static int hyperloglog_count(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 1); - uint64_t card; - /* Check if the cached cardinality is valid. */ - if (HLL_VALID_CACHE(hll)) { - /* Just return the cached value. */ - card = (uint64_t)hll->card[0]; - card |= (uint64_t)hll->card[1] << 8; - card |= (uint64_t)hll->card[2] << 16; - card |= (uint64_t)hll->card[3] << 24; - card |= (uint64_t)hll->card[4] << 32; - card |= (uint64_t)hll->card[5] << 40; - card |= (uint64_t)hll->card[6] << 48; - card |= (uint64_t)hll->card[7] << 56; - } else { - /* Recompute it and update the cached value. */ - card = hllCount(hll); - hll->card[0] = card & 0xff; - hll->card[1] = (card >> 8) & 0xff; - hll->card[2] = (card >> 16) & 0xff; - hll->card[3] = (card >> 24) & 0xff; - hll->card[4] = (card >> 32) & 0xff; - hll->card[5] = (card >> 40) & 0xff; - hll->card[6] = (card >> 48) & 0xff; - hll->card[7] = (card >> 56) & 0xff; - } - - lua_pushnumber(lua, (double)card); - return 1; -} - - -static int hyperloglog_clear(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 1); - memset(hll->registers, 0, HLL_REGISTERS_SIZE); - HLL_INVALIDATE_CACHE(hll); - return 0; -} - - -static int hyperloglog_fromstring(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 2); - size_t len = 0; - const char* values = luaL_checklstring(lua, 2, &len); - if (len != sizeof(hyperloglog) - 1) { - luaL_error(lua, "fromstring() bytes found: %d, expected %d", - len, sizeof(hyperloglog) - 1); - } - if (memcmp(values, hll_magic, sizeof(hll->magic)) != 0) { - luaL_error(lua, "fromstring() HYLL header not found"); - } - if (values[5] != HLL_DENSE) { - luaL_error(lua, "fromstring() invalid encoding"); - } - memcpy(hll, values, sizeof(hyperloglog) - 1); - return 0; -} - - -int serialize_hyperloglog(const char* key, hyperloglog* hll, output_data* output) -{ - output->pos = 0; - if (lsb_appendf(output, - "if %s == nil then %s = hyperloglog.new() end\n", key, key)) { - return 1; - } - - if (lsb_appendf(output, "%s:fromstring(\"", key)) { - return 1; - } - if (serialize_binary(hll, sizeof(hyperloglog) - 1, output)) return 1; - if (lsb_appends(output, "\")\n", 3)) { - return 1; - } - return 0; -} - - -static const struct luaL_reg hyperlogloglib_f[] = -{ - { "new", hyperloglog_new } - , { NULL, NULL } -}; - - -static const struct luaL_reg hyperlogloglib_m[] = -{ - { "add", hyperloglog_add } - , { "count", hyperloglog_count } - , { "clear", hyperloglog_clear } - , { "fromstring", hyperloglog_fromstring } // used for data restoration - , { NULL, NULL } -}; - - -int luaopen_hyperloglog(lua_State* lua) -{ - luaL_newmetatable(lua, lsb_hyperloglog); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, hyperlogloglib_m); - luaL_register(lua, lsb_hyperloglog_table, hyperlogloglib_f); - return 1; -} diff --git a/src/lua_hyperloglog.h b/src/lua_hyperloglog.h deleted file mode 100644 index f07b8cd..0000000 --- a/src/lua_hyperloglog.h +++ /dev/null @@ -1,84 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua HyperLogLog probabilistic cardinality approximation leveraging - * the Redis HLL dense represention/implementation @file */ - -#ifndef lua_hyperloglog_h_ -#define lua_hyperloglog_h_ - -#include -#include -#include "lua_sandbox_private.h" - -extern const char* lsb_hyperloglog; -extern const char* lsb_hyperloglog_table; - -#define HLL_P 14 /* The greater is P, the smaller the error. */ -#define HLL_REGISTERS (1< -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#define PATH_DELIMITER '\\' -#else -#define PATH_DELIMITER '/' -#endif - -#ifndef MAX_PATH -#define MAX_PATH 260 -#endif - -const char* disable_none[] = { NULL }; -const char* package_table = "package"; -const char* loaded_table = "loaded"; - - -// If necessary, add an empty metatable to flag the table as non-data -// during preservation. -static void add_empty_metatable(lua_State* lua) -{ - if (lua_getmetatable(lua, -1) == 0) { - lua_newtable(lua); - lua_setmetatable(lua, -2); - } else { - lua_pop(lua, 1); - } -} - - -void load_library(lua_State* lua, const char* table, lua_CFunction f, - const char** disable) -{ - lua_pushcfunction(lua, f); - lua_call(lua, 0, 1); - - if (strlen(table) == 0) { // Handle the special "" base table. - for (int i = 0; disable[i]; ++i) { - lua_pushnil(lua); - lua_setfield(lua, LUA_GLOBALSINDEX, disable[i]); - } - } else { - for (int i = 0; disable[i]; ++i) { - lua_pushnil(lua); - lua_setfield(lua, -2, disable[i]); - } - add_empty_metatable(lua); - } -} - -#ifndef LUA_JIT -void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize) -{ - lua_sandbox* lsb = (lua_sandbox*)ud; - - void* nptr = NULL; - if (nsize == 0) { - free(ptr); - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= (unsigned)osize; - } else { - unsigned new_state_memory = - (unsigned)(lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize); - if (0 == lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] - || new_state_memory - <= lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]) { - nptr = realloc(ptr, nsize); - if (nptr != NULL) { - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = - new_state_memory; - if (lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] - > lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; - } - } - } - } - return nptr; -} -#endif - -void instruction_manager(lua_State* lua, lua_Debug* ar) -{ - if (LUA_HOOKCOUNT == ar->event) { - luaL_error(lua, "instruction_limit exceeded"); - } -} - - -size_t instruction_usage(lua_sandbox* lsb) -{ - return lua_gethookcount(lsb->lua) - lua_gethookcountremaining(lsb->lua); -} - - -void sandbox_terminate(lua_sandbox* lsb) -{ - if (lsb->lua) { - lua_close(lsb->lua); - lsb->lua = NULL; - } - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = 0; - lsb->state = LSB_TERMINATED; -} - - -void update_output_stats(lua_sandbox* lsb) -{ - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = (unsigned)lsb->output.pos; - if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] - > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; - } -} - - -int output(lua_State* lua) -{ - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "output() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - int n = lua_gettop(lua); - if (n == 0) { - return luaL_argerror(lsb->lua, 0, "must have at least one argument"); - } - lsb_output(lsb, 1, n, 1); - return 0; -} - -int require_library(lua_State* lua) -{ - const char* name = luaL_checkstring(lua, 1); - lua_getglobal(lua, package_table); - if (!lua_istable(lua, -1)) { - return luaL_error(lua, "%s table is missing", package_table); - } - lua_getfield(lua, -1, loaded_table); - if (!lua_istable(lua, -1)) { - return luaL_error(lua, "%s.%s table is missing", package_table, loaded_table); - } - lua_getfield(lua, -1, name); - if (!lua_isnil(lua, -1)) { - return 1; // returned the cache copy - } - lua_pop(lua, 1); // remove the nil - int pos = lua_gettop(lua); - lua_pushboolean(lua, 1); - lua_setfield(lua, pos, name); // mark it as loaded to prevent a dependency loop - - if (strcmp(name, LUA_STRLIBNAME) == 0) { - load_library(lua, name, luaopen_string, disable_none); - } else if (strcmp(name, LUA_MATHLIBNAME) == 0) { - load_library(lua, name, luaopen_math, disable_none); - } else if (strcmp(name, LUA_TABLIBNAME) == 0) { - load_library(lua, name, luaopen_table, disable_none); - } else if (strcmp(name, LUA_OSLIBNAME) == 0) { - const char* disable[] = { "execute", "exit", "remove", "rename", - "setlocale", "tmpname", NULL }; - load_library(lua, name, luaopen_os, disable); - } else if (strcmp(name, lsb_circular_buffer_table) == 0) { - load_library(lua, name, luaopen_circular_buffer, disable_none); - } else if (strcmp(name, lsb_bloom_filter_table) == 0) { - load_library(lua, name, luaopen_bloom_filter, disable_none); - } else if (strcmp(name, lsb_hyperloglog_table) == 0) { - load_library(lua, name, luaopen_hyperloglog, disable_none); - } else if (strcmp(name, "lpeg") == 0) { - load_library(lua, name, luaopen_lpeg, disable_none); - } else if (strcmp(name, "cjson") == 0) { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "require_library() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - const char* disable[] = { "new", "encode_keep_buffer", NULL }; - load_library(lua, name, luaopen_cjson, disable); - if (set_encode_max_buffer(lua, -1, (unsigned)lsb->output.maxsize)) { - return luaL_error(lua, "cjson encode buffer could not be configured"); - } - lua_pushvalue(lua, -1); - lua_setglobal(lua, name); - } else if (strcmp(name, "struct") == 0) { - load_library(lua, name, luaopen_struct, disable_none); - } else { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "require_library() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - if (!lsb->require_path) { - return luaL_error(lua, "require_library() external modules are disabled"); - } - - int i = 0; - while (name[i]) { - if (!isalnum(name[i]) && name[i] != '_') { - return luaL_error(lua, "invalid module name '%s'", name); - } - ++i; - } - char fn[MAX_PATH]; - i = snprintf(fn, MAX_PATH, "%s%c%s.lua", lsb->require_path, PATH_DELIMITER, - name); - if (i < 0 || i >= MAX_PATH) { - return luaL_error(lua, "require_path exceeded %d", MAX_PATH); - } - - if (luaL_dofile(lua, fn) != 0) { - return luaL_error(lua, "%s", lua_tostring(lua, -1)); - } - add_empty_metatable(lua); - } - lua_pushvalue(lua, -1); - lua_setfield(lua, pos, name); - return 1; -} diff --git a/src/lua_sandbox_private.h b/src/lua_sandbox_private.h deleted file mode 100644 index 67614c0..0000000 --- a/src/lua_sandbox_private.h +++ /dev/null @@ -1,126 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// Lua sandbox private functions @file -#ifndef lua_sandbox_private_h_ -#define lua_sandbox_private_h_ - -#include "lsb_output.h" -#include "lsb.h" - -#include -#include - -#ifdef _WIN32 -#define snprintf _snprintf -#endif - -struct lua_sandbox { - lua_State* lua; - void* parent; - lsb_state state; - output_data output; - char* lua_file; - char* require_path; - unsigned usage[LSB_UT_MAX][LSB_US_MAX]; - char error_message[LSB_ERROR_SIZE]; -}; - -extern const char* disable_none[]; -extern const char* package_table; -extern const char* loaded_table; - -//////////////////////////////////////////////////////////////////////////////// -/// Sandbox management functions. -//////////////////////////////////////////////////////////////////////////////// -/** - * Performs the library load and secures the sandbox environment for use. - * - * @param lua Pointer to the Lua state. - * @param table Name of the table being loaded. - * @param f Pointer to the table load function. - * @param disable Array of function names to disable in the loaded table. - */ -void load_library(lua_State* lua, const char* table, lua_CFunction f, - const char** disable); -#ifndef LUA_JIT -/** -* Implementation of the memory allocator for the Lua state. -* -* See: http://www.lua.org/manual/5.1/manual.html#lua_Alloc -* -* @param ud Pointer to the lua_sandbox -* @param ptr Pointer to the memory block being allocated/reallocated/freed. -* @param osize The original size of the memory block. -* @param nsize The new size of the memory block. -* -* @return void* A pointer to the memory block. -*/ -void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize); -#endif - - -/** - * Lua hook to monitor the instruction usage of the sandbox. - * - * @param lua Pointer to the Lua state. - * @param ar Pointer to the Lua debug interface. - */ -void instruction_manager(lua_State* lua, lua_Debug* ar); - -/** - * Extracts the current instruction usage count from the Lua state. - * - * @param lsb Pointer to the sandbox. - * - * @return size_t The number of instructions used in the last function call - */ -size_t instruction_usage(lua_sandbox* lsb); - -/** - * Tears down the sandbox on error. - * - * @param lsb Pointer to the sandbox. - */ -void sandbox_terminate(lua_sandbox* lsb); - -/** - * Helper function to update the output statistics. - * - * @param lsb Pointer to the sandbox. - */ -void update_output_stats(lua_sandbox* lsb); - - -//////////////////////////////////////////////////////////////////////////////// -/// Lua to C function interface -//////////////////////////////////////////////////////////////////////////////// -/** - * Collect sandbox output into an in memory buffer. - * - * @param lua Pointer to the Lua state. - * - * @return int Returns zero values on the stack. - */ -int output(lua_State* lua); - -/** - * Overridden 'require' used to load optional sandbox libraries in global space. - * - * @param lua Pointer to the Lua state. - * - * @return int Returns 1 value on the stack (for the standard modules a table - * for the LPEG grammars, userdata). - */ -int require_library(lua_State* lua); - -LUALIB_API int luaopen_cjson(lua_State* L); -int set_encode_max_buffer(lua_State* L, int index, unsigned maxsize); - -LUALIB_API int luaopen_lpeg(lua_State* L); -LUALIB_API int luaopen_struct(lua_State* L); - -#endif diff --git a/src/lua_serialize.h b/src/lua_serialize.h deleted file mode 100644 index 19ececb..0000000 --- a/src/lua_serialize.h +++ /dev/null @@ -1,173 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Lua sandbox serialization @file */ - -#ifndef lua_serialize_h_ -#define lua_serialize_h_ - -#include -#include "lua_sandbox_private.h" - -extern const char* not_a_number; - -typedef int (*lsb_SerializeFunction) (lua_State *L, - const char* key, - void* ud, - output_data* output); - -typedef struct -{ - const void* ptr; - size_t name_pos; -} table_ref; - -typedef struct -{ - size_t size; - size_t pos; - table_ref* array; -} table_ref_array; - -typedef struct -{ - FILE* fh; - output_data keys; - table_ref_array tables; - const void* globals; -} serialization_data; - -/** - * Serialize all user global data to disk. - * - * @param lsb Pointer to the sandbox. - * @param data_file Filename where the data will be written (create/overwrite) - * - * @return int Zero on success, non-zero on failure. - */ -int preserve_global_data(lua_sandbox* lsb, const char* data_file); - -/** - * More efficient serialization of a double to a string - * - * @param output Pointer the output collector. - * @param d Double value to convert to a string. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_double(output_data* output, double d); - -/** - * Serializes a Lua table structure. - * - * @param lsb Pointer to the sandbox. - * @param data Pointer to the serialization state data. - * @param parent Index pointing to the parent's name in the table array. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent); - -/** - * Serializes a Lua data value. - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index where the data resides. - * @param output Pointer the output collector. - * - * @return int - */ -int serialize_data(lua_sandbox* lsb, int index, output_data* output); - -/** - * Determines the name of the userdata type - * - * @param lua Lua State - * @param index Index on the stack where the userdata pointer resides - * @param tname userdata table name - * - * @return const char* NULL if not found - */ -void *userdata_type(lua_State* lua, int index, const char *tname); - -/** - * Serializes a table key value pair. - * - * @param lsb Pointer to the sandbox. - * @param data Pointer to the serialization state data. - * @param parent Index pointing to the parent's name in the table array. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent); - -/** - * Checks to see a string key starts with a '_' in which case it will be - * ignored. - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index where the key resides. - * - * @return int True if the key should not be serialized. - */ -int ignore_key(lua_sandbox* lsb, int index); - - -/** - * Looks for a table to see if it has already been processed. - * - * @param tra Pointer to the table references. - * @param ptr Pointer value of the table. - * - * @return table_ref* NULL if not found. - */ -table_ref* find_table_ref(table_ref_array* tra, const void* ptr); - -/** - * Adds a table to the processed array. - * - * @param tra Pointer to the table references. - * @param ptr Pointer value of the table. - * @param name_pos Index pointing to name in the table array. - * - * @return table_ref* Pointer to the table reference or NULL if out of memory. - */ -table_ref* add_table_ref(table_ref_array* tra, const void* ptr, - size_t name_pos); - -/** - * Helper function to determine what data should not be serialized. - * - * @param lsb Pointer to the sandbox. - * @param data Pointer to the serialization state data. - * @param index Lua stack index where the data resides. - * - * @return int - */ -int ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index); - -/** - * Restores previously serialized data from disk. - * - * @param lsb Pointer to the sandbox. - * @param data_file Filename from where the data will be read. - * - * @return int Zero on success, non-zero on failure. - */ -int restore_global_data(lua_sandbox* lsb, const char* data_file); - -/** - * Serializes a binary data to a Lua string. - * - * @param src Pointer to the binary data. - * @param len Length in bytes of the data to output. - * @param output Pointer the output collector. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_binary(const void *src, size_t len, output_data* output); - -#endif diff --git a/src/redis_hyperloglog.c b/src/redis_hyperloglog.c deleted file mode 100644 index 8e5c450..0000000 --- a/src/redis_hyperloglog.c +++ /dev/null @@ -1,503 +0,0 @@ -/* hyperloglog.c - Redis HyperLogLog probabilistic cardinality approximation. - * This file implements the algorithm and the exported Redis commands. - * - * Copyright (c) 2014, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* This file has been modified for use in the Mozilla lua_sandbox. Dependencies - on the redis.h header file and the unneed sparse and raw implementations have - been removed. The dense data representation remains unchanged. */ - -#include -#include -#include - -#include "lua_hyperloglog.h" - - -/* The Redis HyperLogLog implementation is based on the following ideas: - * - * * The use of a 64 bit hash function as proposed in [1], in order to don't - * limited to cardinalities up to 10^9, at the cost of just 1 additional - * bit per register. - * * The use of 16384 6-bit registers for a great level of accuracy, using - * a total of 12k per key. - * * The use of the Redis string data type. No new type is introduced. - * * No attempt is made to compress the data structure as in [1]. Also the - * algorithm used is the original HyperLogLog Algorithm as in [2], with - * the only difference that a 64 bit hash function is used, so no correction - * is performed for values near 2^32 as in [1]. - * - * [1] Heule, Nunkesser, Hall: HyperLogLog in Practice: Algorithmic - * Engineering of a State of The Art Cardinality Estimation Algorithm. - * - * [2] P. Flajolet, �?ric Fusy, O. Gandouet, and F. Meunier. Hyperloglog: The - * analysis of a near-optimal cardinality estimation algorithm. - * - * The "dense" representation where every entry is represented by a - * 6-bit integer. - * - * HLL header - * === - * - * Both the dense and sparse representation have a 16 byte header as follows: - * - * +------+---+-----+----------+ - * | HYLL | E | N/U | Cardin. | - * +------+---+-----+----------+ - * - * The first 4 bytes are a magic string set to the bytes "HYLL". - * "E" is one byte encoding, currently set to HLL_DENSE or - * HLL_SPARSE. N/U are three not used bytes. - * - * The "Cardin." field is a 64 bit integer stored in little endian format - * with the latest cardinality computed that can be reused if the data - * structure was not modified since the last computation (this is useful - * because there are high probabilities that HLLADD operations don't - * modify the actual data structure and hence the approximated cardinality). - * - * When the most significant bit in the most significant byte of the cached - * cardinality is set, it means that the data structure was modified and - * we can't reuse the cached value that must be recomputed. - * - * Dense representation - * === - * - * The dense representation used by Redis is the following: - * - * +--------+--------+--------+------// //--+ - * |11000000|22221111|33333322|55444444 .... | - * +--------+--------+--------+------// //--+ - * - * The 6 bits counters are encoded one after the other starting from the - * LSB to the MSB, and using the next bytes as needed. - * - */ - -#define HLL_P_MASK (HLL_REGISTERS-1) /* Mask to index register. */ -#define HLL_REGISTER_MAX ((1< 6 - * - * Right shift b0 of 'fb' bits. - * - * +--------+ - * |11000000| <- Initial value of b0 - * |00000011| <- After right shift of 6 pos. - * +--------+ - * - * Left shift b1 of bits 8-fb bits (2 bits) - * - * +--------+ - * |22221111| <- Initial value of b1 - * |22111100| <- After left shift of 2 bits. - * +--------+ - * - * OR the two bits, and finally AND with 111111 (63 in decimal) to - * clean the higher order bits we are not interested in: - * - * +--------+ - * |00000011| <- b0 right shifted - * |22111100| <- b1 left shifted - * |22111111| <- b0 OR b1 - * | 111111| <- (b0 OR b1) AND 63, our value. - * +--------+ - * - * We can try with a different example, like pos = 0. In this case - * the 6-bit counter is actually contained in a single byte. - * - * b0 = 6 * pos / 8 = 0 - * - * +--------+ - * |11000000| <- Our byte at b0 - * +--------+ - * - * fb = 6 * pos % 8 = 0 - * - * So we right shift of 0 bits (no shift in practice) and - * left shift the next byte of 8 bits, even if we don't use it, - * but this has the effect of clearing the bits so the result - * will not be affacted after the OR. - * - * ------------------------------------------------------------------------- - * - * Setting the register is a bit more complex, let's assume that 'val' - * is the value we want to set, already in the right range. - * - * We need two steps, in one we need to clear the bits, and in the other - * we need to bitwise-OR the new bits. - * - * Let's try with 'pos' = 1, so our first byte at 'b' is 0, - * - * "fb" is 6 in this case. - * - * +--------+ - * |11000000| <- Our byte at b0 - * +--------+ - * - * To create a AND-mask to clear the bits about this position, we just - * initialize the mask with the value 63, left shift it of "fs" bits, - * and finally invert the result. - * - * +--------+ - * |00111111| <- "mask" starts at 63 - * |11000000| <- "mask" after left shift of "ls" bits. - * |00111111| <- "mask" after invert. - * +--------+ - * - * Now we can bitwise-AND the byte at "b" with the mask, and bitwise-OR - * it with "val" left-shifted of "ls" bits to set the new bits. - * - * Now let's focus on the next byte b1: - * - * +--------+ - * |22221111| <- Initial value of b1 - * +--------+ - * - * To build the AND mask we start again with the 63 value, right shift - * it by 8-fb bits, and invert it. - * - * +--------+ - * |00111111| <- "mask" set at 2&6-1 - * |00001111| <- "mask" after the right shift by 8-fb = 2 bits - * |11110000| <- "mask" after bitwise not. - * +--------+ - * - * Now we can mask it with b+1 to clear the old bits, and bitwise-OR - * with "val" left-shifted by "rs" bits to set the new value. - */ - -/* Note: if we access the last counter, we will also access the b+1 byte - * that is out of the array, but sds strings always have an implicit null - * term, so the byte exists, and we can skip the conditional (or the need - * to allocate 1 byte more explicitly). */ - -/* Store the value of the register at position 'regnum' into variable 'target'. - * 'p' is an array of unsigned bytes. */ -#define HLL_DENSE_GET_REGISTER(target,p,regnum) do { \ - uint8_t *_p = (uint8_t*) p; \ - unsigned long _byte = regnum*HLL_BITS/8; \ - unsigned long _fb = regnum*HLL_BITS&7; \ - unsigned long _fb8 = 8 - _fb; \ - unsigned long b0 = _p[_byte]; \ - unsigned long b1 = _p[_byte+1]; \ - target = ((b0 >> _fb) | (b1 << _fb8)) & HLL_REGISTER_MAX; \ -} while(0) - -/* Set the value of the register at position 'regnum' to 'val'. - * 'p' is an array of unsigned bytes. */ -#define HLL_DENSE_SET_REGISTER(p,regnum,val) do { \ - uint8_t *_p = (uint8_t*) p; \ - unsigned long _byte = regnum*HLL_BITS/8; \ - unsigned long _fb = regnum*HLL_BITS&7; \ - unsigned long _fb8 = 8 - _fb; \ - unsigned long _v = val; \ - _p[_byte] &= ~(HLL_REGISTER_MAX << _fb); \ - _p[_byte] |= _v << _fb; \ - _p[_byte+1] &= ~(HLL_REGISTER_MAX >> _fb8); \ - _p[_byte+1] |= _v >> _fb8; \ -} while(0) - - -/* ========================= HyperLogLog algorithm ========================= */ - -/* Our hash function is MurmurHash2, 64 bit version. - * It was modified for Redis in order to provide the same result in - * big and little endian archs (endian neutral). */ -uint64_t MurmurHash64A(const void* key, int len, unsigned int seed) -{ - const uint64_t m = 0xc6a4a7935bd1e995; - const int r = 47; - uint64_t h = seed ^ (len * m); - const uint8_t* data = (const uint8_t*)key; - const uint8_t* end = data + (len - (len & 7)); - - while (data != end) { - uint64_t k; - -#if (BYTE_ORDER == LITTLE_ENDIAN) - k = *((uint64_t*)data); -#else - k = (uint64_t)data[0]; - k |= (uint64_t)data[1] << 8; - k |= (uint64_t)data[2] << 16; - k |= (uint64_t)data[3] << 24; - k |= (uint64_t)data[4] << 32; - k |= (uint64_t)data[5] << 40; - k |= (uint64_t)data[6] << 48; - k |= (uint64_t)data[7] << 56; -#endif - - k *= m; - k ^= k >> r; - k *= m; - h ^= k; - h *= m; - data += 8; - } - - switch (len & 7) { - case 7: - h ^= (uint64_t)data[6] << 48; - case 6: - h ^= (uint64_t)data[5] << 40; - case 5: - h ^= (uint64_t)data[4] << 32; - case 4: - h ^= (uint64_t)data[3] << 24; - case 3: - h ^= (uint64_t)data[2] << 16; - case 2: - h ^= (uint64_t)data[1] << 8; - case 1: - h ^= (uint64_t)data[0]; - h *= m; - }; - - h ^= h >> r; - h *= m; - h ^= h >> r; - return h; -} - -/* Given a string element to add to the HyperLogLog, returns the length - * of the pattern 000..1 of the element hash. As a side effect 'regp' is - * set to the register index this element hashes to. */ -int hllPatLen(unsigned char* ele, size_t elesize, long* regp) -{ - uint64_t hash, bit, index; - int count; - - /* Count the number of zeroes starting from bit HLL_REGISTERS - * (that is a power of two corresponding to the first bit we don't use - * as index). The max run can be 64-P+1 bits. - * - * Note that the final "1" ending the sequence of zeroes must be - * included in the count, so if we find "001" the count is 3, and - * the smallest count possible is no zeroes at all, just a 1 bit - * at the first position, that is a count of 1. - * - * This may sound like inefficient, but actually in the average case - * there are high probabilities to find a 1 after a few iterations. */ - hash = MurmurHash64A(ele, (int)elesize, 0xadc83b19ULL); - index = hash & HLL_P_MASK; /* Register index. */ - hash |= ((uint64_t)1 << 63); /* Make sure the loop terminates. */ - bit = HLL_REGISTERS; /* First bit not used to address the register. */ - count = 1; /* Initialized to 1 since we count the "00000...1" pattern. */ - while ((hash & bit) == 0) { - count++; - bit <<= 1; - } - *regp = (int)index; - return count; -} - -/* ================== Dense representation implementation ================== */ - -/* "Add" the element in the dense hyperloglog data structure. - * Actually nothing is added, but the max 0 pattern counter of the subset - * the element belongs to is incremented if needed. - * - * 'registers' is expected to have room for HLL_REGISTERS plus an - * additional byte on the right. This requirement is met by sds strings - * automatically since they are implicitly null terminated. - * - * The function always succeed, however if as a result of the operation - * the approximated cardinality changed, 1 is returned. Otherwise 0 - * is returned. */ -int hllDenseAdd(uint8_t* registers, unsigned char* ele, size_t elesize) -{ - uint8_t oldcount, count; - long index; - - /* Update the register if this element produced a longer run of zeroes. */ - count = hllPatLen(ele, elesize, &index); - HLL_DENSE_GET_REGISTER(oldcount, registers, index); - if (count > oldcount) { - HLL_DENSE_SET_REGISTER(registers, index, count); - return 1; - } else { - return 0; - } -} - -/* Compute SUM(2^-reg) in the dense representation. - * PE is an array with a pre-computer table of values 2^-reg indexed by reg. - * As a side effect the integer pointed by 'ezp' is set to the number - * of zero registers. */ -double hllDenseSum(uint8_t* registers, double* PE, int* ezp) -{ - double E = 0; - int j, ez = 0; - - /* Redis default is to use 16384 registers 6 bits each. The code works - * with other values by modifying the defines, but for our target value - * we take a faster path with unrolled loops. */ - if (HLL_REGISTERS == 16384 && HLL_BITS == 6) { - uint8_t* r = registers; - unsigned long r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, - r10, r11, r12, r13, r14, r15; - for (j = 0; j < 1024; j++) { - /* Handle 16 registers per iteration. */ - r0 = r[0] & 63; if (r0 == 0) ez++; - r1 = (r[0] >> 6 | r[1] << 2) & 63; if (r1 == 0) ez++; - r2 = (r[1] >> 4 | r[2] << 4) & 63; if (r2 == 0) ez++; - r3 = (r[2] >> 2) & 63; if (r3 == 0) ez++; - r4 = r[3] & 63; if (r4 == 0) ez++; - r5 = (r[3] >> 6 | r[4] << 2) & 63; if (r5 == 0) ez++; - r6 = (r[4] >> 4 | r[5] << 4) & 63; if (r6 == 0) ez++; - r7 = (r[5] >> 2) & 63; if (r7 == 0) ez++; - r8 = r[6] & 63; if (r8 == 0) ez++; - r9 = (r[6] >> 6 | r[7] << 2) & 63; if (r9 == 0) ez++; - r10 = (r[7] >> 4 | r[8] << 4) & 63; if (r10 == 0) ez++; - r11 = (r[8] >> 2) & 63; if (r11 == 0) ez++; - r12 = r[9] & 63; if (r12 == 0) ez++; - r13 = (r[9] >> 6 | r[10] << 2) & 63; if (r13 == 0) ez++; - r14 = (r[10] >> 4 | r[11] << 4) & 63; if (r14 == 0) ez++; - r15 = (r[11] >> 2) & 63; if (r15 == 0) ez++; - - /* Additional parens will allow the compiler to optimize the - * code more with a loss of precision that is not very relevant - * here (floating point math is not commutative!). */ - E += (PE[r0] + PE[r1]) + (PE[r2] + PE[r3]) + (PE[r4] + PE[r5]) + - (PE[r6] + PE[r7]) + (PE[r8] + PE[r9]) + (PE[r10] + PE[r11]) + - (PE[r12] + PE[r13]) + (PE[r14] + PE[r15]); - r += 12; - } - } else { - for (j = 0; j < HLL_REGISTERS; j++) { - unsigned long reg; - - HLL_DENSE_GET_REGISTER(reg, registers, j); - if (reg == 0) { - ez++; - /* Increment E at the end of the loop. */ - } else { - E += PE[reg]; /* Precomputed 2^(-reg[j]). */ - } - } - E += ez; /* Add 2^0 'ez' times. */ - } - *ezp = ez; - return E; -} - -/* ========================= HyperLogLog Count ============================== - * This is the core of the algorithm where the approximated count is computed. - * The function uses the lower level hllDenseSum() function as helpers to - * compute the SUM(2^-reg) part of the computation, which is - * representation-specific, while all the rest is common. */ - -/* Return the approximated cardinality of the set based on the harmonic - * mean of the registers values. 'hdr' points to the start of the SDS - * representing the String object holding the HLL representation. - * - * If the sparse representation of the HLL object is not valid, the integer - * pointed by 'invalid' is set to non-zero, otherwise it is left untouched. */ -uint64_t hllCount(hyperloglog* hdr) -{ - double m = HLL_REGISTERS; - double E, alpha = 0.7213 / (1 + 1.079 / m); - int j, ez; /* Number of registers equal to 0. */ - -/* We precompute 2^(-reg[j]) in a small table in order to - * speedup the computation of SUM(2^-register[0..i]). */ - static int initialized = 0; - static double PE[64]; - if (!initialized) { - PE[0] = 1; /* 2^(-reg[j]) is 1 when m is 0. */ - for (j = 1; j < 64; j++) { - /* 2^(-reg[j]) is the same as 1/2^reg[j]. */ - PE[j] = 1.0 / (1ULL << j); - } - initialized = 1; - } - -/* Compute SUM(2^-register[0..i]). */ - E = hllDenseSum(hdr->registers, PE, &ez); - -/* Muliply the inverse of E for alpha_m * m^2 to have the raw estimate. */ - E = (1 / E) * alpha * m * m; - -/* Use the LINEARCOUNTING algorithm for small cardinalities. - * For larger values but up to 72000 HyperLogLog raw approximation is - * used since linear counting error starts to increase. However HyperLogLog - * shows a strong bias in the range 2.5*16384 - 72000, so we try to - * compensate for it. */ - if (E < m * 2.5 && ez != 0) { - E = m * log(m / ez); /* LINEARCOUNTING() */ - } else if (m == 16384 && E < 72000) { - /* We did polynomial regression of the bias for this range, this - * way we can compute the bias for a given cardinality and correct - * according to it. Only apply the correction for P=14 that's what - * we use and the value the correction was verified with. */ - double bias = 5.9119 * 1.0e-18 * (E * E * E * E) - - 1.4253 * 1.0e-12 * (E * E * E) + - 1.2940 * 1.0e-7 * (E * E) - - 5.2921 * 1.0e-3 * E + - 83.3216; - E -= E * (bias / 100); - } -/* We don't apply the correction for E > 1/30 of 2^32 since we use - * a 64 bit function and 6 bit counters. To apply the correction for - * 1/30 of 2^64 is not needed since it would require a huge set - * to approach such a value. */ - return (uint64_t)E; -} diff --git a/src/test/lua/bloom_filter_errors.lua b/src/test/lua/bloom_filter_errors.lua deleted file mode 100644 index 0ff7936..0000000 --- a/src/test/lua/bloom_filter_errors.lua +++ /dev/null @@ -1,43 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "bloom_filter" - -function process(tc) - if tc == 0 then - local bf = bloom_filter.new(2) -- new() incorrect # args - elseif tc == 1 then - local bf = bloom_filter.new(nil, 0.01) -- new() non numeric item - elseif tc == 2 then - local bf = bloom_filter.new(0, 0.01) -- invalid items - elseif tc == 3 then - local bf = bloom_filter.new(2, nil) -- nil probability - elseif tc == 4 then - local bf = bloom_filter.new(2, 0) -- invalid probability - elseif tc == 5 then - local bf = bloom_filter.new(2, 1) -- invalid probability - elseif tc == 6 then - local bf = bloom_filter.new(20, 0.01) - bf:add() --incorrect # args - elseif tc == 7 then - local bf = bloom_filter.new(20, 0.01) - bf:add({}) --incorrect argument type - elseif tc == 8 then - local bf = bloom_filter.new(20, 0.01) - bf:query() --incorrect # args - elseif tc == 9 then - local bf = bloom_filter.new(20, 0.01) - bf:query({}) --incorrect argument type - elseif tc == 10 then - local bf = bloom_filter.new(20, 0.01) - bf:clear(1) --incorrect # args - elseif tc == 11 then - local bf = bloom_filter.new(20, 0.01) - bf:fromstring({}) --incorrect argument type - elseif tc == 12 then - local bf = bloom_filter.new(20, 0.01) - bf:fromstring(" ") --incorrect argument length - end -return 0 -end diff --git a/src/test/lua/circular_buffer.lua b/src/test/lua/circular_buffer.lua index 238759e..1a9fc45 100644 --- a/src/test/lua/circular_buffer.lua +++ b/src/test/lua/circular_buffer.lua @@ -21,233 +21,5 @@ function process(ts) end function report(tc) - if tc == 0 then - write_output(data) - elseif tc == 1 then - cbufs = {} - for i=1,3,1 do - cbufs[i] = circular_buffer.new(2,1,1) - cbufs[i]:set_header(1, "Header_1", "count") - end - elseif tc == 2 then - write_output(cbufs[1]) - elseif tc == 3 then - local stats = circular_buffer.new(5, 1, 1) - stats:set(1e9, 1, 1) - stats:set(2e9, 1, 2) - stats:set(3e9, 1, 3) - stats:set(4e9, 1, 4) - local t, c = stats:compute("sum", 1) - if 10 ~= t then - error(string.format("no range sum = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("avg", 1) - if 2.5 ~= t then - error(string.format("no range avg = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("variance", 1) - if 1.25 ~= t then - error(string.format("no range variance = %G", t)) - end - t, c = stats:compute("sd", 1) - if math.sqrt(1.25) ~= t then - error(string.format("no range sd = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("min", 1) - if 1 ~= t then - error(string.format("no range min = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("max", 1) - if 4 ~= t then - error(string.format("no range max = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - - t = stats:compute("sum", 1, 3e9, 4e9) - if 7 ~= t then - error(string.format("range 3-4 sum = %G", t)) - end - t = stats:compute("avg", 1, 3e9, 4e9) - if 3.5 ~= t then - error(string.format("range 3-4 avg = %G", t)) - end - t = stats:compute("sd", 1, 3e9, 4e9) - if math.sqrt(0.25) ~= t then - error(string.format("range 3-4 sd = %G", t)) - end - - t = stats:compute("sum", 1, 3e9) - if 7 ~= t then - error(string.format("range 3- = %G", t)) - end - t = stats:compute("sum", 1, 3e9, nil) - if 7 ~= t then - error(string.format("range 3-nil = %G", t)) - end - t = stats:compute("sum", 1, nil, 2e9) - if 3 ~= t then - error(string.format("range nil-2 sum = %G", t)) - end - t = stats:compute("sum", 1, 11e9, 14e9) - if nil ~= t then - error(string.format("out of range = %G", t)) - end - elseif tc == 4 then - local stats = circular_buffer.new(4, 1, 1) - stats:set(1e9, 1, 0/0) - stats:set(2e9, 1, 8) - stats:set(3e9, 1, 8) - local t = stats:compute("avg", 1) - if 8 ~= t then - error(string.format("no range avg = %G", t)) - end - elseif tc == 5 then - local stats = circular_buffer.new(2, 1, 1) - local nan = stats:get(0, 1) - if nan == nan then - error(string.format("initial value is a number %G", nan)) - end - local v = stats:set(0, 1, 1) - if v ~= 1 then - error(string.format("set failed = %G", v)) - end - v = stats:add(0, 1, 0/0) - if v == v then - error(string.format("adding nan returned a number %G", v)) - end - elseif tc == 6 then - local stats = circular_buffer.new(2, 1, 1) - local cbuf_time = stats:current_time() - if cbuf_time ~= 1e9 then - error(string.format("current_time = %G", cbuf_time)) - end - local v = stats:set(0, 1, 1) - if stats:get(0, 1) ~= 1 then - error(string.format("set failed = %G", v)) - end - stats:fromstring("1 1 nan 99") - local nan = stats:get(0, 1) - if nan == nan then - error(string.format("restored value is a number %G", nan)) - end - v = stats:get(1e9, 1) - if v ~= 99 then - error(string.format("restored value is %G", v)) - end - elseif tc == 7 then - local empty = circular_buffer.new(4,1,1) - local nan = empty:compute("avg", 1) - if nan == nan then - error(string.format("avg is a number %G", nan)) - end - nan = empty:compute("sd", 1) - if nan == nan then - error(string.format("std is a number %G", nan)) - end - nan = empty:compute("max", 1) - if nan == nan then - error(string.format("max is a number %G", m)) - end - nan = empty:compute("min", 1) - if nan == nan then - error(string.format("min is a number %G", m)) - end - elseif tc == 8 then - local cb = circular_buffer.new(20,1,1) - local u, p = cb:mannwhitneyu(1, 0e9, 9e9, 10e9, 19e9) - if u or p then - error("all the same values should return nil results") - end - elseif tc == 9 then -- default - local cb = circular_buffer.new(40,1,1) - local data = {15309,14092,13661,13412,14205,15042,14142,13820,14917,13953,14320,14472,15133,13790,14539,14129,14363,14202,13841,13610,13759,14428,14851,13838,13819,14468,14989,15557,14380,13500,14818,14632,13631,14663,14532,14188,14537,14109,13925,15022} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 20e9, 21e9, 40e9) - if u ~= 171 or math.floor(p * 100000) ~= 22037 then - error(string.format("u is %g p is %g", u, p)) - end - elseif tc == 10 then -- no continuity correction - local cb = circular_buffer.new(40,1,1) - local data = {15309,14092,13661,13412,14205,15042,14142,13820,14917,13953,14320,14472,15133,13790,14539,14129,14363,14202,13841,13610,13759,14428,14851,13838,13819,14468,14989,15557,14380,13500,14818,14632,13631,14663,14532,14188,14537,14109,13925,15022} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 20e9, 21e9, 40e9, false) - if u ~= 171 or math.floor(p * 100000) ~= 21638 then - error(string.format("u is %g p is %g", u, p)) - end - elseif tc == 11 then -- tie correction - local cb = circular_buffer.new(40,1,1) - local data = {15309,14092,13661,13412,14205,15042,14142,13820,14917,13953,14320,14472,15133,13790,14539,14129,14363,14202,13841,13610,13759,14428,14851,13838,13819,14468,14989,15557,14380,13500,14818,14632,13631,14663,14532,14188,14537,14109,13925,15309} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 20e9, 21e9, 40e9) - if u ~= 168.5 or math.floor(p * 100000) ~= 20084 then - error(string.format("u is %g p is %g", u, p)) - end - elseif tc == 12 then - local cb = circular_buffer.new(40,1,1) - local u, p = cb:mannwhitneyu(1, 41e9, 60e9, 61e9, 80e9) - if u or p then - error("times outside of buffer should return nil results") - end - elseif tc == 13 then - local cb = circular_buffer.new(10,1,1) - local data = {1,1,1,1,1,1,1,1,1,1} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 5e9, 6e9, 10e9) - if u or p then - error("all the same values should return nil results") - end - elseif tc == 14 then - local cb = circular_buffer.new(10,1,1) - local rows, cols, spr = cb:get_configuration() - assert(rows == 10, "invalid rows") - assert(cols == 1 , "invalid columns") - assert(spr == 1 , "invalid seconds_per_row") - elseif tc == 15 then - local cb = circular_buffer.new(10,1,1) - local args = {"widget", "count", "max"} - local col = cb:set_header(1, args[1], args[2], args[3]) - assert(col == 1, "invalid column") - local n, u, m = cb:get_header(col) - assert(n == args[1], "invalid name") - assert(u == args[2], "invalid unit") - assert(m == args[3], "invalid aggregation_method") - elseif tc == 16 then - local cb = circular_buffer.new(10,1,1) - assert(not cb:get(10*1e9, 1), "value found beyond the end of the buffer") - cb:set(20*1e9, 1, 1) - assert(not cb:get(10*1e9, 1), "value found beyond the start of the buffer") - elseif tc == 17 then -- default - local cb = circular_buffer.new(120,1,1) - local data = {1,1,1,2,1,3,3,6,4,0/0,0/0,0/0,1,0/0,2,0/0,0/0,0/0,0/0,0/0,1,5,1,0/0,1,1,0/0,0/0,3,4,1,1,1,0/0,7,1,0/0,6,0/0,0/0,1,3,4,3,0/0,1,5,0/0,1,0/0,0/0,1,6,4,0/0,4,2,6,4,3,2,6,2,11,2,0/0,2,0/0,2,0/0,0/0,0/0,4,0/0,3,2,0/0,0/0,1,2,2,2,1,1,0/0,3,0/0,4,0/0,0/0,2,3,5,6,3,1,0/0,0/0,3,2,0/0,4,1,2,1,1,0/0,0/0,0/0,0/0,0/0,0/0,0/0,7,1,1,2,1,0/0,0/0} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u1 = cb:mannwhitneyu(1, 61e9, 120e9, 1e9, 60e9) - local u2 = cb:mannwhitneyu(1, 1e9, 60e9, 61e9, 120e9) - if u1 + u2 ~= 3600 then - error(string.format("u1 is %g u2 is %g %g", u1, u2, maxu)) - end - end + write_output(data) end diff --git a/src/test/lua/circular_buffer_errors.lua b/src/test/lua/circular_buffer_errors.lua index a6eddd0..0151999 100644 --- a/src/test/lua/circular_buffer_errors.lua +++ b/src/test/lua/circular_buffer_errors.lua @@ -5,112 +5,8 @@ require "circular_buffer" function process(tc) - if tc == 0 then - local cb = circular_buffer.new(2) -- new() incorrect # args - elseif tc == 1 then - local cb = circular_buffer.new(nil, 1, 1) -- new() non numeric row - elseif tc == 2 then - local cb = circular_buffer.new(1, 1, 1) -- new() 1 row - elseif tc == 3 then - local cb = circular_buffer.new(2, nil, 1) -- new() non numeric column - elseif tc == 4 then - local cb = circular_buffer.new(2, 0, 1) -- new() zero column - elseif tc == 5 then - local cb = circular_buffer.new(2, 1, nil) -- new() non numeric seconds_per_row - elseif tc == 6 then - local cb = circular_buffer.new(2, 1, 0) -- new() zero seconds_per_row - elseif tc == 7 then - error("removed test") - elseif tc == 8 then + if tc == 0 then local cb = circular_buffer.new(1000, 10, 1) -- new() too much memory - elseif tc == 9 then - local cb = circular_buffer.new(2, 1, 1) -- set() out of range column - cb:set(0, 2, 1.0) - elseif tc == 10 then - local cb = circular_buffer.new(2, 1, 1) -- set() zero column - cb:set(0, 0, 1.0) - elseif tc == 11 then - local cb = circular_buffer.new(2, 1, 1) -- set() non numeric column - cb:set(0, nil, 1.0) - elseif tc == 12 then - local cb = circular_buffer.new(2, 1, 1) -- set() non numeric time - cb:set(nil, 1, 1.0) - elseif tc == 13 then - local cb = circular_buffer.new(2, 1, 1) -- get() invalid object - local invalid = 1 - cb.get(invalid, 1, 1) - elseif tc == 14 then - local cb = circular_buffer.new(2, 1, 1) -- set() non numeric value - cb:set(0, 1, nil) - elseif tc == 15 then - local cb = circular_buffer.new(2, 1, 1) -- set() incorrect # args - cb:set(0) - elseif tc == 16 then - local cb = circular_buffer.new(2, 1, 1) -- add() incorrect # args - cb:add(0) - elseif tc == 17 then - local cb = circular_buffer.new(2, 1, 1) -- get() incorrect # args - cb:get(0) - elseif tc == 18 then - local cb = circular_buffer.new(2, 1, 1) -- compute() incorrect # args - cb:compute(0) - elseif tc == 19 then - local cb = circular_buffer.new(2, 1, 1) -- compute() incorrect function - cb:compute("func", 1) - elseif tc == 20 then - local cb = circular_buffer.new(2, 1, 1) -- compute() incorrect column - cb:compute("sum", 0) - elseif tc == 21 then - local cb = circular_buffer.new(2, 1, 1) -- compute() start > end - cb:compute("sum", 1, 2e9, 1e9) - elseif tc == 22 then - local cb = circular_buffer.new(2, 1, 1) -- format() invalid - cb:format("invalid") - elseif tc == 23 then - local cb = circular_buffer.new(2, 1, 1) -- format() extra - cb:format("cbuf", true) - elseif tc == 24 then - local cb = circular_buffer.new(2, 1, 1) -- format() missing - cb:format() - elseif tc == 25 then - local cb = circular_buffer.new(2, 1, 1) -- too few - cb:fromstring("") - elseif tc == 26 then - local cb = circular_buffer.new(2, 1, 1) -- too few invalid - cb:fromstring("0 0 na 1") - elseif tc == 27 then - local cb = circular_buffer.new(2, 1, 1) -- too many - cb:fromstring("0 0 1 2 3") - elseif tc == 28 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu() -- incorrect # args - elseif tc == 29 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(nil, 0, 0, 0, 0) -- non numeric column - elseif tc == 30 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(0, 0, 0, 0, 0) -- invalid column - elseif tc == 31 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 0, 5, 2, 7) -- overlapping x,y - elseif tc == 32 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 5, 0, 2, 7) -- inverted x - elseif tc == 33 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 0, 5, 10, 6) -- inverted y - elseif tc == 34 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 0, 1, 2, 3, true, 5) -- incorrect # args - elseif tc == 35 then - local cb = circular_buffer.new(10, 1, 1) - cb:mannwhitneyu(1, 0, 5, 6, 10, "a") -- invalid use_continuity flag - elseif tc == 36 then - local cb = circular_buffer.new(10, 1, 1) - cb:get_header() -- incorrect # args - elseif tc == 37 then - local cb = circular_buffer.new(10, 1, 1) - cb:get_header(99) -- out of range column end -return 0 + return 0 end diff --git a/src/test/lua/hyperloglog_errors.lua b/src/test/lua/hyperloglog_errors.lua deleted file mode 100644 index d7f9db0..0000000 --- a/src/test/lua/hyperloglog_errors.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "hyperloglog" - -function process(tc) - if tc == 0 then - local hll = hyperloglog.new(2) -- new() incorrect # args - elseif tc == 1 then - local hll = hyperloglog.new() - hll:add({}) --incorrect argument type - elseif tc == 2 then - local hll = hyperloglog.new() - hll:add() --incorrect # args - elseif tc == 3 then - local hll = hyperloglog.new() - hll:count(1) --incorrect # args - elseif tc == 4 then - local hll = hyperloglog.new() - hll:clear(1) --incorrect # args - elseif tc == 5 then - local hll = hyperloglog.new() - hll:fromstring({}) --incorrect argument type - elseif tc == 6 then - local hll = hyperloglog.new() - hll:fromstring(" ") --incorrect argument length - end -return 0 -end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 1bc3944..2ccf9a2 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -34,6 +34,8 @@ function process(tc) t[#t+1] = "this is a test" end cjson.encode(t) + elseif tc == 8 then -- invalid type + write_message("string") end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 6446777..43e5938 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -6,16 +6,17 @@ /** @brief Lua sandbox unit tests @file */ -#include "lsb.h" - #include -#include #include +#include #include #include #include #include +#include "lsb.h" +#include "lsb_output.h" + #ifdef _WIN32 #define snprintf _snprintf #endif @@ -200,10 +201,6 @@ int write_message(lua_State* lua) } lua_sandbox* lsb = (lua_sandbox*)luserdata; - if (lua_gettop(lua) != 1 || lua_type(lua, 1) != LUA_TTABLE) { - luaL_error(lua, "write_message() takes a table argument"); - } - if (lsb_output_protobuf(lsb, 1, 0) != 0) { luaL_error(lua, "write_message() could not encode protobuf - %s", lsb_get_error(lsb)); @@ -269,7 +266,7 @@ static char* test_init_error() static char* test_destroy_error() { - const char* expected = "preserve_global_data could not open: " + const char* expected = "lsb_preserve_global_data could not open: " "invaliddir/simple.preserve"; e = lsb_destroy(NULL, NULL); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -491,6 +488,7 @@ static char* test_output_errors() , "process() lua/output_errors.lua:27: output_limit exceeded" , "process() lua/output_errors.lua:30: write_message() could not encode protobuf - unsupported array type: table" , "process() lua/output_errors.lua:36: strbuf max_size exceeded" + , "process() lua/output_errors.lua:38: write_message() could not encode protobuf - takes a single table argument" , NULL }; @@ -524,44 +522,7 @@ static char* test_cbuf_errors() { const char* tests[] = { - "process() lua/circular_buffer_errors.lua:9: bad argument #0 to 'new' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:11: bad argument #1 to 'new' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:13: bad argument #1 to 'new' (rows must be > 1)" - , "process() lua/circular_buffer_errors.lua:15: bad argument #2 to 'new' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:17: bad argument #2 to 'new' (columns must be > 0)" - , "process() lua/circular_buffer_errors.lua:19: bad argument #3 to 'new' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:21: bad argument #3 to 'new' (seconds_per_row is out of range)" - , "process() lua/circular_buffer_errors.lua:23: removed test" - , "process() not enough memory" - , "process() lua/circular_buffer_errors.lua:28: bad argument #2 to 'set' (column out of range)" - , "process() lua/circular_buffer_errors.lua:31: bad argument #2 to 'set' (column out of range)" - , "process() lua/circular_buffer_errors.lua:34: bad argument #2 to 'set' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:37: bad argument #1 to 'set' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:41: bad argument #1 to 'get' (lsb.circular_buffer expected, got number)" - , "process() lua/circular_buffer_errors.lua:44: bad argument #3 to 'set' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:47: bad argument #-1 to 'set' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:50: bad argument #-1 to 'add' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:53: bad argument #-1 to 'get' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:56: bad argument #-1 to 'compute' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:59: bad argument #1 to 'compute' (invalid option 'func')" - , "process() lua/circular_buffer_errors.lua:62: bad argument #2 to 'compute' (column out of range)" - , "process() lua/circular_buffer_errors.lua:65: bad argument #4 to 'compute' (end must be >= start)" - , "process() lua/circular_buffer_errors.lua:68: bad argument #1 to 'format' (invalid option 'invalid')" - , "process() lua/circular_buffer_errors.lua:71: bad argument #-1 to 'format' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:74: bad argument #-1 to 'format' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:77: fromstring() too few values: 0, expected 2" - , "process() lua/circular_buffer_errors.lua:80: fromstring() too few values: 0, expected 2" - , "process() lua/circular_buffer_errors.lua:83: fromstring() too many values, more than: 2" - , "process() lua/circular_buffer_errors.lua:86: bad argument #-1 to 'mannwhitneyu' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:89: bad argument #1 to 'mannwhitneyu' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:92: bad argument #1 to 'mannwhitneyu' (column out of range)" - , "process() lua/circular_buffer_errors.lua:95: bad argument #3 to 'mannwhitneyu' (ranges must not overlap)" - , "process() lua/circular_buffer_errors.lua:98: bad argument #3 to 'mannwhitneyu' (end_1 must be >= start_1)" - , "process() lua/circular_buffer_errors.lua:101: bad argument #5 to 'mannwhitneyu' (end_2 must be >= start_2)" - , "process() lua/circular_buffer_errors.lua:104: bad argument #-1 to 'mannwhitneyu' (too many arguments)" - , "process() lua/circular_buffer_errors.lua:107: bad argument #6 to 'mannwhitneyu' (use_continuity must be a boolean)" - , "process() lua/circular_buffer_errors.lua:110: bad argument #-1 to 'get_header' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:113: bad argument #1 to 'get_header' (column out of range)" + "process() not enough memory" , NULL }; @@ -638,12 +599,6 @@ static char* test_cbuf() mu_assert(strcmp(outputs[3], written_data) == 0, "received: %s", written_data); - for (int i = 1; i < 18; ++i) { - result = report(sb, i); - mu_assert(result == 0, "report() test: %d received: %d error: %s", i, - result, lsb_get_error(sb)); - } - e = lsb_destroy(sb, "circular_buffer.preserve"); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -972,7 +927,7 @@ static char* test_serialize_failure() static char* test_serialize_noglobal() { const char* output_file = "serialize_noglobal.preserve"; - const char* expected = "preserve_global_data cannot access the global table"; + const char* expected = "lsb_preserve_global_data cannot access the global table"; lua_sandbox* sb = lsb_create(NULL, "lua/serialize_noglobal.lua", "../../modules", @@ -993,50 +948,6 @@ static char* test_serialize_noglobal() } -static char* test_bloom_filter_errors() -{ - const char* tests[] = - { - "process() lua/bloom_filter_errors.lua:9: bad argument #0 to 'new' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:11: bad argument #1 to 'new' (number expected, got nil)" - , "process() lua/bloom_filter_errors.lua:13: bad argument #1 to 'new' (items must be > 1)" - , "process() lua/bloom_filter_errors.lua:15: bad argument #2 to 'new' (number expected, got nil)" - , "process() lua/bloom_filter_errors.lua:17: bad argument #2 to 'new' (probability must be between 0 and 1)" - , "process() lua/bloom_filter_errors.lua:19: bad argument #2 to 'new' (probability must be between 0 and 1)" - , "process() lua/bloom_filter_errors.lua:22: bad argument #-1 to 'add' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:25: bad argument #1 to 'add' (must be a string or number)" - , "process() lua/bloom_filter_errors.lua:28: bad argument #-1 to 'query' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:31: bad argument #1 to 'query' (must be a string or number)" - , "process() lua/bloom_filter_errors.lua:34: bad argument #-1 to 'clear' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:37: bad argument #1 to 'fromstring' (string expected, got table)" - , "process() lua/bloom_filter_errors.lua:40: fromstring() bytes found: 23, expected 24" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/bloom_filter_errors.lua", - "../../modules", 32767, 1000, 128); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - static char* test_bloom_filter() { const char* output_file = "bloom_filter.preserve"; @@ -1111,45 +1022,6 @@ static char* test_bloom_filter() } -static char* test_hyperloglog_errors() -{ - const char* tests[] = - { - "process() lua/hyperloglog_errors.lua:9: bad argument #0 to 'new' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:12: bad argument #1 to 'add' (must be a string or number)" - , "process() lua/hyperloglog_errors.lua:15: bad argument #-1 to 'add' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:18: bad argument #-1 to 'count' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:21: bad argument #-1 to 'clear' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:24: bad argument #1 to 'fromstring' (string expected, got table)" - , "process() lua/hyperloglog_errors.lua:27: fromstring() bytes found: 23, expected 12304" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, - "lua/hyperloglog_errors.lua", "../../modules", - 32767, 1000, 128); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - static char* test_hyperloglog() { const char* output_file = "hyperloglog.preserve"; @@ -1530,9 +1402,7 @@ static char* all_tests() mu_run_test(test_restore); mu_run_test(test_serialize_failure); mu_run_test(test_serialize_noglobal); - mu_run_test(test_bloom_filter_errors); mu_run_test(test_bloom_filter); - mu_run_test(test_hyperloglog_errors); mu_run_test(test_hyperloglog); mu_run_test(test_struct); diff --git a/src/xxhash.c b/src/xxhash.c deleted file mode 100644 index e39ac2e..0000000 --- a/src/xxhash.c +++ /dev/null @@ -1,476 +0,0 @@ -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -*/ - - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -//#define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -# define FORCE_INLINE static __forceinline -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -//************************************** -// Includes & Memory related functions -//************************************** -#include "xxhash.h" -// Modify the local functions below should you wish to use some other memory related routines -// for malloc(), free() -#include -FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } -FORCE_INLINE void XXH_free (void* p) { free(p); } -// for memcpy() -#include -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S { U32 v; } _PACKED U32_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((U32_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -# define XXH_swap32 _byteswap_ulong -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static inline U32 XXH_swap32 (U32 x) { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff );} -#endif - - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch - static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } - - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U32 h32; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; } -#endif - - if (len>=16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32(const void* input, int len, U32 seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - void* state = XXH32_init(seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -//**************************** -// Advanced Hash Functions -//**************************** - -struct XXH_state32_t -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - int memsize; - char memory[16]; -}; - - -int XXH32_sizeofState() -{ - XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough - return sizeof(struct XXH_state32_t); -} - - -XXH_errorcode XXH32_resetState(void* state_in, U32 seed) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - - -void* XXH32_init (U32 seed) -{ - void* state = XXH_malloc (sizeof(struct XXH_state32_t)); - XXH32_resetState(state, seed); - return state; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); - { - const U32* p32 = (const U32*)state->memory; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->memory, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH32_update (void* state_in, const void* input, int len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE * p = (const BYTE*)state->memory; - BYTE* bEnd = (BYTE*)state->memory + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32_intermediateDigest (void* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); - else - return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); -} - - -U32 XXH32_digest (void* state_in) -{ - U32 h32 = XXH32_intermediateDigest(state_in); - - XXH_free(state_in); - - return h32; -} - diff --git a/src/xxhash.h b/src/xxhash.h deleted file mode 100644 index fa2d0f4..0000000 --- a/src/xxhash.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - xxHash - Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -//**************************** -// Type -//**************************** -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - - -//**************************** -// Simple Hash Functions -//**************************** - -unsigned int XXH32 (const void* input, int len, unsigned int seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". - The memory between input & input+len must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s - Note that "len" is type "int", which means it is limited to 2^31-1. - If your data is larger, use the advanced functions below. -*/ - - - -//**************************** -// Advanced Hash Functions -//**************************** - -void* XXH32_init (unsigned int seed); -XXH_errorcode XXH32_update (void* state, const void* input, int len); -unsigned int XXH32_digest (void* state); - -/* -These functions calculate the xxhash of an input provided in several small packets, -as opposed to an input provided as a single block. - -It must be started with : -void* XXH32_init() -The function returns a pointer which holds the state of calculation. - -This pointer must be provided as "void* state" parameter for XXH32_update(). -XXH32_update() can be called as many times as necessary. -The user must provide a valid (allocated) input. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Note that "len" is type "int", which means it is limited to 2^31-1. -If your data is larger, it is recommended to chunk your data into blocks -of size for example 2^30 (1GB) to avoid any "int" overflow issue. - -Finally, you can end the calculation anytime, by using XXH32_digest(). -This function returns the final 32-bits hash. -You must provide the same "void* state" parameter created by XXH32_init(). -Memory will be freed by XXH32_digest(). -*/ - - -int XXH32_sizeofState(void); -XXH_errorcode XXH32_resetState(void* state, unsigned int seed); - -#define XXH32_SIZEOFSTATE 48 -typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; -/* -These functions allow user application to make its own allocation for state. - -XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state. -Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. -This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state. - -For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), -use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. -*/ - - -unsigned int XXH32_intermediateDigest (void* state); -/* -This function does the same as XXH32_digest(), generating a 32-bit hash, -but preserve memory context. -This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update(). -To free memory context, use XXH32_digest(), or free(). -*/ - - - -//**************************** -// Deprecated function names -//**************************** -// The following translations are provided to ease code transition -// You are encouraged to no longer this function names -#define XXH32_feed XXH32_update -#define XXH32_result XXH32_digest -#define XXH32_getIntermediateResult XXH32_intermediateDigest - - - -#if defined (__cplusplus) -} -#endif - From b3dc21094d6757a2c231f179419743ea7e0c2cd0 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 15 Jan 2015 12:24:34 -0800 Subject: [PATCH 032/235] Pull in the new modules that don't strip the lib prefix --- cmake/externals.cmake | 6 +++--- src/CMakeLists.txt | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 1d36246..f032f9e 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -88,7 +88,7 @@ add_dependencies(lua_cjson ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG 9d195247363052bed9d6de7457dfee25a86abf06 + GIT_TAG d305c436fa0bf225c7029b638dd202fad68c1cc9 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -98,7 +98,7 @@ externalproject_add( lua_circular_buffer URL /work/git/lua_circular_buffer1.tgz GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG 8e8b224a9e64e9d0f95c0c7acdf55fdce34cfa11 + GIT_TAG 22cea7cea0d5a8f81f79998751b8f36c351d4f10 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -108,7 +108,7 @@ externalproject_add( lua_hyperloglog URL /work/git/lua_hyperloglog1.tgz GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 50780c5ce2c83be0510cf27ebfa742c4e74683aa + GIT_TAG a601f320bc814ecc5a64344d911498ec8acc6d56 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6fa60d4..d5a1855 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -30,9 +30,9 @@ elseif(MINGW) "${EP_BASE}/lib/liblua.dll" "${EP_BASE}/lib/liblpeg.dll" "${EP_BASE}/lib/libcjson.dll" - "${EP_BASE}/lib/bloom_filter.dll" - "${EP_BASE}/lib/circular_buffer.dll" - "${EP_BASE}/lib/hyperloglog.dll" + "${EP_BASE}/lib/libbloom_filter.dll" + "${EP_BASE}/lib/libcircular_buffer.dll" + "${EP_BASE}/lib/libhyperloglog.dll" ) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) @@ -52,9 +52,9 @@ else() "${EP_BASE}/lib/liblua.a" "${EP_BASE}/lib/liblpeg.a" "${EP_BASE}/lib/libcjson.a" - "${EP_BASE}/lib/bloom_filter.a" - "${EP_BASE}/lib/circular_buffer.a" - "${EP_BASE}/lib/hyperloglog.a" + "${EP_BASE}/lib/libbloom_filter.a" + "${EP_BASE}/lib/libcircular_buffer.a" + "${EP_BASE}/lib/libhyperloglog.a" ${LINK_DL} -lm ) add_library(luasandbox STATIC ${LUA_SANDBOX_SRC}) From 75454fa504242ab398987f6f0ca82d852a0b2ce7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 21 Jan 2015 15:12:34 -0800 Subject: [PATCH 033/235] Update the module loader to support shared libraries * the LuaJIT version still needs to be done --- .gitignore | 1 + README.md | 16 +- cmake/externals.cmake | 10 +- cmake/lua-5_1_5.patch | 411 ++++++++++++++++++++++++++-- docs/sandbox_api.md | 22 +- src/CMakeLists.txt | 38 +-- src/lsb.c | 100 +++++-- src/lsb_modules.c | 156 ----------- src/lsb_modules.h | 50 ---- src/lsb_private.h | 1 - src/lsb_serialize.c | 24 +- src/test/CMakeLists.txt | 12 + src/test/lua/errors.lua | 6 - src/test/lua/serialize_noglobal.lua | 5 - src/test/test_lua_sandbox.c | 116 +++----- 15 files changed, 572 insertions(+), 396 deletions(-) delete mode 100644 src/lsb_modules.c delete mode 100644 src/lsb_modules.h delete mode 100644 src/test/lua/serialize_noglobal.lua diff --git a/.gitignore b/.gitignore index 8a24132..cfe34ee 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.preserve release/ +src/test/modules/ diff --git a/README.md b/README.md index 92047a7..7aa11b4 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ lua_sandbox - UNIX Build Instructions lua_sandbox - Windows Build Instructions ---- - # in a VS2013 command prompt window +# in a VS2013 command prompt window git clone https://github.com/mozilla-services/lua_sandbox.git cd lua_sandbox @@ -31,10 +31,14 @@ lua_sandbox - Windows Build Instructions cd release cmake -DCMAKE_BUILD_TYPE=release -G "NMake Makefiles" .. nmake + ctest - # To run the tests you must install - cmake -DCMAKE_INSTALL_PREFIX="" .. - nmake install DESTDIR=test - cd ..\src\test - ..\..\release\test\lib\test_lua_sandbox.exe +# in a MinGW command prompt window + git clone https://github.com/mozilla-services/lua_sandbox.git + cd lua_sandbox + mkdir release + cd release + cmake -DCMAKE_BUILD_TYPE=release -G "MinGW Makefiles" .. + mingw32-make + ctest diff --git a/cmake/externals.cmake b/cmake/externals.cmake index f032f9e..36910db 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -79,7 +79,7 @@ add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG be85986dc481ad2a0d3abc06ac57e1d1241d9d4c + GIT_TAG 9c3a1805796747d59e5b4f0a789796f368285251 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -88,7 +88,7 @@ add_dependencies(lua_cjson ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG d305c436fa0bf225c7029b638dd202fad68c1cc9 + GIT_TAG b264c84cca536f20a41a9950556383e8dc5fd2ae CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -96,9 +96,8 @@ add_dependencies(lua_bloom_filter ${LUA_PROJECT}) externalproject_add( lua_circular_buffer - URL /work/git/lua_circular_buffer1.tgz GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG 22cea7cea0d5a8f81f79998751b8f36c351d4f10 + GIT_TAG 5e7726d70bd5cf6e61e1faccae2a2fabf5e9bf3d CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -106,9 +105,8 @@ add_dependencies(lua_circular_buffer ${LUA_PROJECT}) externalproject_add( lua_hyperloglog - URL /work/git/lua_hyperloglog1.tgz GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG a601f320bc814ecc5a64344d911498ec8acc6d56 + GIT_TAG e5d3399b0d0a6913d18cc5a0a33326391363b936 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch index 56ee374..6321516 100644 --- a/cmake/lua-5_1_5.patch +++ b/cmake/lua-5_1_5.patch @@ -1,6 +1,6 @@ diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt --- /work/lua-5.1.5/CMakeLists.txt 1969-12-31 16:00:00.000000000 -0800 -+++ lua-5_1_5/CMakeLists.txt 2014-11-27 10:36:18.352979806 -0800 ++++ lua-5_1_5/CMakeLists.txt 2015-01-16 10:26:29.623543211 -0800 @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this @@ -68,21 +68,396 @@ diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt +install(FILES src/lua.h src/luaconf.h src/lualib.h src/lauxlib.h DESTINATION include) diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c --- /work/lua-5.1.5/src/ldebug.c 2008-05-08 09:56:26.000000000 -0700 -+++ lua-5_1_5/src/ldebug.c 2014-11-27 10:32:27.052988495 -0800 ++++ lua-5_1_5/src/ldebug.c 2015-01-16 10:26:29.623543211 -0800 @@ -80,6 +80,10 @@ return L->basehookcount; } - + +LUA_API int lua_gethookcountremaining (lua_State *L) { + return L->hookcount; +} + - + LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar) { int status; +diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c +--- /work/lua-5.1.5/src/loadlib.c 2009-09-09 06:17:16.000000000 -0700 ++++ lua-5_1_5/src/loadlib.c 2015-01-21 06:37:34.841228660 -0800 +@@ -23,23 +23,22 @@ + + + /* prefix for open functions in C libraries */ +-#define LUA_POF "luaopen_" ++#define LUA_POF "luaopen_" + + /* separator for open functions in C libraries */ +-#define LUA_OFSEP "_" ++#define LUA_OFSEP "_" + + +-#define LIBPREFIX "LOADLIB: " ++#define LIBPREFIX "LOADLIB: " + +-#define POF LUA_POF +-#define LIB_FAIL "open" ++#define POF LUA_POF ++#define LIB_FAIL "open" + + + /* error codes for ll_loadfunc */ +-#define ERRLIB 1 +-#define ERRFUNC 2 ++#define ERRLIB 1 ++#define ERRFUNC 2 + +-#define setprogdir(L) ((void)0) + + + static void ll_unloadlib (void *lib); +@@ -91,24 +90,6 @@ + + #include + +- +-#undef setprogdir +- +-static void setprogdir (lua_State *L) { +- char buff[MAX_PATH + 1]; +- char *lb; +- DWORD nsize = sizeof(buff)/sizeof(char); +- DWORD n = GetModuleFileNameA(NULL, buff, nsize); +- if (n == 0 || n == nsize || (lb = strrchr(buff, '\\')) == NULL) +- luaL_error(L, "unable to get ModuleFileName"); +- else { +- *lb = '\0'; +- luaL_gsub(L, lua_tostring(L, -1), LUA_EXECDIR, buff); +- lua_remove(L, -2); /* remove original string */ +- } +-} +- +- + static void pusherror (lua_State *L) { + int error = GetLastError(); + char buffer[128]; +@@ -153,7 +134,7 @@ + + /* Mac appends a `_' before C function names */ + #undef POF +-#define POF "_" LUA_POF ++#define POF "_" LUA_POF + + + static void pusherror (lua_State *L) { +@@ -230,10 +211,10 @@ + */ + + #undef LIB_FAIL +-#define LIB_FAIL "absent" ++#define LIB_FAIL "absent" + + +-#define DLMSG "dynamic libraries not enabled; check your Lua installation" ++#define DLMSG "dynamic libraries not enabled; check your Lua installation" + + + static void ll_unloadlib (void *lib) { +@@ -351,11 +332,20 @@ + static const char *findfile (lua_State *L, const char *name, + const char *pname) { + const char *path; +- name = luaL_gsub(L, name, ".", LUA_DIRSEP); +- lua_getfield(L, LUA_ENVIRONINDEX, pname); ++ int i = 0; ++ while (name[i]) { ++ if (!isalnum(name[i]) && name[i] != '_') { ++ luaL_error(L, "invalid module name '%s'", name); ++ return NULL; // never reached, just silences the compiler ++ } ++ ++i; ++ } ++ lua_pushstring(L, name); ++ lua_getfield(L, LUA_REGISTRYINDEX, pname); + path = lua_tostring(L, -1); +- if (path == NULL) +- luaL_error(L, LUA_QL("package.%s") " must be a string", pname); ++ if (path == NULL) { ++ return NULL; ++ } + lua_pushliteral(L, ""); /* error accumulator */ + while ((path = pushnexttemplate(L, path)) != NULL) { + const char *filename; +@@ -380,7 +370,7 @@ + static int loader_Lua (lua_State *L) { + const char *filename; + const char *name = luaL_checkstring(L, 1); +- filename = findfile(L, name, "path"); ++ filename = findfile(L, name, "lsb_require_path"); + if (filename == NULL) return 1; /* library not found in this path */ + if (luaL_loadfile(L, filename) != 0) + loaderror(L, filename); +@@ -402,7 +392,7 @@ + static int loader_C (lua_State *L) { + const char *funcname; + const char *name = luaL_checkstring(L, 1); +- const char *filename = findfile(L, name, "cpath"); ++ const char *filename = findfile(L, name, "lsb_require_cpath"); + if (filename == NULL) return 1; /* library not found in this path */ + funcname = mkfuncname(L, name); + if (ll_loadfunc(L, filename, funcname) != 0) +@@ -411,41 +401,29 @@ + } + + +-static int loader_Croot (lua_State *L) { +- const char *funcname; +- const char *filename; +- const char *name = luaL_checkstring(L, 1); +- const char *p = strchr(name, '.'); +- int stat; +- if (p == NULL) return 0; /* is root */ +- lua_pushlstring(L, name, p - name); +- filename = findfile(L, lua_tostring(L, -1), "cpath"); +- if (filename == NULL) return 1; /* root not found */ +- funcname = mkfuncname(L, name); +- if ((stat = ll_loadfunc(L, filename, funcname)) != 0) { +- if (stat != ERRFUNC) loaderror(L, filename); /* real error */ +- lua_pushfstring(L, "\n\tno module " LUA_QS " in file " LUA_QS, +- name, filename); +- return 1; /* function not found */ +- } +- return 1; +-} +- +- + static int loader_preload (lua_State *L) { + const char *name = luaL_checkstring(L, 1); + lua_getfield(L, LUA_ENVIRONINDEX, "preload"); + if (!lua_istable(L, -1)) +- luaL_error(L, LUA_QL("package.preload") " must be a table"); ++ luaL_error(L, LUA_QL("preload") " must be a table"); + lua_getfield(L, -1, name); +- if (lua_isnil(L, -1)) /* not found? */ +- lua_pushfstring(L, "\n\tno field package.preload['%s']", name); + return 1; + } + ++// If necessary, add an empty metatable to flag the table as non-data ++// during preservation. ++static void add_empty_metatable(lua_State* lua) ++{ ++ if (lua_getmetatable(lua, -1) == 0) { ++ lua_newtable(lua); ++ lua_setmetatable(lua, -2); ++ } else { ++ lua_pop(lua, 1); ++ } ++} + + static const int sentinel_ = 0; +-#define sentinel ((void *)&sentinel_) ++#define sentinel ((void *)&sentinel_) + + + static int ll_require (lua_State *L) { +@@ -462,7 +440,7 @@ + /* else must load it; iterate over available loaders */ + lua_getfield(L, LUA_ENVIRONINDEX, "loaders"); + if (!lua_istable(L, -1)) +- luaL_error(L, LUA_QL("package.loaders") " must be a table"); ++ luaL_error(L, LUA_QL("loaders") " must be a table"); + lua_pushliteral(L, ""); /* error message accumulator */ + for (i=1; ; i++) { + lua_rawgeti(L, -2, i); /* get a loader */ +@@ -490,138 +468,27 @@ + lua_pushvalue(L, -1); /* extra copy to be returned */ + lua_setfield(L, 2, name); /* _LOADED[name] = true */ + } ++ add_empty_metatable(L); ++ /* todo apply restrictions */ + return 1; + } + + /* }====================================================== */ + + +- +-/* +-** {====================================================== +-** 'module' function +-** ======================================================= +-*/ +- +- +-static void setfenv (lua_State *L) { +- lua_Debug ar; +- if (lua_getstack(L, 1, &ar) == 0 || +- lua_getinfo(L, "f", &ar) == 0 || /* get calling function */ +- lua_iscfunction(L, -1)) +- luaL_error(L, LUA_QL("module") " not called from a Lua function"); +- lua_pushvalue(L, -2); +- lua_setfenv(L, -2); +- lua_pop(L, 1); +-} +- +- +-static void dooptions (lua_State *L, int n) { +- int i; +- for (i = 2; i <= n; i++) { +- lua_pushvalue(L, i); /* get option (a function) */ +- lua_pushvalue(L, -2); /* module */ +- lua_call(L, 1, 0); +- } +-} +- +- +-static void modinit (lua_State *L, const char *modname) { +- const char *dot; +- lua_pushvalue(L, -1); +- lua_setfield(L, -2, "_M"); /* module._M = module */ +- lua_pushstring(L, modname); +- lua_setfield(L, -2, "_NAME"); +- dot = strrchr(modname, '.'); /* look for last dot in module name */ +- if (dot == NULL) dot = modname; +- else dot++; +- /* set _PACKAGE as package name (full module name minus last part) */ +- lua_pushlstring(L, modname, dot - modname); +- lua_setfield(L, -2, "_PACKAGE"); +-} +- +- +-static int ll_module (lua_State *L) { +- const char *modname = luaL_checkstring(L, 1); +- int loaded = lua_gettop(L) + 1; /* index of _LOADED table */ +- lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); +- lua_getfield(L, loaded, modname); /* get _LOADED[modname] */ +- if (!lua_istable(L, -1)) { /* not found? */ +- lua_pop(L, 1); /* remove previous result */ +- /* try global variable (and create one if it does not exist) */ +- if (luaL_findtable(L, LUA_GLOBALSINDEX, modname, 1) != NULL) +- return luaL_error(L, "name conflict for module " LUA_QS, modname); +- lua_pushvalue(L, -1); +- lua_setfield(L, loaded, modname); /* _LOADED[modname] = new table */ +- } +- /* check whether table already has a _NAME field */ +- lua_getfield(L, -1, "_NAME"); +- if (!lua_isnil(L, -1)) /* is table an initialized module? */ +- lua_pop(L, 1); +- else { /* no; initialize it */ +- lua_pop(L, 1); +- modinit(L, modname); +- } +- lua_pushvalue(L, -1); +- setfenv(L); +- dooptions(L, loaded - 1); +- return 0; +-} +- +- +-static int ll_seeall (lua_State *L) { +- luaL_checktype(L, 1, LUA_TTABLE); +- if (!lua_getmetatable(L, 1)) { +- lua_createtable(L, 0, 1); /* create new metatable */ +- lua_pushvalue(L, -1); +- lua_setmetatable(L, 1); +- } +- lua_pushvalue(L, LUA_GLOBALSINDEX); +- lua_setfield(L, -2, "__index"); /* mt.__index = _G */ +- return 0; +-} +- +- +-/* }====================================================== */ +- +- +- +-/* auxiliary mark (for internal use) */ +-#define AUXMARK "\1" +- +-static void setpath (lua_State *L, const char *fieldname, const char *envname, +- const char *def) { +- const char *path = getenv(envname); +- if (path == NULL) /* no environment variable? */ +- lua_pushstring(L, def); /* use default */ +- else { +- /* replace ";;" by ";AUXMARK;" and then AUXMARK by default path */ +- path = luaL_gsub(L, path, LUA_PATHSEP LUA_PATHSEP, +- LUA_PATHSEP AUXMARK LUA_PATHSEP); +- luaL_gsub(L, path, AUXMARK, def); +- lua_remove(L, -2); +- } +- setprogdir(L); +- lua_setfield(L, -2, fieldname); +-} +- +- + static const luaL_Reg pk_funcs[] = { +- {"loadlib", ll_loadlib}, +- {"seeall", ll_seeall}, + {NULL, NULL} + }; + + + static const luaL_Reg ll_funcs[] = { +- {"module", ll_module}, + {"require", ll_require}, + {NULL, NULL} + }; + + + static const lua_CFunction loaders[] = +- {loader_preload, loader_Lua, loader_C, loader_Croot, NULL}; ++ {loader_preload, loader_Lua, loader_C, NULL}; + + + LUALIB_API int luaopen_package (lua_State *L) { +@@ -630,12 +497,8 @@ + luaL_newmetatable(L, "_LOADLIB"); + lua_pushcfunction(L, gctm); + lua_setfield(L, -2, "__gc"); +- /* create `package' table */ +- luaL_register(L, LUA_LOADLIBNAME, pk_funcs); +-#if defined(LUA_COMPAT_LOADLIB) +- lua_getfield(L, -1, "loadlib"); +- lua_setfield(L, LUA_GLOBALSINDEX, "loadlib"); +-#endif ++ /* create `package' environment table (not exposed to Lua) */ ++ lua_newtable(L); + lua_pushvalue(L, -1); + lua_replace(L, LUA_ENVIRONINDEX); + /* create `loaders' table */ +@@ -646,21 +509,17 @@ + lua_rawseti(L, -2, i+1); + } + lua_setfield(L, -2, "loaders"); /* put it in field `loaders' */ +- setpath(L, "path", LUA_PATH, LUA_PATH_DEFAULT); /* set field `path' */ +- setpath(L, "cpath", LUA_CPATH, LUA_CPATH_DEFAULT); /* set field `cpath' */ +- /* store config information */ +- lua_pushliteral(L, LUA_DIRSEP "\n" LUA_PATHSEP "\n" LUA_PATH_MARK "\n" +- LUA_EXECDIR "\n" LUA_IGMARK); +- lua_setfield(L, -2, "config"); +- /* set field `loaded' */ +- luaL_findtable(L, LUA_REGISTRYINDEX, "_LOADED", 2); +- lua_setfield(L, -2, "loaded"); ++ + /* set field `preload' */ +- lua_newtable(L); ++ luaL_findtable(L, LUA_REGISTRYINDEX, "_PRELOADED", 0); + lua_setfield(L, -2, "preload"); ++ ++ /* open lib into global table */ + lua_pushvalue(L, LUA_GLOBALSINDEX); +- luaL_register(L, NULL, ll_funcs); /* open lib into global table */ ++ luaL_register(L, NULL, ll_funcs); + lua_pop(L, 1); ++ ++ /* create an empty `package' table */ ++ luaL_register(L, LUA_LOADLIBNAME, pk_funcs); + return 1; /* return 'package' table */ + } +- diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c --- /work/lua-5.1.5/src/loslib.c 2008-01-18 08:38:18.000000000 -0800 -+++ lua-5_1_5/src/loslib.c 2014-11-27 10:32:27.052988495 -0800 ++++ lua-5_1_5/src/loslib.c 2015-01-16 10:26:29.623543211 -0800 @@ -33,6 +33,61 @@ return 3; } @@ -142,13 +517,13 @@ diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c + +#endif + - - + + static int os_execute (lua_State *L) { @@ -121,16 +176,40 @@ } - - + + +static const char *checkoption (lua_State *L, const char *conv, char *buff) { + static const char *const options[] = LUA_STRFTIMEOPTIONS; + unsigned int i; @@ -215,7 +590,7 @@ diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c } diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h --- /work/lua-5.1.5/src/luaconf.h 2008-02-11 08:25:08.000000000 -0800 -+++ lua-5_1_5/src/luaconf.h 2014-11-27 10:32:27.052988495 -0800 ++++ lua-5_1_5/src/luaconf.h 2015-01-16 10:26:29.623543211 -0800 @@ -1,5 +1,5 @@ /* -** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ @@ -226,7 +601,7 @@ diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h @@ -33,14 +33,26 @@ #define LUA_WIN #endif - + +#if defined(LUA_WIN) +#define LUA_DL_DLL +#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ @@ -240,7 +615,7 @@ diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h +#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ +#define LUA_USE_LONGLONG /* assume support for long long */ #endif - + #if defined(LUA_USE_MACOSX) #define LUA_USE_POSIX +#define LUA_USE_DLOPEN /* does not need -ldl */ @@ -249,23 +624,23 @@ diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h +#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ #define LUA_DL_DYLD /* does not need extra library */ #endif - + @@ -56,6 +68,7 @@ #define LUA_USE_ISATTY #define LUA_USE_POPEN #define LUA_USE_ULONGJMP +#define LUA_USE_GMTIME_R #endif - - + + diff -Naur /work/lua-5.1.5/src/lua.h lua-5_1_5/src/lua.h --- /work/lua-5.1.5/src/lua.h 2012-01-13 12:36:20.000000000 -0800 -+++ lua-5_1_5/src/lua.h 2014-11-27 10:32:27.052988495 -0800 ++++ lua-5_1_5/src/lua.h 2015-01-16 10:26:29.623543211 -0800 @@ -341,6 +341,7 @@ LUA_API lua_Hook lua_gethook (lua_State *L); LUA_API int lua_gethookmask (lua_State *L); LUA_API int lua_gethookcount (lua_State *L); +LUA_API int lua_gethookcountremaining (lua_State *L); - - + + struct lua_Debug { diff --git a/docs/sandbox_api.md b/docs/sandbox_api.md index 74ab361..85fa4f7 100644 --- a/docs/sandbox_api.md +++ b/docs/sandbox_api.md @@ -27,12 +27,12 @@ By default only the base library is loaded additional libraries must be explicit *Arguments* - libraryName (string) - - **base library** + - [base library(http://www.lua.org/manual/5.1/manual.html#5.1) - The standard require() function is overridden by this version. - Disabled functions: collectgarbage, coroutine, dofile, load, loadfile, loadstring, module, print. - - **bloom_filter** https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md - - **circular_buffer** https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md - - **cjson** http://www.kyne.com.au/~mark/software/lua-cjson-manual.html. With the following modifications: + - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) + - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) + - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) With the following modifications: - Loads the cjson module in a global cjson table - The encode buffer is limited to the sandbox output_limit. - The decode buffer will be roughly limited to one half of the sandbox memory_limit. @@ -40,15 +40,15 @@ By default only the base library is loaded additional libraries must be explicit If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - The new() function has been disabled so only a single cjson parser can be created. - The encode_keep_buffer() function has been disabled (the buffer is always reused). - - **hyperloglog** https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md - - **lpeg** loads the Lua Parsing Expression Grammar Library http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html - - **math** - - **os** + - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) + - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library + - [math](http://www.lua.org/manual/5.1/manual.html#5.6) + - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. - Disabled functions: execute, exit, remove, rename, setlocale, tmpname. - - **string** - - **struct** http://www.inf.puc-rio.br/~roberto/struct/ - - **table** + - [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - [struct](http://www.inf.puc-rio.br/~roberto/struct/) + - [table](http://www.lua.org/manual/5.1/manual.html#5.5) - _user provided_ *Return* diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d5a1855..e906899 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,69 +4,57 @@ set(LUA_SANDBOX_SRC lsb.c -lsb_modules.c lsb_output.c lsb_serialize.c lsb_serialize_protobuf.c struct.c ) +add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) +add_dependencies(luasandbox ${LUA_PROJECT} lua_lpeg lua_cjson) +add_dependencies(lua_bloom_filter luasandbox) +add_dependencies(lua_circular_buffer luasandbox) +add_dependencies(lua_hyperloglog luasandbox) + if(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/lua.lib" "${EP_BASE}/lib/lpeg.lib" "${EP_BASE}/lib/cjson.lib" - "${EP_BASE}/lib/bloom_filter.lib" - "${EP_BASE}/lib/circular_buffer.lib" - "${EP_BASE}/lib/hyperloglog.lib" ) - add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.dll") - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.lib") elseif(MINGW) add_definitions(-D_MINGW) set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/liblua.dll" "${EP_BASE}/lib/liblpeg.dll" "${EP_BASE}/lib/libcjson.dll" - "${EP_BASE}/lib/libbloom_filter.dll" - "${EP_BASE}/lib/libcircular_buffer.dll" - "${EP_BASE}/lib/libhyperloglog.dll" ) - add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) if (ADDRESS_MODEL EQUAL 32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") set_target_properties(luasandbox PROPERTIES LINK_FLAGS "-s -m32") endif() - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.dll") else() - if (LUA_JIT) - if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - set(LINK_DL "-ldl") - endif() - endif() + if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + set(LINK_DL "-ldl") + endif() set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/liblua.a" "${EP_BASE}/lib/liblpeg.a" "${EP_BASE}/lib/libcjson.a" - "${EP_BASE}/lib/libbloom_filter.a" - "${EP_BASE}/lib/libcircular_buffer.a" - "${EP_BASE}/lib/libhyperloglog.a" - ${LINK_DL} -lm + ${LINK_DL} -lm ) - add_library(luasandbox STATIC ${LUA_SANDBOX_SRC}) - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.a") - install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) endif() target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) -add_dependencies(luasandbox ${LUA_PROJECT} lua_lpeg lua_cjson lua_bloom_filter lua_circular_buffer lua_hyperloglog) install(TARGETS luasandbox DESTINATION lib) +install(DIRECTORY "${EP_BASE}/modules/" DESTINATION modules) +install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib) install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) install(DIRECTORY "${LUA_INCLUDE_DIR}/" DESTINATION include) +install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) add_subdirectory(test) diff --git a/src/lsb.c b/src/lsb.c index ba2b946..8d81650 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -18,18 +18,55 @@ #include #include -#include "lsb_modules.h" #include "lsb_output.h" #include "lsb_private.h" #include "lsb_serialize.h" #include "lsb_serialize_protobuf.h" static const char* disable_base_functions[] = { "collectgarbage", "coroutine", - "dofile", "load", "loadfile", "loadstring", "module", "print", "require", - NULL }; + "dofile", "load", "loadfile", "loadstring", "module", "print", NULL }; + +static const char* require_lpath = "lsb_require_path"; +static const char* require_cpath = "lsb_require_cpath"; static jmp_buf g_jbuf; +LUALIB_API int luaopen_struct(lua_State* L); +LUALIB_API int luaopen_cjson(lua_State* L); +LUALIB_API int luaopen_lpeg(lua_State* L); + +static const luaL_Reg preload_module_list[] = { + { LUA_TABLIBNAME, luaopen_table }, + { LUA_IOLIBNAME, luaopen_io }, + { LUA_OSLIBNAME, luaopen_os }, + { LUA_STRLIBNAME, luaopen_string }, + { LUA_MATHLIBNAME, luaopen_math }, + { "struct", luaopen_struct }, + { "cjson", luaopen_cjson }, + { "lpeg", luaopen_lpeg }, + { NULL, NULL } +}; + +static int libsize (const luaL_Reg *l) { + int size = 0; + for (; l->name; l++) size++; + return size; +} + +static void preload_modules(lua_State* lua) +{ + const luaL_Reg *lib = preload_module_list; + luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", + libsize(preload_module_list)); + for (; lib->func; lib++) { + lua_pushstring(lua, lib->name); + lua_pushcfunction(lua, lib->func); + lua_rawset(lua, -3); + } + lua_pop(lua, 1); // remove the preloaded table +} + + #ifndef LUA_JIT /** * Implementation of the memory allocator for the Lua state. @@ -166,28 +203,36 @@ lua_sandbox* lsb_create(void* parent, lsb->output.data = malloc(lsb->output.size); size_t len = strlen(lua_file); lsb->lua_file = malloc(len + 1); - lsb->require_path = NULL; if (require_path) { - len = strlen(require_path); - lsb->require_path = malloc(len + 1); +#if defined(_WIN32) + lua_pushfstring(lsb->lua, "%s\\?.lua", require_path); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_lpath); + lua_pushfstring(lsb->lua, "%s\\?.dll", require_path); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_cpath); +#else + lua_pushfstring(lsb->lua, "%s/?.lua", require_path); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_lpath); + lua_pushfstring(lsb->lua, "%s/?.so", require_path); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_cpath); +#endif } - if (!lsb->output.data || !lsb->lua_file - || (require_path && !lsb->require_path)) { + + lua_pushinteger(lsb->lua, output_limit); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, "lsb_output_limit"); + + if (!lsb->output.data || !lsb->lua_file) { free(lsb); free(lsb->lua_file); - free(lsb->require_path); lua_close(lsb->lua); lsb->lua = NULL; return NULL; } strcpy(lsb->lua_file, lua_file); - if (lsb->require_path) { - strcpy(lsb->require_path, require_path); - } srand((unsigned int)time(NULL)); return lsb; } + int lsb_init(lua_sandbox* lsb, const char* data_file) { if (!lsb) { @@ -198,24 +243,24 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif - lsb_load_library(lsb->lua, "", luaopen_base, disable_base_functions); - lua_pop(lsb->lua, 1); + // load base module + lua_pushcfunction(lsb->lua, luaopen_base); + lua_pushstring(lsb->lua, ""); + lua_call(lsb->lua, 1, 0); + for (int i = 0; disable_base_functions[i]; ++i) { + lua_pushnil(lsb->lua); + lua_setfield(lsb->lua, LUA_GLOBALSINDEX, disable_base_functions[i]); + } - // Create a simple package cache - lua_createtable(lsb->lua, 0, 1); - lua_pushvalue(lsb->lua, -1); - lua_setglobal(lsb->lua, lsb_package_table); - // Add empty metatable to prevent serialization + preload_modules(lsb->lua); + + // load package module + lua_pushcfunction(lsb->lua, luaopen_package); + lua_pushstring(lsb->lua, LUA_LOADLIBNAME); + lua_call(lsb->lua, 1, 1); lua_newtable(lsb->lua); lua_setmetatable(lsb->lua, -2); - // add the loaded table - lua_newtable(lsb->lua); - lua_setfield(lsb->lua, -2, lsb_loaded_table); - lua_pop(lsb->lua, 1); // remove the package table - - lua_pushlightuserdata(lsb->lua, (void*)lsb); - lua_pushcclosure(lsb->lua, &lsb_require_library, 1); - lua_setglobal(lsb->lua, "require"); + lua_pop(lsb->lua, 1); lua_pushlightuserdata(lsb->lua, (void*)lsb); lua_pushcclosure(lsb->lua, &output, 1); @@ -277,7 +322,6 @@ char* lsb_destroy(lua_sandbox* lsb, const char* data_file) lsb_terminate(lsb, NULL); free(lsb->output.data); free(lsb->lua_file); - free(lsb->require_path); free(lsb); return err; } diff --git a/src/lsb_modules.c b/src/lsb_modules.c deleted file mode 100644 index 4cd8804..0000000 --- a/src/lsb_modules.c +++ /dev/null @@ -1,156 +0,0 @@ - -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua sandbox modules implementation @file */ - -#include "lsb_modules.h" - -#include -#include -#include -#include -#include - -#include "lsb_private.h" - -#ifndef MAX_PATH -#define MAX_PATH 260 -#endif - -#ifdef _WIN32 -#define PATH_DELIMITER '\\' -#else -#define PATH_DELIMITER '/' -#endif - -const char* lsb_disable_none[] = { NULL }; -const char* lsb_package_table = "package"; -const char* lsb_loaded_table = "loaded"; - -// If necessary, add an empty metatable to flag the table as non-data -// during preservation. -static void add_empty_metatable(lua_State* lua) -{ - if (lua_getmetatable(lua, -1) == 0) { - lua_newtable(lua); - lua_setmetatable(lua, -2); - } else { - lua_pop(lua, 1); - } -} - - -void lsb_load_library(lua_State* lua, const char* table, lua_CFunction f, - const char** disable) -{ - lua_pushcfunction(lua, f); - lua_call(lua, 0, 1); - - if (strlen(table) == 0) { // Handle the special "" base table. - for (int i = 0; disable[i]; ++i) { - lua_pushnil(lua); - lua_setfield(lua, LUA_GLOBALSINDEX, disable[i]); - } - } else { - for (int i = 0; disable[i]; ++i) { - lua_pushnil(lua); - lua_setfield(lua, -2, disable[i]); - } - add_empty_metatable(lua); - } -} - - -int lsb_require_library(lua_State* lua) -{ - const char* name = luaL_checkstring(lua, 1); - lua_getglobal(lua, lsb_package_table); - if (!lua_istable(lua, -1)) { - return luaL_error(lua, "%s table is missing", lsb_package_table); - } - lua_getfield(lua, -1, lsb_loaded_table); - if (!lua_istable(lua, -1)) { - return luaL_error(lua, "%s.%s table is missing", lsb_package_table, - lsb_loaded_table); - } - lua_getfield(lua, -1, name); - if (!lua_isnil(lua, -1)) { - return 1; // returned the cache copy - } - lua_pop(lua, 1); // remove the nil - int pos = lua_gettop(lua); - lua_pushboolean(lua, 1); - lua_setfield(lua, pos, name); // mark it as loaded to prevent a dependency loop - - if (strcmp(name, LUA_STRLIBNAME) == 0) { - lsb_load_library(lua, name, luaopen_string, lsb_disable_none); - } else if (strcmp(name, LUA_MATHLIBNAME) == 0) { - lsb_load_library(lua, name, luaopen_math, lsb_disable_none); - } else if (strcmp(name, LUA_TABLIBNAME) == 0) { - lsb_load_library(lua, name, luaopen_table, lsb_disable_none); - } else if (strcmp(name, LUA_OSLIBNAME) == 0) { - const char* disable[] = { "execute", "exit", "remove", "rename", - "setlocale", "tmpname", NULL }; - lsb_load_library(lua, name, luaopen_os, disable); - } else if (strcmp(name, mozsvc_circular_buffer_table) == 0) { - lsb_load_library(lua, name, luaopen_circular_buffer, lsb_disable_none); - } else if (strcmp(name, mozsvc_bloom_filter_table) == 0) { - lsb_load_library(lua, name, luaopen_bloom_filter, lsb_disable_none); - } else if (strcmp(name, mozsvc_hyperloglog_table) == 0) { - lsb_load_library(lua, name, luaopen_hyperloglog, lsb_disable_none); - } else if (strcmp(name, "lpeg") == 0) { - lsb_load_library(lua, name, luaopen_lpeg, lsb_disable_none); - } else if (strcmp(name, "cjson") == 0) { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "require_library() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - const char* disable[] = { "new", "encode_keep_buffer", NULL }; - lsb_load_library(lua, name, luaopen_cjson, disable); - if (set_encode_max_buffer(lua, -1, (unsigned)lsb->output.maxsize)) { - return luaL_error(lua, "cjson encode buffer could not be configured"); - } - lua_pushvalue(lua, -1); - lua_setglobal(lua, name); - } else if (strcmp(name, "struct") == 0) { - lsb_load_library(lua, name, luaopen_struct, lsb_disable_none); - } else { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "require_library() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - if (!lsb->require_path) { - return luaL_error(lua, "require_library() external modules are disabled"); - } - - int i = 0; - while (name[i]) { - if (!isalnum(name[i]) && name[i] != '_') { - return luaL_error(lua, "invalid module name '%s'", name); - } - ++i; - } - char fn[MAX_PATH]; - i = snprintf(fn, MAX_PATH, "%s%c%s.lua", lsb->require_path, PATH_DELIMITER, - name); - if (i < 0 || i >= MAX_PATH) { - return luaL_error(lua, "require_path exceeded %d", MAX_PATH); - } - - if (luaL_dofile(lua, fn) != 0) { - return luaL_error(lua, "%s", lua_tostring(lua, -1)); - } - add_empty_metatable(lua); - } - lua_pushvalue(lua, -1); - lua_setfield(lua, pos, name); - return 1; -} diff --git a/src/lsb_modules.h b/src/lsb_modules.h deleted file mode 100644 index a62b3bc..0000000 --- a/src/lsb_modules.h +++ /dev/null @@ -1,50 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Lua sandbox module management @file */ - -#ifndef lsb_modules_h_ -#define lsb_modules_h_ - -#include - -#include "lua_bloom_filter.h" -#include "lua_circular_buffer.h" -#include "lua_hyperloglog.h" - -extern const char* lsb_disable_none[]; -extern const char* lsb_package_table; -extern const char* lsb_loaded_table; - -/** - * Performs the library load and secures the sandbox environment for use. - * - * @param lua Pointer to the Lua state. - * @param table Name of the table being loaded. - * @param f Pointer to the table load function. - * @param disable Array of function names to disable in the loaded table. - */ -void lsb_load_library(lua_State* lua, const char* table, lua_CFunction f, - const char** disable); - -/** - * Overridden 'require' used to load optional sandbox libraries in global space. - * - * @param lua Pointer to the Lua state. - * - * @return int Returns 1 value on the stack (for the standard modules a table - * for the LPEG grammars, userdata). - */ -int lsb_require_library(lua_State* lua); - -/* declarations for modules without headers */ -LUALIB_API int luaopen_cjson(lua_State* L); -int set_encode_max_buffer(lua_State* L, int index, unsigned maxsize); - -LUALIB_API int luaopen_lpeg(lua_State* L); -LUALIB_API int luaopen_struct(lua_State* L); - -#endif diff --git a/src/lsb_private.h b/src/lsb_private.h index 31fa2e7..26b1f00 100644 --- a/src/lsb_private.h +++ b/src/lsb_private.h @@ -33,7 +33,6 @@ struct lua_sandbox { lsb_state state; lsb_output_data output; char* lua_file; - char* require_path; unsigned usage[LSB_UT_MAX][LSB_US_MAX]; char error_message[LSB_ERROR_SIZE]; }; diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index 489daca..ba0db2b 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -14,7 +14,6 @@ #include #include -#include "lsb_modules.h" #include "lsb_private.h" typedef struct @@ -366,22 +365,21 @@ serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) { - static const char* G = "_G"; - // make sure the string library is loaded before we start lua_getglobal(lsb->lua, LUA_STRLIBNAME); if (!lua_istable(lsb->lua, -1)) { - lsb_load_library(lsb->lua, LUA_STRLIBNAME, luaopen_string, - lsb_disable_none); + lua_getglobal(lsb->lua, "require"); + if (!lua_iscfunction(lsb->lua, -1)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_preserve_global_data 'require' not found"); + return 1; + } + lua_pushstring(lsb->lua, LUA_STRLIBNAME); + lua_call(lsb->lua, 1, 1); } - lua_pop(lsb->lua, 1); // Remove string table. + lua_pop(lsb->lua, 1); - lua_getglobal(lsb->lua, G); - if (!lua_istable(lsb->lua, -1)) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_preserve_global_data cannot access the global table"); - return 1; - } + lua_pushvalue(lsb->lua, LUA_GLOBALSINDEX); FILE* fh = fopen(data_file, "wb"); if (fh == NULL) { @@ -427,7 +425,7 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) preservation_version, preservation_version, get_preservation_version(lsb->lua)); - lsb_appendf(&data.keys, "%s", G); + lsb_appends(&data.keys, "_G", 2); data.keys.pos += 1; data.globals = lua_topointer(lsb->lua, -1); lua_checkstack(lsb->lua, 2); diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 77ccf5e..5f364a3 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -5,6 +5,18 @@ add_executable(test_lua_sandbox test_lua_sandbox.c) target_link_libraries(test_lua_sandbox luasandbox) +add_test(NAME test_setup_cmod COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/modules ${CMAKE_CURRENT_SOURCE_DIR}/modules) +if(WIN32) + add_test(NAME test_setup_lib COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib ${CMAKE_CURRENT_SOURCE_DIR}) + if(MSVC) + set(_DLL_NAME "luasandbox.dll") + else() + set(_DLL_NAME "libluasandbox.dll") + endif() + add_test(NAME test_setup_target COMMAND cmake -E copy ${CMAKE_BINARY_DIR}/src/${_DLL_NAME} ${CMAKE_CURRENT_SOURCE_DIR}) +endif() + +add_test(NAME test_setup_lua COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_SOURCE_DIR}/modules) add_test(NAME test_sandbox WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND test_lua_sandbox) install(TARGETS test_lua_sandbox DESTINATION lib) diff --git a/src/test/lua/errors.lua b/src/test/lua/errors.lua index ab1643e..e9b52b3 100644 --- a/src/test/lua/errors.lua +++ b/src/test/lua/errors.lua @@ -32,12 +32,6 @@ function process(tc) local v = require "../invalid" elseif tc == 10 then local v = require "pathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflow" - elseif tc == 11 then - package = nil - require "os" - elseif tc == 12 then - package.loaded = nil - require "os" end return 0 end diff --git a/src/test/lua/serialize_noglobal.lua b/src/test/lua/serialize_noglobal.lua deleted file mode 100644 index 37c87b4..0000000 --- a/src/test/lua/serialize_noglobal.lua +++ /dev/null @@ -1,5 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -_G = nil diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 43e5938..fd146b1 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -212,7 +212,7 @@ int write_message(lua_State* lua) static char* test_create_error() { - lua_sandbox* sb = lsb_create(NULL, NULL, "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, NULL, "modules", 0, 0, 0); mu_assert(!sb, "lsb_create() null lua_file"); return NULL; @@ -226,7 +226,7 @@ static char* test_init_error() mu_assert(result == 0, "lsb_init() null sandbox ptr"); // load error - lua_sandbox* sb = lsb_create(NULL, "lua/simple1.lua", "../../modules", 0, + lua_sandbox* sb = lsb_create(NULL, "lua/simple1.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, NULL); @@ -238,7 +238,7 @@ static char* test_init_error() mu_assert(!e, "lsb_destroy() received: %s", e); // out of memory - sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 6000, 0, 0); + sb = lsb_create(NULL, "lua/simple.lua", "modules", 6000, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, NULL); mu_assert(result == 2, "lsb_init() received: %d %s", result, @@ -253,7 +253,7 @@ static char* test_init_error() result = lsb_init(sb, NULL); mu_assert(result == 2, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - const char* expected = "lua/lpeg_date_time.lua:7: require_library() external modules are disabled"; + const char* expected = "lua/lpeg_date_time.lua:7: module 'date_time' not found:"; mu_assert(strcmp(lsb_get_error(sb), expected) == 0, "lsb_get_error() received: %s", lsb_get_error(sb)); @@ -271,7 +271,7 @@ static char* test_destroy_error() e = lsb_destroy(NULL, NULL); mu_assert(!e, "lsb_destroy() received: %s", e); - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -292,7 +292,7 @@ static char* test_usage_error() unsigned u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); mu_assert(u == 0, "NULL sandbox memory usage received: %u", u); - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); u = lsb_usage(NULL, LSB_UT_MAX + 1, LSB_US_CURRENT); @@ -327,7 +327,7 @@ static char* test_misc() static char* test_simple() { - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 65765, 1000, 1024); mu_assert(sb, "lsb_create() received: NULL"); @@ -378,9 +378,10 @@ static char* test_simple() return NULL; } + static char* test_simple_error() { - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -434,7 +435,7 @@ static char* test_output() enum {output_size = 63 * 1024}; - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", 0, 0 + lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 0, 0 , output_size); mu_assert(sb, "lsb_create() received: NULL"); @@ -487,13 +488,13 @@ static char* test_output_errors() , "process() lua/output_errors.lua:24: bad argument #1 to 'write_output' (unknown userdata type)" , "process() lua/output_errors.lua:27: output_limit exceeded" , "process() lua/output_errors.lua:30: write_message() could not encode protobuf - unsupported array type: table" - , "process() lua/output_errors.lua:36: strbuf max_size exceeded" + , "process() lua/output_errors.lua:36: strbuf output_limit exceeded" , "process() lua/output_errors.lua:38: write_message() could not encode protobuf - takes a single table argument" , NULL }; for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/output_errors.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/output_errors.lua", "modules", 0, 0, 128); mu_assert(sb, "lsb_create() received: NULL"); @@ -528,7 +529,7 @@ static char* test_cbuf_errors() for (int i = 0; tests[i]; ++i) { lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer_errors.lua", - "../../modules", 32767, 0, 0); + "modules", 32767, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -560,7 +561,7 @@ static char* test_cbuf() , NULL }; - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); @@ -630,7 +631,7 @@ static char* test_cbuf_delta() }; lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer_delta.lua", - "../../modules", 0, 0, 0); + "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -671,7 +672,7 @@ static char* test_cbuf_delta() static char* test_cjson() { - lua_sandbox* sb = lsb_create(NULL, "lua/cjson.lua", "../../modules", 0, 0, 64); + lua_sandbox* sb = lsb_create(NULL, "lua/cjson.lua", "modules", 0, 0, 64); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -691,7 +692,7 @@ static char* test_cjson() static char* test_cjson_unlimited() { - lua_sandbox* sb = lsb_create(NULL, "lua/cjson_unlimited.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/cjson_unlimited.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -716,9 +717,9 @@ static char* test_errors() { const char* tests[] = { #ifdef _WIN32 - "process() lua/errors.lua:9: cannot open lua\\unknown.lua: No such file or directory" + "process() lua/errors.lua:9: module 'unknown' not found:\n\tno file 'lua\\unknown.lua'\n\tno file 'lua\\unknown.dll'" #else - "process() lua/errors.lua:9: cannot open lua/unknown.lua: No such file or directory" + "process() lua/errors.lua:9: module 'unknown' not found:\n\tno file 'lua/unknown.lua'\n\tno file 'lua/unknown.so'" #endif , "process() lua/errors.lua:11: bad argument #0 to 'output' (must have at least one argument)" , "process() not enough memory" @@ -728,14 +729,12 @@ static char* test_errors() , "process() must return a single numeric value" , "process() lua/errors.lua:27: output_limit exceeded" #ifdef _WIN32 - , "process() lua/errors.lua:30: lua\\bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" + , "process() lua\\bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" #else - , "process() lua/errors.lua:30: lua/bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" + , "process() lua/bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" #endif - , "process() lua/errors.lua:32: invalid module name '../invalid'" - , "process() lua/errors.lua:34: require_path exceeded 260" - , "process() lua/errors.lua:37: package table is missing" - , "process() lua/errors.lua:40: package.loaded table is missing" + , "process() invalid module name '../invalid'" + , "process() lua/errors.lua:34: module 'pathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpa" , NULL }; @@ -778,7 +777,7 @@ static char* test_lpeg() }; for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, tests[i], "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, tests[i], "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -800,7 +799,7 @@ static char* test_lpeg() static char* test_util() { - lua_sandbox* sb = lsb_create(NULL, "lua/util_test.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/util_test.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -821,7 +820,7 @@ static char* test_util() static char* test_serialize() { const char* output_file = "serialize.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "modules", 64000, 1000, 64000); mu_assert(sb, "lsb_create() received: NULL"); @@ -849,7 +848,7 @@ static char* test_restore() { const char* output_file = "restore.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/restore.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/restore.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -865,7 +864,7 @@ static char* test_restore() mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/restore.lua", "../../modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/restore.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -882,7 +881,7 @@ static char* test_restore() mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data with a version change - sb = lsb_create(NULL, "lua/restore.lua", "../../modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/restore.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -906,31 +905,7 @@ static char* test_serialize_failure() const char* expected = "serialize_data cannot preserve type 'function'"; lua_sandbox* sb = lsb_create(NULL, - "lua/serialize_failure.lua", "../../modules", - 32767, 1000, 1024); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); - mu_assert(e, "lsb_destroy() received: no error"); - mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); - free(e); - e = NULL; - mu_assert(file_exists(output_file) == 0, "output file was not cleaned up"); - - return NULL; -} - - -static char* test_serialize_noglobal() -{ - const char* output_file = "serialize_noglobal.preserve"; - const char* expected = "lsb_preserve_global_data cannot access the global table"; - - lua_sandbox* sb = lsb_create(NULL, - "lua/serialize_noglobal.lua", "../../modules", + "lua/serialize_failure.lua", "modules", 32767, 1000, 1024); mu_assert(sb, "lsb_create() received: NULL"); @@ -958,7 +933,7 @@ static char* test_bloom_filter() , NULL }; - lua_sandbox* sb = lsb_create(NULL, "lua/bloom_filter.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/bloom_filter.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); @@ -990,7 +965,7 @@ static char* test_bloom_filter() mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/bloom_filter.lua", "../../modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/bloom_filter.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); @@ -1026,7 +1001,7 @@ static char* test_hyperloglog() { const char* output_file = "hyperloglog.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); @@ -1056,7 +1031,7 @@ static char* test_hyperloglog() mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/hyperloglog.lua", "../../modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/hyperloglog.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); @@ -1095,7 +1070,7 @@ static char* test_hyperloglog() static char* test_struct() { - lua_sandbox* sb = lsb_create(NULL, "lua/struct.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/struct.lua", "modules", 65765, 1000, 1024); mu_assert(sb, "lsb_create() received: NULL"); @@ -1117,7 +1092,7 @@ static char* benchmark_counter() { int iter = 10000000; - lua_sandbox* sb = lsb_create(NULL, "lua/counter.lua", "../../modules", 32000, + lua_sandbox* sb = lsb_create(NULL, "lua/counter.lua", "modules", 32000, 10, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -1144,7 +1119,7 @@ static char* benchmark_serialize() clock_t t = clock(); for (int x = 0; x < iter; ++x) { - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "modules", 64000, 1000, 1024); mu_assert(sb, "lsb_create() received: NULL"); @@ -1168,7 +1143,7 @@ static char* benchmark_deserialize() clock_t t = clock(); for (int x = 0; x < iter; ++x) { - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); @@ -1190,7 +1165,7 @@ static char* benchmark_lpeg_decoder() { int iter = 10000; - lua_sandbox* sb = lsb_create(NULL, "lua/decoder.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/decoder.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1216,7 +1191,7 @@ static char* benchmark_lua_types_output() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 100000, 1000, 1024 * 63); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -1242,7 +1217,7 @@ static char* benchmark_cbuf_output() { int iter = 10000; - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1267,7 +1242,7 @@ static char* benchmark_message_output() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", 0, 0, 0); + lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1293,7 +1268,7 @@ static char* benchmark_cbuf_add() int iter = 1000000; lua_sandbox* sb = lsb_create(NULL, - "lua/circular_buffer_add.lua", "../../modules", + "lua/circular_buffer_add.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -1323,7 +1298,7 @@ static char* benchmark_bloom_filter_add() int iter = 1000000; lua_sandbox* sb = lsb_create(NULL, - "lua/bloom_filter_benchmark.lua", "../../modules", + "lua/bloom_filter_benchmark.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -1353,7 +1328,7 @@ static char* benchmark_hyperloglog_add() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "../../modules", + lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -1401,7 +1376,6 @@ static char* all_tests() mu_run_test(test_serialize); mu_run_test(test_restore); mu_run_test(test_serialize_failure); - mu_run_test(test_serialize_noglobal); mu_run_test(test_bloom_filter); mu_run_test(test_hyperloglog); mu_run_test(test_struct); From c77a01f3fada3a6f1c716cf21e0132156da674c7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 22 Jan 2015 14:15:51 -0800 Subject: [PATCH 034/235] Allow the sandbox restrictions to be more configurable --- cmake/externals.cmake | 2 +- cmake/lua-5_1_5.patch | 102 +++++++++++++++----------- include/lsb.h | 27 +++++++ src/lsb.c | 142 +++++++++++++++++++++++++++--------- src/test/test_lua_sandbox.c | 6 +- 5 files changed, 201 insertions(+), 78 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 36910db..9a217b6 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -79,7 +79,7 @@ add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG 9c3a1805796747d59e5b4f0a789796f368285251 + GIT_TAG 1ebc6061435a45ce23bf7d26d7ed04db995a8ba9 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch index 6321516..2469cdd 100644 --- a/cmake/lua-5_1_5.patch +++ b/cmake/lua-5_1_5.patch @@ -1,6 +1,6 @@ diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt --- /work/lua-5.1.5/CMakeLists.txt 1969-12-31 16:00:00.000000000 -0800 -+++ lua-5_1_5/CMakeLists.txt 2015-01-16 10:26:29.623543211 -0800 ++++ lua-5_1_5/CMakeLists.txt 2015-01-22 12:06:12.578775487 -0800 @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this @@ -68,7 +68,7 @@ diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt +install(FILES src/lua.h src/luaconf.h src/lualib.h src/lauxlib.h DESTINATION include) diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c --- /work/lua-5.1.5/src/ldebug.c 2008-05-08 09:56:26.000000000 -0700 -+++ lua-5_1_5/src/ldebug.c 2015-01-16 10:26:29.623543211 -0800 ++++ lua-5_1_5/src/ldebug.c 2015-01-22 12:06:12.578775487 -0800 @@ -80,6 +80,10 @@ return L->basehookcount; } @@ -82,8 +82,8 @@ diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c int status; diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c --- /work/lua-5.1.5/src/loadlib.c 2009-09-09 06:17:16.000000000 -0700 -+++ lua-5_1_5/src/loadlib.c 2015-01-21 06:37:34.841228660 -0800 -@@ -23,23 +23,22 @@ ++++ lua-5_1_5/src/loadlib.c 2015-01-22 13:40:46.057329536 -0800 +@@ -23,23 +23,24 @@ /* prefix for open functions in C libraries */ @@ -111,10 +111,12 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c +#define ERRFUNC 2 -#define setprogdir(L) ((void)0) ++ ++#define LSB_CONFIG "lsb_config" static void ll_unloadlib (void *lib); -@@ -91,24 +90,6 @@ +@@ -91,24 +92,6 @@ #include @@ -139,7 +141,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static void pusherror (lua_State *L) { int error = GetLastError(); char buffer[128]; -@@ -153,7 +134,7 @@ +@@ -153,7 +136,7 @@ /* Mac appends a `_' before C function names */ #undef POF @@ -148,7 +150,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static void pusherror (lua_State *L) { -@@ -230,10 +211,10 @@ +@@ -230,10 +213,10 @@ */ #undef LIB_FAIL @@ -161,7 +163,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static void ll_unloadlib (void *lib) { -@@ -351,11 +332,20 @@ +@@ -351,11 +334,24 @@ static const char *findfile (lua_State *L, const char *name, const char *pname) { const char *path; @@ -175,8 +177,12 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c + } + ++i; + } ++ lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); ++ if (lua_type(L, -1) != LUA_TTABLE) { ++ return NULL; ++ } + lua_pushstring(L, name); -+ lua_getfield(L, LUA_REGISTRYINDEX, pname); ++ lua_getfield(L, -2, pname); path = lua_tostring(L, -1); - if (path == NULL) - luaL_error(L, LUA_QL("package.%s") " must be a string", pname); @@ -186,25 +192,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c lua_pushliteral(L, ""); /* error accumulator */ while ((path = pushnexttemplate(L, path)) != NULL) { const char *filename; -@@ -380,7 +370,7 @@ - static int loader_Lua (lua_State *L) { - const char *filename; - const char *name = luaL_checkstring(L, 1); -- filename = findfile(L, name, "path"); -+ filename = findfile(L, name, "lsb_require_path"); - if (filename == NULL) return 1; /* library not found in this path */ - if (luaL_loadfile(L, filename) != 0) - loaderror(L, filename); -@@ -402,7 +392,7 @@ - static int loader_C (lua_State *L) { - const char *funcname; - const char *name = luaL_checkstring(L, 1); -- const char *filename = findfile(L, name, "cpath"); -+ const char *filename = findfile(L, name, "lsb_require_cpath"); - if (filename == NULL) return 1; /* library not found in this path */ - funcname = mkfuncname(L, name); - if (ll_loadfunc(L, filename, funcname) != 0) -@@ -411,41 +401,29 @@ +@@ -411,41 +407,61 @@ } @@ -243,14 +231,46 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c +// If necessary, add an empty metatable to flag the table as non-data +// during preservation. -+static void add_empty_metatable(lua_State* lua) ++static void add_empty_metatable(lua_State* L) +{ -+ if (lua_getmetatable(lua, -1) == 0) { -+ lua_newtable(lua); -+ lua_setmetatable(lua, -2); ++ if (lua_getmetatable(L, -1) == 0) { ++ lua_newtable(L); ++ lua_setmetatable(L, -2); + } else { -+ lua_pop(lua, 1); ++ lua_pop(L, 1); ++ } ++} ++ ++ ++static void remove_entries(lua_State* L, const char* name) ++{ ++ lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); ++ if (lua_type(L, -1) != LUA_TTABLE) { ++ lua_pop(L, 1); ++ return; ++ } ++ lua_getfield(L, -1, "remove_entries"); ++ if (lua_type(L, -1) != LUA_TTABLE) { ++ lua_pop(L, 2); ++ return; ++ } ++ lua_getfield(L, -1, name); ++ if (lua_type(L, -1) != LUA_TTABLE) { ++ lua_pop(L, 3); ++ return; ++ } ++ int n = 1; ++ while (1) { ++ lua_rawgeti(L, -1, n); ++ if (lua_type(L, -1) == LUA_TNIL) { ++ lua_pop(L, 1); ++ break; ++ } ++ lua_pushnil(L); ++ lua_settable(L, -6); ++ ++n; + } ++ lua_pop(L, 3); +} static const int sentinel_ = 0; @@ -259,7 +279,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static int ll_require (lua_State *L) { -@@ -462,7 +440,7 @@ +@@ -462,7 +478,7 @@ /* else must load it; iterate over available loaders */ lua_getfield(L, LUA_ENVIRONINDEX, "loaders"); if (!lua_istable(L, -1)) @@ -268,12 +288,12 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c lua_pushliteral(L, ""); /* error message accumulator */ for (i=1; ; i++) { lua_rawgeti(L, -2, i); /* get a loader */ -@@ -490,138 +468,27 @@ +@@ -490,138 +506,27 @@ lua_pushvalue(L, -1); /* extra copy to be returned */ lua_setfield(L, 2, name); /* _LOADED[name] = true */ } + add_empty_metatable(L); -+ /* todo apply restrictions */ ++ remove_entries(L, name); return 1; } @@ -410,7 +430,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c LUALIB_API int luaopen_package (lua_State *L) { -@@ -630,12 +497,8 @@ +@@ -630,12 +535,8 @@ luaL_newmetatable(L, "_LOADLIB"); lua_pushcfunction(L, gctm); lua_setfield(L, -2, "__gc"); @@ -425,7 +445,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c lua_pushvalue(L, -1); lua_replace(L, LUA_ENVIRONINDEX); /* create `loaders' table */ -@@ -646,21 +509,17 @@ +@@ -646,21 +547,17 @@ lua_rawseti(L, -2, i+1); } lua_setfield(L, -2, "loaders"); /* put it in field `loaders' */ @@ -457,7 +477,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c - diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c --- /work/lua-5.1.5/src/loslib.c 2008-01-18 08:38:18.000000000 -0800 -+++ lua-5_1_5/src/loslib.c 2015-01-16 10:26:29.623543211 -0800 ++++ lua-5_1_5/src/loslib.c 2015-01-22 12:06:12.578775487 -0800 @@ -33,6 +33,61 @@ return 3; } @@ -590,7 +610,7 @@ diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c } diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h --- /work/lua-5.1.5/src/luaconf.h 2008-02-11 08:25:08.000000000 -0800 -+++ lua-5_1_5/src/luaconf.h 2015-01-16 10:26:29.623543211 -0800 ++++ lua-5_1_5/src/luaconf.h 2015-01-22 12:06:12.578775487 -0800 @@ -1,5 +1,5 @@ /* -** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ @@ -635,7 +655,7 @@ diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h diff -Naur /work/lua-5.1.5/src/lua.h lua-5_1_5/src/lua.h --- /work/lua-5.1.5/src/lua.h 2012-01-13 12:36:20.000000000 -0800 -+++ lua-5_1_5/src/lua.h 2015-01-16 10:26:29.623543211 -0800 ++++ lua-5_1_5/src/lua.h 2015-01-22 12:06:12.578775487 -0800 @@ -341,6 +341,7 @@ LUA_API lua_Hook lua_gethook (lua_State *L); LUA_API int lua_gethookmask (lua_State *L); diff --git a/include/lsb.h b/include/lsb.h index e995c50..5c23630 100644 --- a/include/lsb.h +++ b/include/lsb.h @@ -69,6 +69,33 @@ LSB_EXPORT lua_sandbox* lsb_create(void* parent, unsigned instruction_limit, unsigned output_limit); +/** + * Allocates and initializes the structure around the Lua sandbox allowing + * full specification of the sandbox configuration using a Lua configuration + * string. + * + * { + * memory_limit = 1024*1024*1, + * instruction_limit = 10000, + * output_limit = 64*1024, + * path = '/modules/?.lua', + * cpath = '/modules/?.so', + * remove_entries = { + * [''] = { 'collectgarbage', 'coroutine', 'dofile', 'load', 'loadfile', + * 'loadstring', 'module', 'print'}, + * os = {'execute', 'exit', 'remove', 'rename', 'setlocale', 'tmpname'} + * } + * } + * + * @param parent Pointer to associate the owner to this sandbox. + * @param lua_file Filename of the Lua script to run in this sandbox. + * @param config Lua structure defining the full sandbox restrictions. + * @return lua_sandbox Sandbox pointer or NULL on failure. + */ +LSB_EXPORT lua_sandbox* lsb_create_custom(void* parent, + const char* lua_file, + const char* config); + /** * Initializes the Lua sandbox and loads/runs the Lua script that was specified * in lua_create_sandbox. diff --git a/src/lsb.c b/src/lsb.c index 8d81650..1b7d97d 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -23,11 +23,17 @@ #include "lsb_serialize.h" #include "lsb_serialize_protobuf.h" -static const char* disable_base_functions[] = { "collectgarbage", "coroutine", - "dofile", "load", "loadfile", "loadstring", "module", "print", NULL }; - -static const char* require_lpath = "lsb_require_path"; -static const char* require_cpath = "lsb_require_cpath"; +static const char* standard_config = "{" + "memory_limit = %u," + "instruction_limit = %u," + "output_limit = %u," + "path = '%s'," + "cpath = '%s'," + "remove_entries = {" + "[''] = { 'collectgarbage', 'coroutine', 'dofile', 'load', 'loadfile'" + ",'loadstring', 'module', 'print'}," + "os = {'execute', 'exit', 'remove', 'rename', 'setlocale', 'tmpname'}" + "}}"; static jmp_buf g_jbuf; @@ -36,6 +42,7 @@ LUALIB_API int luaopen_cjson(lua_State* L); LUALIB_API int luaopen_lpeg(lua_State* L); static const luaL_Reg preload_module_list[] = { + { "", luaopen_base }, { LUA_TABLIBNAME, luaopen_table }, { LUA_IOLIBNAME, luaopen_io }, { LUA_OSLIBNAME, luaopen_os }, @@ -47,7 +54,8 @@ static const luaL_Reg preload_module_list[] = { { NULL, NULL } }; -static int libsize (const luaL_Reg *l) { +static int libsize(const luaL_Reg* l) +{ int size = 0; for (; l->name; l++) size++; return size; @@ -55,7 +63,7 @@ static int libsize (const luaL_Reg *l) { static void preload_modules(lua_State* lua) { - const luaL_Reg *lib = preload_module_list; + const luaL_Reg* lib = preload_module_list; luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", libsize(preload_module_list)); for (; lib->func; lib++) { @@ -150,6 +158,32 @@ static int unprotected_panic(lua_State* lua) } +static unsigned get_usage_config(lua_State* lua, int idx, const char* item) +{ + lua_getfield(lua, idx, item); + unsigned u = (unsigned)lua_tonumber(lua, -1); // defaults to zero + lua_pop(lua, 1); + return u; +} + + +static int expand_path(const char* path, int n, const char* fmt, char* opath) +{ + // not an exhaustive check, just making sure the Lua markers are accounted for + for (int i = 0; (path[i]); ++i) { + if (path[i] == '\'' || path[i] == ';' || path[i] == '?' + || !isprint(path[i])) { + return 1; + } + } + int result = snprintf(opath, n, fmt, path); + if (result < 0 || result > n - 1) { + return 1; + } + return 0; +} + + lua_sandbox* lsb_create(void* parent, const char* lua_file, const char* require_path, @@ -161,6 +195,47 @@ lua_sandbox* lsb_create(void* parent, return NULL; } + char config[1024 * 2]; + char lpath[260] = { 0 }; + char cpath[260] = { 0 }; + + if (require_path) { +#if defined(_WIN32) + if (expand_path(require_path, sizeof(lpath), "%s\\?.lua", lpath)) { + return NULL; + } + if (expand_path(require_path, sizeof(cpath), "%s\\?.dll", cpath)) { + return NULL; + } +#else + if (expand_path(require_path, sizeof(lpath), "%s/?.lua", lpath)) { + return NULL; + } + if (expand_path(require_path, sizeof(cpath), "%s/?.so", cpath)) { + return NULL; + } +#endif + } + + int result = snprintf(config, sizeof(config), standard_config, memory_limit, + instruction_limit, output_limit, lpath, cpath); + + if (result < 0 || result > (int)sizeof(config) - 1) { + return NULL; + } + + return lsb_create_custom(parent, lua_file, config); +} + + +lua_sandbox* lsb_create_custom(void* parent, + const char* lua_file, + const char* config) +{ + if (!lua_file) { + return NULL; + } + #if _WIN32 if (_putenv("TZ=UTC") != 0) { return NULL; @@ -187,6 +262,21 @@ lua_sandbox* lsb_create(void* parent, return NULL; } + // create config table + lua_pushfstring(lsb->lua, "return %s", config); + if (luaL_dostring(lsb->lua, lua_tostring(lsb->lua, -1)) + || lua_type(lsb->lua, -1) != LUA_TTABLE) { + lua_close(lsb->lua); + lsb->lua = NULL; + free(lsb); + return NULL; + } + unsigned memory_limit = get_usage_config(lsb->lua, -1, "memory_limit"); + unsigned instruction_limit = get_usage_config(lsb->lua, -1, "instruction_limit"); + unsigned output_limit = get_usage_config(lsb->lua, -1, "output_limit"); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, "lsb_config"); + lua_pop(lsb->lua, 1); // remove configuration string + lsb->parent = parent; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = memory_limit; lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] = instruction_limit; @@ -203,28 +293,12 @@ lua_sandbox* lsb_create(void* parent, lsb->output.data = malloc(lsb->output.size); size_t len = strlen(lua_file); lsb->lua_file = malloc(len + 1); - if (require_path) { -#if defined(_WIN32) - lua_pushfstring(lsb->lua, "%s\\?.lua", require_path); - lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_lpath); - lua_pushfstring(lsb->lua, "%s\\?.dll", require_path); - lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_cpath); -#else - lua_pushfstring(lsb->lua, "%s/?.lua", require_path); - lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_lpath); - lua_pushfstring(lsb->lua, "%s/?.so", require_path); - lua_setfield(lsb->lua, LUA_REGISTRYINDEX, require_cpath); -#endif - } - - lua_pushinteger(lsb->lua, output_limit); - lua_setfield(lsb->lua, LUA_REGISTRYINDEX, "lsb_output_limit"); if (!lsb->output.data || !lsb->lua_file) { - free(lsb); free(lsb->lua_file); lua_close(lsb->lua); lsb->lua = NULL; + free(lsb); return NULL; } strcpy(lsb->lua_file, lua_file); @@ -243,15 +317,6 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif - // load base module - lua_pushcfunction(lsb->lua, luaopen_base); - lua_pushstring(lsb->lua, ""); - lua_call(lsb->lua, 1, 0); - for (int i = 0; disable_base_functions[i]; ++i) { - lua_pushnil(lsb->lua); - lua_setfield(lsb->lua, LUA_GLOBALSINDEX, disable_base_functions[i]); - } - preload_modules(lsb->lua); // load package module @@ -262,6 +327,17 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) lua_setmetatable(lsb->lua, -2); lua_pop(lsb->lua, 1); + // load base module + lua_getglobal(lsb->lua, "require"); + if (!lua_iscfunction(lsb->lua, -1)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_init 'require' not found"); + lsb_terminate(lsb, NULL); + return 1; + } + lua_pushstring(lsb->lua, ""); + lua_call(lsb->lua, 1, 0); + lua_pushlightuserdata(lsb->lua, (void*)lsb); lua_pushcclosure(lsb->lua, &output, 1); lua_setglobal(lsb->lua, "output"); diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index fd146b1..91b4dfa 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -113,7 +113,7 @@ int process(lua_sandbox* lsb, double ts) if (!lua_isnumber(lua, 1)) { char err[LSB_ERROR_SIZE]; int len = snprintf(err, LSB_ERROR_SIZE, - "%s() must return a single numeric value", func_name); + "%s() must return a numeric error code", func_name); if (len >= LSB_ERROR_SIZE || len < 0) { err[LSB_ERROR_SIZE - 1] = 0; } @@ -725,8 +725,8 @@ static char* test_errors() , "process() not enough memory" , "process() instruction_limit exceeded" , "process() lua/errors.lua:20: attempt to perform arithmetic on global 'x' (a nil value)" - , "process() must return a single numeric value" - , "process() must return a single numeric value" + , "process() must return a numeric error code" + , "process() must return a numeric error code" , "process() lua/errors.lua:27: output_limit exceeded" #ifdef _WIN32 , "process() lua\\bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" From 8ed212735e4cda6d05d955295be196b2d154b9d6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 26 Jan 2015 10:46:17 -0800 Subject: [PATCH 035/235] Add the module function and submodule support back into the sandbox --- cmake/lua-5_1_5.patch | 186 ++++++--------------------- src/lsb.c | 2 +- src/test/lua/errors.lua | 2 + src/test/output/serialize.lua51.data | 4 +- src/test/test_lua_sandbox.c | 5 + 5 files changed, 49 insertions(+), 150 deletions(-) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch index 2469cdd..fb2ec20 100644 --- a/cmake/lua-5_1_5.patch +++ b/cmake/lua-5_1_5.patch @@ -1,6 +1,6 @@ diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt --- /work/lua-5.1.5/CMakeLists.txt 1969-12-31 16:00:00.000000000 -0800 -+++ lua-5_1_5/CMakeLists.txt 2015-01-22 12:06:12.578775487 -0800 ++++ lua-5_1_5/CMakeLists.txt 2015-01-22 13:55:49.917296821 -0800 @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this @@ -68,7 +68,7 @@ diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt +install(FILES src/lua.h src/luaconf.h src/lualib.h src/lauxlib.h DESTINATION include) diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c --- /work/lua-5.1.5/src/ldebug.c 2008-05-08 09:56:26.000000000 -0700 -+++ lua-5_1_5/src/ldebug.c 2015-01-22 12:06:12.578775487 -0800 ++++ lua-5_1_5/src/ldebug.c 2015-01-22 13:55:49.917296821 -0800 @@ -80,6 +80,10 @@ return L->basehookcount; } @@ -82,8 +82,8 @@ diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c int status; diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c --- /work/lua-5.1.5/src/loadlib.c 2009-09-09 06:17:16.000000000 -0700 -+++ lua-5_1_5/src/loadlib.c 2015-01-22 13:40:46.057329536 -0800 -@@ -23,23 +23,24 @@ ++++ lua-5_1_5/src/loadlib.c 2015-01-26 10:00:16.095518127 -0800 +@@ -23,24 +23,24 @@ /* prefix for open functions in C libraries */ @@ -111,12 +111,12 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c +#define ERRFUNC 2 -#define setprogdir(L) ((void)0) -+ -+#define LSB_CONFIG "lsb_config" ++#define LSB_CONFIG "lsb_config" static void ll_unloadlib (void *lib); -@@ -91,24 +92,6 @@ + static void *ll_load (lua_State *L, const char *path); +@@ -91,24 +91,6 @@ #include @@ -141,7 +141,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static void pusherror (lua_State *L) { int error = GetLastError(); char buffer[128]; -@@ -153,7 +136,7 @@ +@@ -153,7 +135,7 @@ /* Mac appends a `_' before C function names */ #undef POF @@ -150,7 +150,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static void pusherror (lua_State *L) { -@@ -230,10 +213,10 @@ +@@ -230,10 +212,10 @@ */ #undef LIB_FAIL @@ -163,15 +163,13 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static void ll_unloadlib (void *lib) { -@@ -351,11 +334,24 @@ +@@ -351,8 +333,20 @@ static const char *findfile (lua_State *L, const char *name, const char *pname) { const char *path; -- name = luaL_gsub(L, name, ".", LUA_DIRSEP); -- lua_getfield(L, LUA_ENVIRONINDEX, pname); + int i = 0; + while (name[i]) { -+ if (!isalnum(name[i]) && name[i] != '_') { ++ if (!isalnum(name[i]) && name[i] != '_' && name[i] != '.') { + luaL_error(L, "invalid module name '%s'", name); + return NULL; // never reached, just silences the compiler + } @@ -181,43 +179,13 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c + if (lua_type(L, -1) != LUA_TTABLE) { + return NULL; + } -+ lua_pushstring(L, name); + name = luaL_gsub(L, name, ".", LUA_DIRSEP); +- lua_getfield(L, LUA_ENVIRONINDEX, pname); + lua_getfield(L, -2, pname); path = lua_tostring(L, -1); -- if (path == NULL) -- luaL_error(L, LUA_QL("package.%s") " must be a string", pname); -+ if (path == NULL) { -+ return NULL; -+ } - lua_pushliteral(L, ""); /* error accumulator */ - while ((path = pushnexttemplate(L, path)) != NULL) { - const char *filename; -@@ -411,41 +407,61 @@ - } - - --static int loader_Croot (lua_State *L) { -- const char *funcname; -- const char *filename; -- const char *name = luaL_checkstring(L, 1); -- const char *p = strchr(name, '.'); -- int stat; -- if (p == NULL) return 0; /* is root */ -- lua_pushlstring(L, name, p - name); -- filename = findfile(L, lua_tostring(L, -1), "cpath"); -- if (filename == NULL) return 1; /* root not found */ -- funcname = mkfuncname(L, name); -- if ((stat = ll_loadfunc(L, filename, funcname)) != 0) { -- if (stat != ERRFUNC) loaderror(L, filename); /* real error */ -- lua_pushfstring(L, "\n\tno module " LUA_QS " in file " LUA_QS, -- name, filename); -- return 1; /* function not found */ -- } -- return 1; --} -- -- - static int loader_preload (lua_State *L) { + if (path == NULL) + luaL_error(L, LUA_QL("package.%s") " must be a string", pname); +@@ -436,16 +430,58 @@ const char *name = luaL_checkstring(L, 1); lua_getfield(L, LUA_ENVIRONINDEX, "preload"); if (!lua_istable(L, -1)) @@ -272,6 +240,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c + } + lua_pop(L, 3); +} ++ static const int sentinel_ = 0; -#define sentinel ((void *)&sentinel_) @@ -279,7 +248,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c static int ll_require (lua_State *L) { -@@ -462,7 +478,7 @@ +@@ -462,7 +498,7 @@ /* else must load it; iterate over available loaders */ lua_getfield(L, LUA_ENVIRONINDEX, "loaders"); if (!lua_istable(L, -1)) @@ -288,7 +257,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c lua_pushliteral(L, ""); /* error message accumulator */ for (i=1; ; i++) { lua_rawgeti(L, -2, i); /* get a loader */ -@@ -490,138 +506,27 @@ +@@ -490,6 +526,8 @@ lua_pushvalue(L, -1); /* extra copy to be returned */ lua_setfield(L, 2, name); /* _LOADED[name] = true */ } @@ -297,82 +266,19 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c return 1; } - /* }====================================================== */ +@@ -502,7 +540,7 @@ + ** 'module' function + ** ======================================================= + */ +- ++ + + static void setfenv (lua_State *L) { + lua_Debug ar; +@@ -569,46 +607,11 @@ + } -- --/* --** {====================================================== --** 'module' function --** ======================================================= --*/ -- -- --static void setfenv (lua_State *L) { -- lua_Debug ar; -- if (lua_getstack(L, 1, &ar) == 0 || -- lua_getinfo(L, "f", &ar) == 0 || /* get calling function */ -- lua_iscfunction(L, -1)) -- luaL_error(L, LUA_QL("module") " not called from a Lua function"); -- lua_pushvalue(L, -2); -- lua_setfenv(L, -2); -- lua_pop(L, 1); --} -- -- --static void dooptions (lua_State *L, int n) { -- int i; -- for (i = 2; i <= n; i++) { -- lua_pushvalue(L, i); /* get option (a function) */ -- lua_pushvalue(L, -2); /* module */ -- lua_call(L, 1, 0); -- } --} -- -- --static void modinit (lua_State *L, const char *modname) { -- const char *dot; -- lua_pushvalue(L, -1); -- lua_setfield(L, -2, "_M"); /* module._M = module */ -- lua_pushstring(L, modname); -- lua_setfield(L, -2, "_NAME"); -- dot = strrchr(modname, '.'); /* look for last dot in module name */ -- if (dot == NULL) dot = modname; -- else dot++; -- /* set _PACKAGE as package name (full module name minus last part) */ -- lua_pushlstring(L, modname, dot - modname); -- lua_setfield(L, -2, "_PACKAGE"); --} -- -- --static int ll_module (lua_State *L) { -- const char *modname = luaL_checkstring(L, 1); -- int loaded = lua_gettop(L) + 1; /* index of _LOADED table */ -- lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); -- lua_getfield(L, loaded, modname); /* get _LOADED[modname] */ -- if (!lua_istable(L, -1)) { /* not found? */ -- lua_pop(L, 1); /* remove previous result */ -- /* try global variable (and create one if it does not exist) */ -- if (luaL_findtable(L, LUA_GLOBALSINDEX, modname, 1) != NULL) -- return luaL_error(L, "name conflict for module " LUA_QS, modname); -- lua_pushvalue(L, -1); -- lua_setfield(L, loaded, modname); /* _LOADED[modname] = new table */ -- } -- /* check whether table already has a _NAME field */ -- lua_getfield(L, -1, "_NAME"); -- if (!lua_isnil(L, -1)) /* is table an initialized module? */ -- lua_pop(L, 1); -- else { /* no; initialize it */ -- lua_pop(L, 1); -- modinit(L, modname); -- } -- lua_pushvalue(L, -1); -- setfenv(L); -- dooptions(L, loaded - 1); -- return 0; --} -- -- -static int ll_seeall (lua_State *L) { - luaL_checktype(L, 1, LUA_TTABLE); - if (!lua_getmetatable(L, 1)) { @@ -386,10 +292,10 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c -} - - --/* }====================================================== */ -- -- -- + /* }====================================================== */ + + + -/* auxiliary mark (for internal use) */ -#define AUXMARK "\1" - @@ -416,21 +322,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c {NULL, NULL} }; - - static const luaL_Reg ll_funcs[] = { -- {"module", ll_module}, - {"require", ll_require}, - {NULL, NULL} - }; - - - static const lua_CFunction loaders[] = -- {loader_preload, loader_Lua, loader_C, loader_Croot, NULL}; -+ {loader_preload, loader_Lua, loader_C, NULL}; - - - LUALIB_API int luaopen_package (lua_State *L) { -@@ -630,12 +535,8 @@ +@@ -630,12 +633,8 @@ luaL_newmetatable(L, "_LOADLIB"); lua_pushcfunction(L, gctm); lua_setfield(L, -2, "__gc"); @@ -445,7 +337,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c lua_pushvalue(L, -1); lua_replace(L, LUA_ENVIRONINDEX); /* create `loaders' table */ -@@ -646,21 +547,17 @@ +@@ -646,21 +645,17 @@ lua_rawseti(L, -2, i+1); } lua_setfield(L, -2, "loaders"); /* put it in field `loaders' */ @@ -477,7 +369,7 @@ diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c - diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c --- /work/lua-5.1.5/src/loslib.c 2008-01-18 08:38:18.000000000 -0800 -+++ lua-5_1_5/src/loslib.c 2015-01-22 12:06:12.578775487 -0800 ++++ lua-5_1_5/src/loslib.c 2015-01-22 13:55:49.917296821 -0800 @@ -33,6 +33,61 @@ return 3; } @@ -610,7 +502,7 @@ diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c } diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h --- /work/lua-5.1.5/src/luaconf.h 2008-02-11 08:25:08.000000000 -0800 -+++ lua-5_1_5/src/luaconf.h 2015-01-22 12:06:12.578775487 -0800 ++++ lua-5_1_5/src/luaconf.h 2015-01-22 13:55:49.917296821 -0800 @@ -1,5 +1,5 @@ /* -** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ @@ -655,7 +547,7 @@ diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h diff -Naur /work/lua-5.1.5/src/lua.h lua-5_1_5/src/lua.h --- /work/lua-5.1.5/src/lua.h 2012-01-13 12:36:20.000000000 -0800 -+++ lua-5_1_5/src/lua.h 2015-01-22 12:06:12.578775487 -0800 ++++ lua-5_1_5/src/lua.h 2015-01-22 13:55:49.917296821 -0800 @@ -341,6 +341,7 @@ LUA_API lua_Hook lua_gethook (lua_State *L); LUA_API int lua_gethookmask (lua_State *L); diff --git a/src/lsb.c b/src/lsb.c index 1b7d97d..b2f9af0 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -31,7 +31,7 @@ static const char* standard_config = "{" "cpath = '%s'," "remove_entries = {" "[''] = { 'collectgarbage', 'coroutine', 'dofile', 'load', 'loadfile'" - ",'loadstring', 'module', 'print'}," + ",'loadstring', 'print'}," "os = {'execute', 'exit', 'remove', 'rename', 'setlocale', 'tmpname'}" "}}"; diff --git a/src/test/lua/errors.lua b/src/test/lua/errors.lua index e9b52b3..badaa0b 100644 --- a/src/test/lua/errors.lua +++ b/src/test/lua/errors.lua @@ -32,6 +32,8 @@ function process(tc) local v = require "../invalid" elseif tc == 10 then local v = require "pathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflow" + elseif tc == 11 then + local v = require "foo.bar" end return 0 end diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index eb0b7b0..66a7439 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -1,5 +1,4 @@ if _PRESERVATION_VERSION and _PRESERVATION_VERSION ~= 0 then return end -_G["_VERSION"] = "Lua 5.1" if _G["dataRef"] == nil then _G["dataRef"] = circular_buffer.new(3, 3, 1) end _G["dataRef"]:set_header(1, "Column_1", "count", "sum") _G["dataRef"]:set_header(2, "Column_2", "count", "sum") @@ -28,6 +27,7 @@ _G["cycleb"]["a"] = {} _G["cycleb"]["a"]["b"] = _G["cycleb"] _G["cycleb"]["a"]["type"] = "cycle a" _G["cycleb"]["type"] = "cycle b" +_G["rates"] = _G["kvp"]["r"] if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end _G["delta"]:set_header(1, "Column_1", "count", "sum") _G["delta"]:fromstring("1 1 2 nan 0 2") @@ -68,5 +68,5 @@ _G["nested"]["cb"]:set_header(4, "Column_4", "count", "sum") _G["nested"]["cb"]:set_header(5, "Column_5", "count", "sum") _G["nested"]["cb"]:set_header(6, "Column_6", "count", "sum") _G["nested"]["cb"]:fromstring("1 1 nan nan nan nan nan nan nan nan nan nan nan nan") -_G["rates"] = _G["kvp"]["r"] _G["count"] = 0 +_G["_VERSION"] = "Lua 5.1" diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 91b4dfa..5293293 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -735,6 +735,11 @@ static char* test_errors() #endif , "process() invalid module name '../invalid'" , "process() lua/errors.lua:34: module 'pathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpa" +#ifdef _WIN32 + , "process() lua/errors.lua:36: module 'foo.bar' not found:\n\tno file 'lua\\foo\\bar.lua'\n\tno file 'lua\\foo\\bar.dll'\n\tno file 'lua\\foo.dll'" +#else + , "process() lua/errors.lua:36: module 'foo.bar' not found:\n\tno file 'lua/foo/bar.lua'\n\tno file 'lua/foo/bar.so'\n\tno file 'lua/foo.so'" +#endif , NULL }; From bbe24005146c35e1b58a2e01016f11c12d9aa5ad Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 30 Jan 2015 08:56:35 -0800 Subject: [PATCH 036/235] Move lpeg, cjson, and struct into shared modules --- cmake/FindLua.cmake | 125 +++++++++ cmake/externals.cmake | 28 +- cmake/lpeg-0_12.patch | 53 ---- cmake/lua-5_1_5.patch | 558 ---------------------------------------- docs/sandbox_api.md | 13 +- src/CMakeLists.txt | 40 ++- src/lsb.c | 15 +- src/struct.c | 420 ------------------------------ src/test/CMakeLists.txt | 20 +- 9 files changed, 181 insertions(+), 1091 deletions(-) create mode 100644 cmake/FindLua.cmake delete mode 100644 cmake/lpeg-0_12.patch delete mode 100644 cmake/lua-5_1_5.patch delete mode 100644 src/struct.c diff --git a/cmake/FindLua.cmake b/cmake/FindLua.cmake new file mode 100644 index 0000000..e47f1cb --- /dev/null +++ b/cmake/FindLua.cmake @@ -0,0 +1,125 @@ +# Locate Lua library +# This module defines +# LUA_EXECUTABLE, if found +# LUA_FOUND, if false, do not try to link to Lua +# LUA_LIBRARIES +# LUA_INCLUDE_DIR, where to find lua.h +# LUA_VERSION_STRING, the version of Lua found (since CMake 2.8.8) +# +# Note that the expected include convention is +# #include "lua.h" +# and not +# #include +# This is because, the lua location is not standardized and may exist +# in locations other than lua/ + +#============================================================================= +# Copyright 2007-2009 Kitware, Inc. +# Modified to support Lua 5.2 by LuaDist 2012 +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) +# +# The required version of Lua can be specified using the +# standard syntax, e.g. FIND_PACKAGE(Lua 5.1) +# Otherwise the module will search for any available Lua implementation + + +if (LUA_SANDBOX_INCLUDE) + set(LUA_INCLUDE_DIR ${EP_BASE}/include) + set(LIB_PATH ${EP_BASE}/lib) + find_library(LUA_LIBRARY lua PATHS ${LIB_PATH} NO_DEFAULT_PATH) +else() + # Always search for non-versioned lua first (recommended) + SET(_POSSIBLE_LUA_INCLUDE include include/lua) + SET(_POSSIBLE_LUA_EXECUTABLE lua) + SET(_POSSIBLE_LUA_LIBRARY lua) + + # Determine possible naming suffixes (there is no standard for this) + IF(Lua_FIND_VERSION_MAJOR AND Lua_FIND_VERSION_MINOR) + SET(_POSSIBLE_SUFFIXES "${Lua_FIND_VERSION_MAJOR}${Lua_FIND_VERSION_MINOR}" "${Lua_FIND_VERSION_MAJOR}.${Lua_FIND_VERSION_MINOR}" "-${Lua_FIND_VERSION_MAJOR}.${Lua_FIND_VERSION_MINOR}") + ELSE(Lua_FIND_VERSION_MAJOR AND Lua_FIND_VERSION_MINOR) + SET(_POSSIBLE_SUFFIXES "52" "5.2" "-5.2" "51" "5.1" "-5.1") + ENDIF(Lua_FIND_VERSION_MAJOR AND Lua_FIND_VERSION_MINOR) + + # Set up possible search names and locations + FOREACH(_SUFFIX ${_POSSIBLE_SUFFIXES}) + LIST(APPEND _POSSIBLE_LUA_INCLUDE "include/lua${_SUFFIX}") + LIST(APPEND _POSSIBLE_LUA_EXECUTABLE "lua${_SUFFIX}") + LIST(APPEND _POSSIBLE_LUA_LIBRARY "lua${_SUFFIX}") + ENDFOREACH(_SUFFIX) + + # Find the lua executable + FIND_PROGRAM(LUA_EXECUTABLE + NAMES ${_POSSIBLE_LUA_EXECUTABLE} + ) + + # Find the lua header + FIND_PATH(LUA_INCLUDE_DIR lua.h + HINTS + $ENV{LUA_DIR} + PATH_SUFFIXES ${_POSSIBLE_LUA_INCLUDE} + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw # Fink + /opt/local # DarwinPorts + /opt/csw # Blastwave + /opt + ) + + # Find the lua library + FIND_LIBRARY(LUA_LIBRARY + NAMES ${_POSSIBLE_LUA_LIBRARY} + HINTS + $ENV{LUA_DIR} + PATH_SUFFIXES lib64 lib + PATHS + ~/Library/Frameworks + /Library/Frameworks + /usr/local + /usr + /sw + /opt/local + /opt/csw + /opt + ) +endif() + +IF(LUA_LIBRARY) + # include the math library for Unix + IF(UNIX AND NOT APPLE) + FIND_LIBRARY(LUA_MATH_LIBRARY m) + SET( LUA_LIBRARIES "${LUA_LIBRARY};${LUA_MATH_LIBRARY}" CACHE STRING "Lua Libraries") + # For Windows and Mac, don't need to explicitly include the math library + ELSE(UNIX AND NOT APPLE) + SET( LUA_LIBRARIES "${LUA_LIBRARY}" CACHE STRING "Lua Libraries") + ENDIF(UNIX AND NOT APPLE) +ENDIF(LUA_LIBRARY) + +# Determine Lua version +IF(LUA_INCLUDE_DIR AND EXISTS "${LUA_INCLUDE_DIR}/lua.h") + FILE(STRINGS "${LUA_INCLUDE_DIR}/lua.h" lua_version_str REGEX "^#define[ \t]+LUA_RELEASE[ \t]+\"Lua .+\"") + + STRING(REGEX REPLACE "^#define[ \t]+LUA_RELEASE[ \t]+\"Lua ([^\"]+)\".*" "\\1" LUA_VERSION_STRING "${lua_version_str}") + UNSET(lua_version_str) +ENDIF() + +INCLUDE(FindPackageHandleStandardArgs) +# handle the QUIETLY and REQUIRED arguments and set LUA_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(Lua + REQUIRED_VARS LUA_LIBRARIES LUA_INCLUDE_DIR + VERSION_VAR LUA_VERSION_STRING) + +MARK_AS_ADVANCED(LUA_INCLUDE_DIR LUA_LIBRARIES LUA_LIBRARY LUA_MATH_LIBRARY LUA_EXECUTABLE) + diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 9a217b6..802f18b 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -57,9 +57,8 @@ else() set(LUA_PROJECT "lua-5_1_5") externalproject_add( ${LUA_PROJECT} - URL http://www.lua.org/ftp/lua-5.1.5.tar.gz - URL_MD5 2e115fe26e435e33b0d5c022e4490567 - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/lua-5_1_5.patch + GIT_REPOSITORY https://github.com/trink/lua.git + GIT_TAG 63a4f245d214cdb61e7ae1ce2595e5c1ce3c7f51 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -68,9 +67,8 @@ include_directories(${LUA_INCLUDE_DIR}) externalproject_add( lua_lpeg - URL http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-0.12.tar.gz - URL_MD5 4abb3c28cd8b6565c6a65e88f06c9162 - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/lpeg-0_12.patch + GIT_REPOSITORY https://github.com/LuaDist/lpeg.git + GIT_TAG baf0dc90b9278360be719dbfb8e56d34ce3c61bd CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -79,16 +77,26 @@ add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG 1ebc6061435a45ce23bf7d26d7ed04db995a8ba9 + GIT_TAG af793dbfd83d9be69835faf8b88702ebb74547ed CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) add_dependencies(lua_cjson ${LUA_PROJECT}) +externalproject_add( + lua_struct + GIT_REPOSITORY https://github.com/LuaDist/struct.git + GIT_TAG 1ba845fbfaa7a6e4d96088938b4574d194f13f6a + UPDATE_COMMAND cmake -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_DIR ${EP_BASE} +) +add_dependencies(lua_struct ${LUA_PROJECT}) + externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG b264c84cca536f20a41a9950556383e8dc5fd2ae + GIT_TAG ba756520a51592612c9709ebdfb99392f97eb51f CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -97,7 +105,7 @@ add_dependencies(lua_bloom_filter ${LUA_PROJECT}) externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG 5e7726d70bd5cf6e61e1faccae2a2fabf5e9bf3d + GIT_TAG ca4b0271bd741b857527f8315387142beac9cef9 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -106,7 +114,7 @@ add_dependencies(lua_circular_buffer ${LUA_PROJECT}) externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG e5d3399b0d0a6913d18cc5a0a33326391363b936 + GIT_TAG c9e3a7310b6e65a1ed54bc073ca6b836edb84cd5 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/cmake/lpeg-0_12.patch b/cmake/lpeg-0_12.patch deleted file mode 100644 index d6c15e0..0000000 --- a/cmake/lpeg-0_12.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff -Naur lpeg-0.12.orig/lptree.c lpeg-0_12/lptree.c ---- lpeg-0.12.orig/lptree.c Fri Apr 12 09:31:19 2013 -+++ lpeg-0_12/lptree.c Fri Aug 2 10:15:43 2013 -@@ -1217,8 +1217,7 @@ - }; - - --int luaopen_lpeg (lua_State *L); --int luaopen_lpeg (lua_State *L) { -+LUALIB_API int luaopen_lpeg (lua_State *L) { - luaL_newmetatable(L, PATTERN_T); - lua_pushnumber(L, MAXBACK); /* initialize maximum backtracking */ - lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX); -diff -Naur lpeg-0.12.orig/CMakeLists.txt lpeg-0.12/CMakeLists.txt ---- lpeg-0.12.orig/CMakeLists.txt Wed Dec 31 16:00:00 1969 -+++ lpeg-0.12.orig/CMakeLists.txt Fri Aug 2 09:26:42 2013 -@@ -0,0 +1,36 @@ -+# This Source Code Form is subject to the terms of the Mozilla Public -+# License, v. 2.0. If a copy of the MPL was not distributed with this -+# file, You can obtain one at http://mozilla.org/MPL/2.0/. -+ -+cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -+project(lpeg C) -+ -+set(LPEG_SOURCE -+lpcap.c -+lpcode.c -+lptree.c -+lpvm.c -+) -+ -+include_directories("${EP_BASE}/include") -+if (WIN32) -+ add_library(lpeg SHARED ${LPEG_SOURCE}) -+ add_definitions(-DLUA_BUILD_AS_DLL -DLUA_CORE) -+ if(MSVC) -+ target_link_libraries(lpeg "${EP_BASE}/lib/lua.lib") -+ else(MINGW) -+ set_target_properties(lpeg PROPERTIES LINK_FLAGS -s) -+ if (ADDRESS_MODEL EQUAL 32) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") -+ set_target_properties(lpeg PROPERTIES LINK_FLAGS "-s -m32") -+ endif() -+ target_link_libraries(lpeg "${EP_BASE}/lib/liblua.dll") -+ else() -+ message(FATAL_ERROR "Only MSVC and MinGW compilers are supported on Windows") -+ endif() -+else() -+ add_library(lpeg STATIC ${LPEG_SOURCE}) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -Wall -fPIC") -+endif() -+ -+install(TARGETS lpeg DESTINATION lib) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch deleted file mode 100644 index fb2ec20..0000000 --- a/cmake/lua-5_1_5.patch +++ /dev/null @@ -1,558 +0,0 @@ -diff -Naur /work/lua-5.1.5/CMakeLists.txt lua-5_1_5/CMakeLists.txt ---- /work/lua-5.1.5/CMakeLists.txt 1969-12-31 16:00:00.000000000 -0800 -+++ lua-5_1_5/CMakeLists.txt 2015-01-22 13:55:49.917296821 -0800 -@@ -0,0 +1,64 @@ -+# This Source Code Form is subject to the terms of the Mozilla Public -+# License, v. 2.0. If a copy of the MPL was not distributed with this -+# file, You can obtain one at http://mozilla.org/MPL/2.0/. -+ -+cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -+project(lua C) -+ -+set(LUA_SOURCE -+src/lapi.c -+src/lauxlib.c -+src/lbaselib.c -+src/lcode.c -+src/ldblib.c -+src/ldebug.c -+src/ldo.c -+src/ldump.c -+src/lfunc.c -+src/lgc.c -+src/linit.c -+src/liolib.c -+src/llex.c -+src/lmathlib.c -+src/lmem.c -+src/loadlib.c -+src/lobject.c -+src/lopcodes.c -+src/loslib.c -+src/lparser.c -+src/lstate.c -+src/lstring.c -+src/lstrlib.c -+src/ltable.c -+src/ltablib.c -+src/ltm.c -+src/lundump.c -+src/lvm.c -+src/lzio.c -+) -+ -+set(CMAKE_INSTALL_PREFIX ${EP_BASE}) -+if (WIN32) -+ add_library(lua SHARED ${LUA_SOURCE}) -+ add_definitions(-DLUA_WIN) -+ add_definitions(-DLUA_BUILD_AS_DLL) -+ if (MINGW) -+ set_target_properties(lua PROPERTIES LINK_FLAGS -s) -+ if (ADDRESS_MODEL EQUAL 32) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") -+ set_target_properties(lua PROPERTIES LINK_FLAGS "-s -m32") -+ endif() -+ endif() -+elseif(APPLE) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") -+ add_library(lua STATIC ${LUA_SOURCE}) -+ add_definitions(-DLUA_USE_MACOSX) -+ target_link_libraries(lua -lreadline) -+else() -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") -+ add_library(lua STATIC ${LUA_SOURCE}) -+ add_definitions(-DLUA_USE_LINUX) -+endif() -+ -+install(TARGETS lua DESTINATION lib) -+install(FILES src/lua.h src/luaconf.h src/lualib.h src/lauxlib.h DESTINATION include) -diff -Naur /work/lua-5.1.5/src/ldebug.c lua-5_1_5/src/ldebug.c ---- /work/lua-5.1.5/src/ldebug.c 2008-05-08 09:56:26.000000000 -0700 -+++ lua-5_1_5/src/ldebug.c 2015-01-22 13:55:49.917296821 -0800 -@@ -80,6 +80,10 @@ - return L->basehookcount; - } - -+LUA_API int lua_gethookcountremaining (lua_State *L) { -+ return L->hookcount; -+} -+ - - LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar) { - int status; -diff -Naur /work/lua-5.1.5/src/loadlib.c lua-5_1_5/src/loadlib.c ---- /work/lua-5.1.5/src/loadlib.c 2009-09-09 06:17:16.000000000 -0700 -+++ lua-5_1_5/src/loadlib.c 2015-01-26 10:00:16.095518127 -0800 -@@ -23,24 +23,24 @@ - - - /* prefix for open functions in C libraries */ --#define LUA_POF "luaopen_" -+#define LUA_POF "luaopen_" - - /* separator for open functions in C libraries */ --#define LUA_OFSEP "_" -+#define LUA_OFSEP "_" - - --#define LIBPREFIX "LOADLIB: " -+#define LIBPREFIX "LOADLIB: " - --#define POF LUA_POF --#define LIB_FAIL "open" -+#define POF LUA_POF -+#define LIB_FAIL "open" - - - /* error codes for ll_loadfunc */ --#define ERRLIB 1 --#define ERRFUNC 2 -+#define ERRLIB 1 -+#define ERRFUNC 2 - --#define setprogdir(L) ((void)0) - -+#define LSB_CONFIG "lsb_config" - - static void ll_unloadlib (void *lib); - static void *ll_load (lua_State *L, const char *path); -@@ -91,24 +91,6 @@ - - #include - -- --#undef setprogdir -- --static void setprogdir (lua_State *L) { -- char buff[MAX_PATH + 1]; -- char *lb; -- DWORD nsize = sizeof(buff)/sizeof(char); -- DWORD n = GetModuleFileNameA(NULL, buff, nsize); -- if (n == 0 || n == nsize || (lb = strrchr(buff, '\\')) == NULL) -- luaL_error(L, "unable to get ModuleFileName"); -- else { -- *lb = '\0'; -- luaL_gsub(L, lua_tostring(L, -1), LUA_EXECDIR, buff); -- lua_remove(L, -2); /* remove original string */ -- } --} -- -- - static void pusherror (lua_State *L) { - int error = GetLastError(); - char buffer[128]; -@@ -153,7 +135,7 @@ - - /* Mac appends a `_' before C function names */ - #undef POF --#define POF "_" LUA_POF -+#define POF "_" LUA_POF - - - static void pusherror (lua_State *L) { -@@ -230,10 +212,10 @@ - */ - - #undef LIB_FAIL --#define LIB_FAIL "absent" -+#define LIB_FAIL "absent" - - --#define DLMSG "dynamic libraries not enabled; check your Lua installation" -+#define DLMSG "dynamic libraries not enabled; check your Lua installation" - - - static void ll_unloadlib (void *lib) { -@@ -351,8 +333,20 @@ - static const char *findfile (lua_State *L, const char *name, - const char *pname) { - const char *path; -+ int i = 0; -+ while (name[i]) { -+ if (!isalnum(name[i]) && name[i] != '_' && name[i] != '.') { -+ luaL_error(L, "invalid module name '%s'", name); -+ return NULL; // never reached, just silences the compiler -+ } -+ ++i; -+ } -+ lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); -+ if (lua_type(L, -1) != LUA_TTABLE) { -+ return NULL; -+ } - name = luaL_gsub(L, name, ".", LUA_DIRSEP); -- lua_getfield(L, LUA_ENVIRONINDEX, pname); -+ lua_getfield(L, -2, pname); - path = lua_tostring(L, -1); - if (path == NULL) - luaL_error(L, LUA_QL("package.%s") " must be a string", pname); -@@ -436,16 +430,58 @@ - const char *name = luaL_checkstring(L, 1); - lua_getfield(L, LUA_ENVIRONINDEX, "preload"); - if (!lua_istable(L, -1)) -- luaL_error(L, LUA_QL("package.preload") " must be a table"); -+ luaL_error(L, LUA_QL("preload") " must be a table"); - lua_getfield(L, -1, name); -- if (lua_isnil(L, -1)) /* not found? */ -- lua_pushfstring(L, "\n\tno field package.preload['%s']", name); - return 1; - } - -+// If necessary, add an empty metatable to flag the table as non-data -+// during preservation. -+static void add_empty_metatable(lua_State* L) -+{ -+ if (lua_getmetatable(L, -1) == 0) { -+ lua_newtable(L); -+ lua_setmetatable(L, -2); -+ } else { -+ lua_pop(L, 1); -+ } -+} -+ -+ -+static void remove_entries(lua_State* L, const char* name) -+{ -+ lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); -+ if (lua_type(L, -1) != LUA_TTABLE) { -+ lua_pop(L, 1); -+ return; -+ } -+ lua_getfield(L, -1, "remove_entries"); -+ if (lua_type(L, -1) != LUA_TTABLE) { -+ lua_pop(L, 2); -+ return; -+ } -+ lua_getfield(L, -1, name); -+ if (lua_type(L, -1) != LUA_TTABLE) { -+ lua_pop(L, 3); -+ return; -+ } -+ int n = 1; -+ while (1) { -+ lua_rawgeti(L, -1, n); -+ if (lua_type(L, -1) == LUA_TNIL) { -+ lua_pop(L, 1); -+ break; -+ } -+ lua_pushnil(L); -+ lua_settable(L, -6); -+ ++n; -+ } -+ lua_pop(L, 3); -+} -+ - - static const int sentinel_ = 0; --#define sentinel ((void *)&sentinel_) -+#define sentinel ((void *)&sentinel_) - - - static int ll_require (lua_State *L) { -@@ -462,7 +498,7 @@ - /* else must load it; iterate over available loaders */ - lua_getfield(L, LUA_ENVIRONINDEX, "loaders"); - if (!lua_istable(L, -1)) -- luaL_error(L, LUA_QL("package.loaders") " must be a table"); -+ luaL_error(L, LUA_QL("loaders") " must be a table"); - lua_pushliteral(L, ""); /* error message accumulator */ - for (i=1; ; i++) { - lua_rawgeti(L, -2, i); /* get a loader */ -@@ -490,6 +526,8 @@ - lua_pushvalue(L, -1); /* extra copy to be returned */ - lua_setfield(L, 2, name); /* _LOADED[name] = true */ - } -+ add_empty_metatable(L); -+ remove_entries(L, name); - return 1; - } - -@@ -502,7 +540,7 @@ - ** 'module' function - ** ======================================================= - */ -- -+ - - static void setfenv (lua_State *L) { - lua_Debug ar; -@@ -569,46 +607,11 @@ - } - - --static int ll_seeall (lua_State *L) { -- luaL_checktype(L, 1, LUA_TTABLE); -- if (!lua_getmetatable(L, 1)) { -- lua_createtable(L, 0, 1); /* create new metatable */ -- lua_pushvalue(L, -1); -- lua_setmetatable(L, 1); -- } -- lua_pushvalue(L, LUA_GLOBALSINDEX); -- lua_setfield(L, -2, "__index"); /* mt.__index = _G */ -- return 0; --} -- -- - /* }====================================================== */ - - - --/* auxiliary mark (for internal use) */ --#define AUXMARK "\1" -- --static void setpath (lua_State *L, const char *fieldname, const char *envname, -- const char *def) { -- const char *path = getenv(envname); -- if (path == NULL) /* no environment variable? */ -- lua_pushstring(L, def); /* use default */ -- else { -- /* replace ";;" by ";AUXMARK;" and then AUXMARK by default path */ -- path = luaL_gsub(L, path, LUA_PATHSEP LUA_PATHSEP, -- LUA_PATHSEP AUXMARK LUA_PATHSEP); -- luaL_gsub(L, path, AUXMARK, def); -- lua_remove(L, -2); -- } -- setprogdir(L); -- lua_setfield(L, -2, fieldname); --} -- -- - static const luaL_Reg pk_funcs[] = { -- {"loadlib", ll_loadlib}, -- {"seeall", ll_seeall}, - {NULL, NULL} - }; - -@@ -630,12 +633,8 @@ - luaL_newmetatable(L, "_LOADLIB"); - lua_pushcfunction(L, gctm); - lua_setfield(L, -2, "__gc"); -- /* create `package' table */ -- luaL_register(L, LUA_LOADLIBNAME, pk_funcs); --#if defined(LUA_COMPAT_LOADLIB) -- lua_getfield(L, -1, "loadlib"); -- lua_setfield(L, LUA_GLOBALSINDEX, "loadlib"); --#endif -+ /* create `package' environment table (not exposed to Lua) */ -+ lua_newtable(L); - lua_pushvalue(L, -1); - lua_replace(L, LUA_ENVIRONINDEX); - /* create `loaders' table */ -@@ -646,21 +645,17 @@ - lua_rawseti(L, -2, i+1); - } - lua_setfield(L, -2, "loaders"); /* put it in field `loaders' */ -- setpath(L, "path", LUA_PATH, LUA_PATH_DEFAULT); /* set field `path' */ -- setpath(L, "cpath", LUA_CPATH, LUA_CPATH_DEFAULT); /* set field `cpath' */ -- /* store config information */ -- lua_pushliteral(L, LUA_DIRSEP "\n" LUA_PATHSEP "\n" LUA_PATH_MARK "\n" -- LUA_EXECDIR "\n" LUA_IGMARK); -- lua_setfield(L, -2, "config"); -- /* set field `loaded' */ -- luaL_findtable(L, LUA_REGISTRYINDEX, "_LOADED", 2); -- lua_setfield(L, -2, "loaded"); -+ - /* set field `preload' */ -- lua_newtable(L); -+ luaL_findtable(L, LUA_REGISTRYINDEX, "_PRELOADED", 0); - lua_setfield(L, -2, "preload"); -+ -+ /* open lib into global table */ - lua_pushvalue(L, LUA_GLOBALSINDEX); -- luaL_register(L, NULL, ll_funcs); /* open lib into global table */ -+ luaL_register(L, NULL, ll_funcs); - lua_pop(L, 1); -+ -+ /* create an empty `package' table */ -+ luaL_register(L, LUA_LOADLIBNAME, pk_funcs); - return 1; /* return 'package' table */ - } -- -diff -Naur /work/lua-5.1.5/src/loslib.c lua-5_1_5/src/loslib.c ---- /work/lua-5.1.5/src/loslib.c 2008-01-18 08:38:18.000000000 -0800 -+++ lua-5_1_5/src/loslib.c 2015-01-22 13:55:49.917296821 -0800 -@@ -33,6 +33,61 @@ - return 3; - } - } -+/* -+** list of valid conversion specifiers for the 'strftime' function -+*/ -+#if !defined(LUA_STRFTIMEOPTIONS) -+ -+#if !defined(LUA_USE_POSIX) -+#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYzZ%", "" } -+#else -+#define LUA_STRFTIMEOPTIONS \ -+ { "aAbBcCdDeFgGhHIjklmMnprRSstTuUVwWxXyYzZ%", "" \ -+ "", "E", "cCxXyY", \ -+ "O", "deHImMSuUVwWy" } -+#endif -+ -+#endif -+ -+ -+ -+/* -+** By default, Lua uses tmpnam except when POSIX is available, where it -+** uses mkstemp. -+*/ -+#if defined(LUA_USE_MKSTEMP) -+#include -+#define LUA_TMPNAMBUFSIZE 32 -+#define lua_tmpnam(b,e) { \ -+ strcpy(b, "/tmp/lua_XXXXXX"); \ -+ e = mkstemp(b); \ -+ if (e != -1) close(e); \ -+ e = (e == -1); } -+ -+#elif !defined(lua_tmpnam) -+ -+#define LUA_TMPNAMBUFSIZE L_tmpnam -+#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } -+ -+#endif -+ -+ -+/* -+** By default, Lua uses gmtime/localtime, except when POSIX is available, -+** where it uses gmtime_r/localtime_r -+*/ -+#if defined(LUA_USE_GMTIME_R) -+ -+#define l_gmtime(t,r) gmtime_r(t,r) -+#define l_localtime(t,r) localtime_r(t,r) -+ -+#elif !defined(l_gmtime) -+ -+#define l_gmtime(t,r) ((void)r, gmtime(t)) -+#define l_localtime(t,r) ((void)r, localtime(t)) -+ -+#endif -+ - - - static int os_execute (lua_State *L) { -@@ -121,16 +176,40 @@ - } - - -+static const char *checkoption (lua_State *L, const char *conv, char *buff) { -+ static const char *const options[] = LUA_STRFTIMEOPTIONS; -+ unsigned int i; -+ for (i = 0; i < sizeof(options)/sizeof(options[0]); i += 2) { -+ if (*conv != '\0' && strchr(options[i], *conv) != NULL) { -+ buff[1] = *conv; -+ if (*options[i + 1] == '\0') { /* one-char conversion specifier? */ -+ buff[2] = '\0'; /* end buffer */ -+ return conv + 1; -+ } -+ else if (*(conv + 1) != '\0' && -+ strchr(options[i + 1], *(conv + 1)) != NULL) { -+ buff[2] = *(conv + 1); /* valid two-char conversion specifier */ -+ buff[3] = '\0'; /* end buffer */ -+ return conv + 2; -+ } -+ } -+ } -+ luaL_argerror(L, 1, -+ lua_pushfstring(L, "invalid conversion specifier '%%%s'", conv)); -+ return conv; /* to avoid warnings */ -+} -+ -+ - static int os_date (lua_State *L) { - const char *s = luaL_optstring(L, 1, "%c"); - time_t t = luaL_opt(L, (time_t)luaL_checknumber, 2, time(NULL)); -- struct tm *stm; -+ struct tm tmr, *stm; - if (*s == '!') { /* UTC? */ -- stm = gmtime(&t); -+ stm = l_gmtime(&t, &tmr); - s++; /* skip `!' */ - } - else -- stm = localtime(&t); -+ stm = l_localtime(&t, &tmr); - if (stm == NULL) /* invalid date? */ - lua_pushnil(L); - else if (strcmp(s, "*t") == 0) { -@@ -146,17 +225,17 @@ - setboolfield(L, "isdst", stm->tm_isdst); - } - else { -- char cc[3]; -+ char cc[4]; - luaL_Buffer b; -- cc[0] = '%'; cc[2] = '\0'; -+ cc[0] = '%'; - luaL_buffinit(L, &b); -- for (; *s; s++) { -- if (*s != '%' || *(s + 1) == '\0') /* no conversion specifier? */ -- luaL_addchar(&b, *s); -+ while (*s) { -+ if (*s != '%') /* no conversion specifier? */ -+ luaL_addchar(&b, *s++); - else { - size_t reslen; - char buff[200]; /* should be big enough for any conversion result */ -- cc[1] = *(++s); -+ s = checkoption(L, s + 1, cc); - reslen = strftime(buff, sizeof(buff), cc, stm); - luaL_addlstring(&b, buff, reslen); - } -diff -Naur /work/lua-5.1.5/src/luaconf.h lua-5_1_5/src/luaconf.h ---- /work/lua-5.1.5/src/luaconf.h 2008-02-11 08:25:08.000000000 -0800 -+++ lua-5_1_5/src/luaconf.h 2015-01-22 13:55:49.917296821 -0800 -@@ -1,5 +1,5 @@ - /* --** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ -+** $Id: luaconf.h,v 1.176.1.1 2013/04/12 18:48:47 roberto Exp $ - ** Configuration file for Lua - ** See Copyright Notice in lua.h - */ -@@ -33,14 +33,26 @@ - #define LUA_WIN - #endif - -+#if defined(LUA_WIN) -+#define LUA_DL_DLL -+#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ -+#endif -+ - #if defined(LUA_USE_LINUX) - #define LUA_USE_POSIX - #define LUA_USE_DLOPEN /* needs an extra library: -ldl */ - #define LUA_USE_READLINE /* needs some extra libraries */ -+#define LUA_USE_STRTODHEX /* assume 'strtod' handles hex formats */ -+#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ -+#define LUA_USE_LONGLONG /* assume support for long long */ - #endif - - #if defined(LUA_USE_MACOSX) - #define LUA_USE_POSIX -+#define LUA_USE_DLOPEN /* does not need -ldl */ -+#define LUA_USE_READLINE /* needs an extra library: -lreadline */ -+#define LUA_USE_STRTODHEX /* assume 'strtod' handles hex formats */ -+#define LUA_USE_AFORMAT /* assume 'printf' handles 'aA' specifiers */ - #define LUA_DL_DYLD /* does not need extra library */ - #endif - -@@ -56,6 +68,7 @@ - #define LUA_USE_ISATTY - #define LUA_USE_POPEN - #define LUA_USE_ULONGJMP -+#define LUA_USE_GMTIME_R - #endif - - -diff -Naur /work/lua-5.1.5/src/lua.h lua-5_1_5/src/lua.h ---- /work/lua-5.1.5/src/lua.h 2012-01-13 12:36:20.000000000 -0800 -+++ lua-5_1_5/src/lua.h 2015-01-22 13:55:49.917296821 -0800 -@@ -341,6 +341,7 @@ - LUA_API lua_Hook lua_gethook (lua_State *L); - LUA_API int lua_gethookmask (lua_State *L); - LUA_API int lua_gethookcount (lua_State *L); -+LUA_API int lua_gethookcountremaining (lua_State *L); - - - struct lua_Debug { diff --git a/docs/sandbox_api.md b/docs/sandbox_api.md index 85fa4f7..d4af47d 100644 --- a/docs/sandbox_api.md +++ b/docs/sandbox_api.md @@ -22,14 +22,14 @@ Functions exposed to Lua by the core sandbox ============================================ **require(libraryName)** -By default only the base library is loaded additional libraries must be explicitly specified. +By default only the base library is loaded additional libraries must be loaded with require(). *Arguments* - libraryName (string) - [base library(http://www.lua.org/manual/5.1/manual.html#5.1) - - The standard require() function is overridden by this version. - - Disabled functions: collectgarbage, coroutine, dofile, load, loadfile, loadstring, module, print. + - The require() function has been modified to not expose any of the package table to the sandbox. + - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, print. - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) With the following modifications: @@ -42,14 +42,16 @@ By default only the base library is loaded additional libraries must be explicit - The encode_keep_buffer() function has been disabled (the buffer is always reused). - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library + - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG - [math](http://www.lua.org/manual/5.1/manual.html#5.6) - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. - - Disabled functions: execute, exit, remove, rename, setlocale, tmpname. + - Disabled functions (default): execute, exit, remove, rename, setlocale, tmpname. + - [strict](http://www.lua.org/extras/5.2/strict.lua) Checks for use of undeclared global variables. - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - [struct](http://www.inf.puc-rio.br/~roberto/struct/) - [table](http://www.lua.org/manual/5.1/manual.html#5.5) - - _user provided_ + - _user provided_ (lua, so/dll) *Return* - a table - For non user provided libraries the table is also globally registered @@ -97,6 +99,7 @@ C Functions Exposed to Lua (optional arguments have the same restriction as output). - void **write_message** (table) - serializes the Lua table into a Heka protobuf message and captures the output. + - the table must use the following [schema](https://hekad.readthedocs.org/en/latest/sandbox/index.html#sample-lua-message-structure). [Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_lua_sandbox.c) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e906899..4671edd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,54 +7,46 @@ lsb.c lsb_output.c lsb_serialize.c lsb_serialize_protobuf.c -struct.c ) +set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) +set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) +set ( CMAKE_INSTALL_RPATH "$ORIGIN/../lib" CACHE STRING "" FORCE ) +set ( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE STRING "" FORCE ) +set ( CMAKE_INSTALL_NAME_DIR "@executable_path/../lib" CACHE STRING "" FORCE ) + add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) -add_dependencies(luasandbox ${LUA_PROJECT} lua_lpeg lua_cjson) +add_dependencies(lua_cjson ${LUA_PROJECT}) +add_dependencies(lua_lpeg ${LUA_PROJECT}) + add_dependencies(lua_bloom_filter luasandbox) add_dependencies(lua_circular_buffer luasandbox) add_dependencies(lua_hyperloglog luasandbox) if(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) - set(LUA_SANDBOX_LIBS - "${EP_BASE}/lib/lua.lib" - "${EP_BASE}/lib/lpeg.lib" - "${EP_BASE}/lib/cjson.lib" - ) + set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/lua.lib") elseif(MINGW) add_definitions(-D_MINGW) - set(LUA_SANDBOX_LIBS - "${EP_BASE}/lib/liblua.dll" - "${EP_BASE}/lib/liblpeg.dll" - "${EP_BASE}/lib/libcjson.dll" - ) + set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/liblua.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) if (ADDRESS_MODEL EQUAL 32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") set_target_properties(luasandbox PROPERTIES LINK_FLAGS "-s -m32") endif() else() - if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") endif() - - set(LUA_SANDBOX_LIBS - "${EP_BASE}/lib/liblua.a" - "${EP_BASE}/lib/liblpeg.a" - "${EP_BASE}/lib/libcjson.a" - ${LINK_DL} -lm - ) + set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/liblua.so" ${LINK_DL} -lm) endif() target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) install(TARGETS luasandbox DESTINATION lib) -install(DIRECTORY "${EP_BASE}/modules/" DESTINATION modules) -install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib) -install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) -install(DIRECTORY "${LUA_INCLUDE_DIR}/" DESTINATION include) +install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib PATTERN "lua" EXCLUDE) +install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION modules) install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) add_subdirectory(test) diff --git a/src/lsb.c b/src/lsb.c index b2f9af0..3eefef1 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -37,10 +37,6 @@ static const char* standard_config = "{" static jmp_buf g_jbuf; -LUALIB_API int luaopen_struct(lua_State* L); -LUALIB_API int luaopen_cjson(lua_State* L); -LUALIB_API int luaopen_lpeg(lua_State* L); - static const luaL_Reg preload_module_list[] = { { "", luaopen_base }, { LUA_TABLIBNAME, luaopen_table }, @@ -48,9 +44,6 @@ static const luaL_Reg preload_module_list[] = { { LUA_OSLIBNAME, luaopen_os }, { LUA_STRLIBNAME, luaopen_string }, { LUA_MATHLIBNAME, luaopen_math }, - { "struct", luaopen_struct }, - { "cjson", luaopen_cjson }, - { "lpeg", luaopen_lpeg }, { NULL, NULL } }; @@ -318,7 +311,6 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) #endif preload_modules(lsb->lua); - // load package module lua_pushcfunction(lsb->lua, luaopen_package); lua_pushstring(lsb->lua, LUA_LOADLIBNAME); @@ -336,7 +328,12 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) return 1; } lua_pushstring(lsb->lua, ""); - lua_call(lsb->lua, 1, 0); + if (lua_pcall(lsb->lua, 1, 0, 0)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_init %s", lua_tostring(lsb->lua, -1)); + lsb_terminate(lsb, NULL); + return 1; + } lua_pushlightuserdata(lsb->lua, (void*)lsb); lua_pushcclosure(lsb->lua, &output, 1); diff --git a/src/struct.c b/src/struct.c deleted file mode 100644 index 339df2f..0000000 --- a/src/struct.c +++ /dev/null @@ -1,420 +0,0 @@ -/* -** {====================================================== -** Library for packing/unpacking structures. -** $Id: struct.c,v 1.4 2012/07/04 18:54:29 roberto Exp $ -** See Copyright Notice at the end of this file -** ======================================================= -*/ -/* -** Valid formats: -** > - big endian -** < - little endian -** ![num] - alignment -** x - pading -** b/B - signed/unsigned byte -** h/H - signed/unsigned short -** l/L - signed/unsigned long -** T - size_t -** i/In - signed/unsigned integer with size `n' (default is size of int) -** cn - sequence of `n' chars (from/to a string); when packing, n==0 means - the whole string; when unpacking, n==0 means use the previous - read number as the string length -** s - zero-terminated string -** f - float -** d - double -** ' ' - ignored -*/ - - -#include -#include -#include -#include -#include - - -#include "lua.h" -#include "lauxlib.h" - - -#if (LUA_VERSION_NUM >= 502) - -#define luaL_register(L,n,f) luaL_newlib(L,f) - -#endif - - -/* basic integer type */ -#if !defined(STRUCT_INT) -#define STRUCT_INT long long -#endif - -typedef STRUCT_INT Inttype; - -/* corresponding unsigned version */ -typedef unsigned STRUCT_INT Uinttype; - - -/* maximum size (in bytes) for integral types */ -#define MAXINTSIZE 32 - -/* is 'x' a power of 2? */ -#define isp2(x) ((x) > 0 && ((x) & ((x) - 1)) == 0) - -/* dummy structure to get alignment requirements */ -struct cD { - char c; - double d; -}; - - -#define PADDING (sizeof(struct cD) - sizeof(double)) -#define MAXALIGN (PADDING > sizeof(int) ? PADDING : sizeof(int)) - - -/* endian options */ -#define BIG 0 -#define LITTLE 1 - - -static union { - int dummy; - char endian; -} const native = { 1 }; - - -typedef struct Header { - int endian; - int align; -} Header; - - -static int getnum(const char** fmt, int df) -{ - if (!isdigit(**fmt)) /* no number? */ - return df; /* return default value */ - else { - int a = 0; - do { - a = a * 10 + *((*fmt)++) - '0'; - } - while (isdigit(**fmt)); - return a; - } -} - - -#define defaultoptions(h) ((h)->endian = native.endian, (h)->align = 1) - - - -static size_t optsize(lua_State* L, char opt, const char** fmt) -{ - switch (opt) { - case 'B': case 'b': return sizeof(char); - case 'H': case 'h': return sizeof(short); - case 'L': case 'l': return sizeof(long); - case 'T': return sizeof(size_t); - case 'f': return sizeof(float); - case 'd': return sizeof(double); - case 'x': return 1; - case 'c': return getnum(fmt, 1); - case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); - if (sz > MAXINTSIZE) luaL_error(L, "integral size %d is larger than limit of %d", - sz, MAXINTSIZE); - return sz; - } - default: return 0; /* other cases do not need alignment */ - } -} - - -/* -** return number of bytes needed to align an element of size 'size' -** at current position 'len' -*/ -static int gettoalign(size_t len, Header* h, int opt, size_t size) -{ - if (size == 0 || opt == 'c') return 0; - if (size > (size_t)h->align) size = h->align; /* respect max. alignment */ - return (int)(size - (len & (size - 1))) & (size - 1); -} - - -/* -** options to control endianess and alignment -*/ -static void controloptions(lua_State* L, int opt, const char** fmt, - Header* h) -{ - switch (opt) { - case ' ': return; /* ignore white spaces */ - case '>': h->endian = BIG; return; - case '<': h->endian = LITTLE; return; - case '!': { - int a = getnum(fmt, MAXALIGN); - if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); - h->align = a; - return; - } - default: { - const char* msg = lua_pushfstring(L, "invalid format option '%c'", opt); - luaL_argerror(L, 1, msg); - } - } -} - - -static void putinteger(lua_State* L, luaL_Buffer* b, int arg, int endian, - int size) -{ - lua_Number n = luaL_checknumber(L, arg); - Uinttype value; - char buff[MAXINTSIZE]; - if (n < 0) value = (Uinttype)(Inttype)n; - else value = (Uinttype)n; - if (endian == LITTLE) { - int i; - for (i = 0; i < size; i++) { - buff[i] = (value & 0xff); - value >>= 8; - } - } else { - int i; - for (i = size - 1; i >= 0; i--) { - buff[i] = (value & 0xff); - value >>= 8; - } - } - luaL_addlstring(b, buff, size); -} - - -static void correctbytes(char* b, int size, int endian) -{ - if (endian != native.endian) { - int i = 0; - while (i < --size) { - char temp = b[i]; - b[i++] = b[size]; - b[size] = temp; - } - } -} - - -static int b_pack(lua_State* L) -{ - luaL_Buffer b; - const char* fmt = luaL_checkstring(L, 1); - Header h; - int arg = 2; - size_t totalsize = 0; - defaultoptions(&h); - lua_pushnil(L); /* mark to separate arguments from string buffer */ - luaL_buffinit(L, &b); - while (*fmt != '\0') { - int opt = *fmt++; - size_t size = optsize(L, opt, &fmt); - int toalign = gettoalign(totalsize, &h, opt, size); - totalsize += toalign; - while (toalign-- > 0) luaL_addchar(&b, '\0'); - switch (opt) { - case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ - putinteger(L, &b, arg++, h.endian, (int)size); - break; - } - case 'x': { - luaL_addchar(&b, '\0'); - break; - } - case 'f': { - float f = (float)luaL_checknumber(L, arg++); - correctbytes((char*)&f, (int)size, h.endian); - luaL_addlstring(&b, (char*)&f, size); - break; - } - case 'd': { - double d = luaL_checknumber(L, arg++); - correctbytes((char*)&d, (int)size, h.endian); - luaL_addlstring(&b, (char*)&d, size); - break; - } - case 'c': case 's': { - size_t l; - const char* s = luaL_checklstring(L, arg++, &l); - if (size == 0) size = l; - luaL_argcheck(L, l >= (size_t)size, arg, "string too short"); - luaL_addlstring(&b, s, size); - if (opt == 's') { - luaL_addchar(&b, '\0'); /* add zero at the end */ - size++; - } - break; - } - default: controloptions(L, opt, &fmt, &h); - } - totalsize += size; - } - luaL_pushresult(&b); - return 1; -} - - -static lua_Number getinteger(const char* buff, int endian, - int issigned, int size) -{ - Uinttype l = 0; - int i; - if (endian == BIG) { - for (i = 0; i < size; i++) { - l <<= 8; - l |= (Uinttype)(unsigned char)buff[i]; - } - } else { - for (i = size - 1; i >= 0; i--) { - l <<= 8; - l |= (Uinttype)(unsigned char)buff[i]; - } - } - if (!issigned) return (lua_Number)l; - else { /* signed format */ - Uinttype mask = (Uinttype)(~((Uinttype)0)) << (size * 8 - 1); - if (l & mask) /* negative value? */ - l |= mask; /* signal extension */ - return (lua_Number)(Inttype)l; - } -} - - -static int b_unpack(lua_State* L) -{ - Header h; - const char* fmt = luaL_checkstring(L, 1); - size_t ld; - const char* data = luaL_checklstring(L, 2, &ld); - size_t pos = luaL_optinteger(L, 3, 1) - 1; - defaultoptions(&h); - lua_settop(L, 2); - while (*fmt) { - int opt = *fmt++; - size_t size = optsize(L, opt, &fmt); - pos += gettoalign(pos, &h, opt, size); - luaL_argcheck(L, pos + size <= ld, 2, "data string too short"); - luaL_checkstack(L, 1, "too many results"); - switch (opt) { - case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ - int issigned = islower(opt); - lua_Number res = getinteger(data + pos, h.endian, issigned, (int)size); - lua_pushnumber(L, res); - break; - } - case 'x': { - break; - } - case 'f': { - float f; - memcpy(&f, data + pos, size); - correctbytes((char*)&f, sizeof(f), h.endian); - lua_pushnumber(L, f); - break; - } - case 'd': { - double d; - memcpy(&d, data + pos, size); - correctbytes((char*)&d, sizeof(d), h.endian); - lua_pushnumber(L, d); - break; - } - case 'c': { - if (size == 0) { - if (!lua_isnumber(L, -1)) luaL_error(L, "format `c0' needs a previous size"); - size = (unsigned)lua_tonumber(L, -1); - lua_pop(L, 1); - luaL_argcheck(L, pos + size <= ld, 2, "data string too short"); - } - lua_pushlstring(L, data + pos, size); - break; - } - case 's': { - const char* e = (const char*)memchr(data + pos, '\0', ld - pos); - if (e == NULL) luaL_error(L, "unfinished string in data"); - size = (e - (data + pos)) + 1; - lua_pushlstring(L, data + pos, size - 1); - break; - } - default: controloptions(L, opt, &fmt, &h); - } - pos += size; - } - lua_pushinteger(L, pos + 1); - return lua_gettop(L) - 2; -} - - -static int b_size(lua_State* L) -{ - Header h; - const char* fmt = luaL_checkstring(L, 1); - size_t pos = 0; - defaultoptions(&h); - while (*fmt) { - int opt = *fmt++; - size_t size = optsize(L, opt, &fmt); - pos += gettoalign(pos, &h, opt, size); - if (opt == 's') luaL_argerror(L, 1, "option 's' has no fixed size"); - else if (opt == 'c' && size == 0) luaL_argerror(L, 1, "option 'c0' has no fixed size"); - if (!isalnum(opt)) controloptions(L, opt, &fmt, &h); - pos += size; - } - lua_pushinteger(L, pos); - return 1; -} - -/* }====================================================== */ - - - -static const struct luaL_Reg thislib[] = { - { "pack", b_pack }, - { "unpack", b_unpack }, - { "size", b_size }, - { NULL, NULL } -}; - - -LUALIB_API int luaopen_struct(lua_State* L); - -LUALIB_API int luaopen_struct(lua_State* L) -{ - luaL_register(L, "struct", thislib); - return 1; -} - - -/****************************************************************************** -* Copyright (C) 2010-2012 Lua.org, PUC-Rio. All rights reserved. -* -* Permission is hereby granted, free of charge, to any person obtaining -* a copy of this software and associated documentation files (the -* "Software"), to deal in the Software without restriction, including -* without limitation the rights to use, copy, modify, merge, publish, -* distribute, sublicense, and/or sell copies of the Software, and to -* permit persons to whom the Software is furnished to do so, subject to -* the following conditions: -* -* The above copyright notice and this permission notice shall be -* included in all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -******************************************************************************/ - diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 5f364a3..b6752c3 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -5,18 +5,14 @@ add_executable(test_lua_sandbox test_lua_sandbox.c) target_link_libraries(test_lua_sandbox luasandbox) -add_test(NAME test_setup_cmod COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/modules ${CMAKE_CURRENT_SOURCE_DIR}/modules) +add_test(NAME test_move_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) if(WIN32) - add_test(NAME test_setup_lib COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib ${CMAKE_CURRENT_SOURCE_DIR}) - if(MSVC) - set(_DLL_NAME "luasandbox.dll") - else() - set(_DLL_NAME "libluasandbox.dll") - endif() - add_test(NAME test_setup_target COMMAND cmake -E copy ${CMAKE_BINARY_DIR}/src/${_DLL_NAME} ${CMAKE_CURRENT_SOURCE_DIR}) + add_test(NAME test_move_lua_lib COMMAND cmake -E copy + ${CMAKE_BINARY_DIR}/ep_base/lib/${CMAKE_SHARED_MODULE_PREFIX}lua${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) + add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy + ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_MODULE_PREFIX}luasandbox${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) endif() -add_test(NAME test_setup_lua COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_SOURCE_DIR}/modules) -add_test(NAME test_sandbox WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND test_lua_sandbox) - -install(TARGETS test_lua_sandbox DESTINATION lib) +add_test(NAME test_move_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) +add_test(NAME test_move_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +add_test(NAME test_sandbox WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND test_lua_sandbox) From 979bc778dd84bd612c1cf2557156a528061af117 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Fri, 30 Jan 2015 11:06:27 -0800 Subject: [PATCH 037/235] Fix up the Windows build --- cmake/externals.cmake | 4 ++-- src/CMakeLists.txt | 2 +- src/lsb.c | 4 ++-- src/test/CMakeLists.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 802f18b..30e79f1 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -85,8 +85,8 @@ add_dependencies(lua_cjson ${LUA_PROJECT}) externalproject_add( lua_struct - GIT_REPOSITORY https://github.com/LuaDist/struct.git - GIT_TAG 1ba845fbfaa7a6e4d96088938b4574d194f13f6a + GIT_REPOSITORY https://github.com/trink/struct.git + GIT_TAG 5cf31819bee0d829d058cb5219e95ef0b1dd43a8 UPDATE_COMMAND cmake -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4671edd..ce5e7ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,7 +28,7 @@ if(MSVC) set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/lua.lib") elseif(MINGW) add_definitions(-D_MINGW) - set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/liblua.dll") + set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/liblua.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) if (ADDRESS_MODEL EQUAL 32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") diff --git a/src/lsb.c b/src/lsb.c index 3eefef1..1b6e408 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -194,10 +194,10 @@ lua_sandbox* lsb_create(void* parent, if (require_path) { #if defined(_WIN32) - if (expand_path(require_path, sizeof(lpath), "%s\\?.lua", lpath)) { + if (expand_path(require_path, sizeof(lpath), "%s\\\\?.lua", lpath)) { return NULL; } - if (expand_path(require_path, sizeof(cpath), "%s\\?.dll", cpath)) { + if (expand_path(require_path, sizeof(cpath), "%s\\\\?.dll", cpath)) { return NULL; } #else diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index b6752c3..a7101f5 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries(test_lua_sandbox luasandbox) add_test(NAME test_move_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) if(WIN32) add_test(NAME test_move_lua_lib COMMAND cmake -E copy - ${CMAKE_BINARY_DIR}/ep_base/lib/${CMAKE_SHARED_MODULE_PREFIX}lua${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) + ${CMAKE_BINARY_DIR}/ep_base/bin/${CMAKE_SHARED_MODULE_PREFIX}lua${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_MODULE_PREFIX}luasandbox${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) endif() From 59412542661241fa15da89327fde62c015687a88 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Fri, 30 Jan 2015 14:40:50 -0800 Subject: [PATCH 038/235] Mingw install the liblua.dll to the lib directory --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ce5e7ea..43e6f1f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -34,6 +34,7 @@ elseif(MINGW) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") set_target_properties(luasandbox PROPERTIES LINK_FLAGS "-s -m32") endif() + install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll") else() if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") From 84ffce0b5ff221a903984e9ac1020cb3b8b84860 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sat, 31 Jan 2015 09:56:14 -0800 Subject: [PATCH 039/235] Fix up the Mac build/test - update the docs - clean up some unused cmake code --- CMakeLists.txt | 4 +- README.md | 160 ++++++++++++++++++++++++++++++++++------ cmake/externals.cmake | 5 +- docs/sandbox_api.md | 108 --------------------------- src/CMakeLists.txt | 13 +--- src/test/CMakeLists.txt | 5 ++ 6 files changed, 147 insertions(+), 148 deletions(-) delete mode 100644 docs/sandbox_api.md diff --git a/CMakeLists.txt b/CMakeLists.txt index 00303f1..f634cbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR) project(lua_sandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 7) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_MINOR 8) +set(CPACK_PACKAGE_VERSION_PATCH 0) option(LUA_JIT "Enable LuaJIT" off) if(LUA_JIT) diff --git a/README.md b/README.md index 7aa11b4..16a3ece 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,156 @@ -Prerequisites {#mainpage} -==== +Lua Sandbox Library {#mainpage} +------------------- + +## Overview + +Sandboxes provide a dynamic and isolated execution environment +for data parsing, transformation, and analysis. They allow access to data +without jeopardizing the integrity or performance of the processing +infrastructure. This broadens the audience that the data can be +exposed to and facilitates new uses of the data (i.e. debugging, monitoring, +dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, +etc.) + +The Lua sandbox is library allowing customized control over the Lua execution +environment including functionality like global data preservation/restoration on +shutdown/startup, output collection in textual, binary or [Heka protobuf format] +(https://hekad.readthedocs.org/en/latest/sandbox/index.html#sample-lua-message-structure) +and an array of parsers for various data types (Nginx, Apache, Syslog, MySQL and +many RFC grammars) + +### Features + +- small - memory requirements are as little as 6 KiB for a basic sandbox +- fast - microsecond execution times +- stateful - ability to resume where it left off after a restart/reboot +- isolated - failures are contained and malfunctioning sandboxes are terminated. + Containment is defined in terms of restriction to the operating system, + file system, libraries, memory use, Lua instruction use, and output size. + +## Installation + +### Prerequisites * C compiler (GCC 4.7+, Visual Studio 2013 (LuaJIT), MinGW (Lua 5.1)) * CMake (2.8.7+) - http://cmake.org/cmake/resources/software.html * Git http://git-scm.com/download -Optional (used for documentation) ----- +#### Optional (used for documentation) * Graphviz (2.28.0) - http://graphviz.org/Download..php * Doxygen (1.8+)- http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc -lua_sandbox - UNIX Build Instructions ----- +### CMake Build Instructions git clone https://github.com/mozilla-services/lua_sandbox.git - cd lua_sandbox + cd lua_sandbox mkdir release cd release + + # UNIX cmake -DCMAKE_BUILD_TYPE=release .. make - ctest - -lua_sandbox - Windows Build Instructions ----- -# in a VS2013 command prompt window - - git clone https://github.com/mozilla-services/lua_sandbox.git - cd lua_sandbox - mkdir release - cd release + # Windows Visual Studio 2013 cmake -DCMAKE_BUILD_TYPE=release -G "NMake Makefiles" .. nmake - ctest - -# in a MinGW command prompt window - git clone https://github.com/mozilla-services/lua_sandbox.git - cd lua_sandbox - mkdir release - cd release + # Windows MinGW cmake -DCMAKE_BUILD_TYPE=release -G "MinGW Makefiles" .. mingw32-make + ctest + +## Sandbox API + +### Lua functions exposed to C by the core sandbox + +There are no functions exposed by default, see lsb_pcall_setup() and +lsb_pcall_teardown() when defining an API. + +### Functions exposed to Lua by the core sandbox +**require(libraryName)** + +By default only the base library is loaded additional libraries must be loaded with require(). + +*Arguments* + +- libraryName (string) + - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - The require() function has been modified to not expose any of the package table to the sandbox. + - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, print. + - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) Test whether an element is a member of a set + - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) In memory time series data base and analysis + - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) JSON parser with the following modifications: + - Loads the cjson module in a global cjson table + - The encode buffer is limited to the sandbox output_limit. + - The decode buffer will be roughly limited to one half of the sandbox memory_limit. + - The NULL value is not decoded to cjson.null it is simply discarded. + If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. + - The new() function has been disabled so only a single cjson parser can be created. + - The encode_keep_buffer() function has been disabled (the buffer is always reused). + - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) Efficiently count the number of elements in a set + - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library + - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG + - [math](http://www.lua.org/manual/5.1/manual.html#5.6) + - [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - The local timezone is set to UTC in all sandboxes. + - Disabled functions (default): execute, exit, remove, rename, setlocale, tmpname. + - [strict](http://www.lua.org/extras/5.2/strict.lua) Checks for use of undeclared global variables. + - [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs + - [table](http://www.lua.org/manual/5.1/manual.html#5.5) + - Grammar Libraries + - cbufd - Parses the circular buffer library delta output (use for data aggregation) + - common_log_format - Nginx and Apache meta grammar generators (creates a grammar based on the log_format configuration) + - data_time - RFC3339, RFC3164, strftime, common log format, MySQL and Postgres timestamps + - ip_address - IPv4 and IPv6 address + - mysql - MySQL and MariaDB slow query and short slow query parsers + - syslog - Rsyslog meta grammar generator (creates a grammar based on the template configuration) + - _user provided_ (lua, so/dll) + +*Return* +- a table - For non user provided libraries the table is also globally registered + with the library name. User provided libraries may implement there own semantics + i.e. the grammar libraries return a table but do not globally register the table name + (see the individual module documentation for the correct usage). + +____ +**output(arg0, arg1, ...argN)** + lsb_appends data to the output buffer, which cannot exceed the output_limit + configuration parameter. See lsb_get_output() to connect the output to the + host application. + +*Arguments* +- arg (number, string, bool, nil, circular_buffer) Lua variable or literal to be appended the output buffer + +*Return* +- none + +**Note:** To extend the function set exposed to Lua see lsb_add_function() + +## How to interact with the sandbox (creating an API) + +The best place to start is with some examples: + +### Unit Test API + +[Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_lua_sandbox.c) + +#### Lua Functions Exposed to C + +- int **process** (double) + - exposes a process function that takes a test case number as its argument and returns and integer result code. +- void **report** (double) + - exposes a report function that takes a test case number as its argument and returns nothing. + +#### C Functions Exposed to Lua + +- void **write_output** (optionalArg1, optionalArg2, optionalArgN) + - captures whatever is in the output buffer for use by the host application, appending any optional arguments + (optional arguments have the same restriction as output). +- void **write_message** (table) + - serializes the Lua table into a Heka protobuf message and captures the output. + - the table must use the following [schema](https://hekad.readthedocs.org/en/latest/sandbox/index.html#sample-lua-message-structure). + +### Heka Sandbox API + +[Heka Sandbox](https://hekad.readthedocs.org/en/latest/sandbox/index.html#lua-sandbox) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 30e79f1..a47b4dd 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -12,7 +12,7 @@ endif() set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) -set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DADDRESS_MODEL=${ADDRESS_MODEL} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include --no-warn-unused-cli) +set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include --no-warn-unused-cli) set(LUA_INCLUDE_DIR "${EP_BASE}/include") if (LUA_JIT) @@ -100,7 +100,6 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) -add_dependencies(lua_bloom_filter ${LUA_PROJECT}) externalproject_add( lua_circular_buffer @@ -109,7 +108,6 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) -add_dependencies(lua_circular_buffer ${LUA_PROJECT}) externalproject_add( lua_hyperloglog @@ -118,4 +116,3 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) -add_dependencies(lua_hyperloglog ${LUA_PROJECT}) diff --git a/docs/sandbox_api.md b/docs/sandbox_api.md deleted file mode 100644 index d4af47d..0000000 --- a/docs/sandbox_api.md +++ /dev/null @@ -1,108 +0,0 @@ -Sandbox API -=========== - -Sandboxes provide a dynamic and isolated execution environment -for data parsing, transformation, and analysis. They allow access to data -without jeopardizing the integrity or performance of the processing -infrastructure. This broadens the audience that the data can be -exposed to and facilitates new uses of the data (i.e. debugging, monitoring, -dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, -etc.) - -Features -======== -- small - memory requirements are as little as 6 KiB for a basic sandbox -- fast - microsecond execution times -- stateful - ability to resume where it left off after a restart/reboot -- isolated - failures are contained and malfunctioning sandboxes are terminated. - Containment is defined in terms of restriction to the operating system, - file system, libraries, memory use, Lua instruction use, and output size. - -Functions exposed to Lua by the core sandbox -============================================ -**require(libraryName)** - -By default only the base library is loaded additional libraries must be loaded with require(). - -*Arguments* - -- libraryName (string) - - [base library(http://www.lua.org/manual/5.1/manual.html#5.1) - - The require() function has been modified to not expose any of the package table to the sandbox. - - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, print. - - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) - - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) - - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) With the following modifications: - - Loads the cjson module in a global cjson table - - The encode buffer is limited to the sandbox output_limit. - - The decode buffer will be roughly limited to one half of the sandbox memory_limit. - - The NULL value is not decoded to cjson.null it is simply discarded. - If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - - The new() function has been disabled so only a single cjson parser can be created. - - The encode_keep_buffer() function has been disabled (the buffer is always reused). - - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) - - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library - - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG - - [math](http://www.lua.org/manual/5.1/manual.html#5.6) - - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - - The local timezone is set to UTC in all sandboxes. - - Disabled functions (default): execute, exit, remove, rename, setlocale, tmpname. - - [strict](http://www.lua.org/extras/5.2/strict.lua) Checks for use of undeclared global variables. - - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - - [struct](http://www.inf.puc-rio.br/~roberto/struct/) - - [table](http://www.lua.org/manual/5.1/manual.html#5.5) - - _user provided_ (lua, so/dll) - -*Return* -- a table - For non user provided libraries the table is also globally registered - with the library name. User provided libraries may implement there own semantics - i.e. the grammar libraries return a table but do not globally register the table name - (see the individual module documentation for the correct usage). - -____ -**output(arg0, arg1, ...argN)** - lsb_appends data to the output buffer, which cannot exceed the output_limit - configuration parameter. See lsb_get_output() to connect the output to the - host application. - -*Arguments* -- arg (number, string, bool, nil, circular_buffer) Lua variable or literal to be appended the output buffer - -*Return* -- none - -**Note:** To extend the function set exposed to Lua see lsb_add_function() - - -Lua functions exposed to C by the core sandbox -============================================== -There are no functions exposed by default, see lsb_pcall_setup() and -lsb_pcall_teardown() when defining an API. - -How to interact with the sandbox (creating an API) -================================================== -The best place to start is with some examples: - -Unit Test API -============= -Lua Functions Exposed to C --------------------------- -- int **process** (double) - - exposes a process function that takes a test case number as its argument and returns and integer result code. -- void **report** (double) - - exposes a report function that takes a test case number as its argument and returns nothing. - -C Functions Exposed to Lua --------------------------- -- void **write_output** (optionalArg1, optionalArg2, optionalArgN) - - captures whatever is in the output buffer for use by the host application, appending any optional arguments - (optional arguments have the same restriction as output). -- void **write_message** (table) - - serializes the Lua table into a Heka protobuf message and captures the output. - - the table must use the following [schema](https://hekad.readthedocs.org/en/latest/sandbox/index.html#sample-lua-message-structure). - -[Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_lua_sandbox.c) - -Heka Sandbox API -================ -[Heka Sandbox](https://hekad.readthedocs.org/en/latest/sandbox/index.html#lua-sandbox) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 43e6f1f..3eb3a3d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,9 +16,6 @@ set ( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE STRING "" FORCE ) set ( CMAKE_INSTALL_NAME_DIR "@executable_path/../lib" CACHE STRING "" FORCE ) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) -add_dependencies(lua_cjson ${LUA_PROJECT}) -add_dependencies(lua_lpeg ${LUA_PROJECT}) - add_dependencies(lua_bloom_filter luasandbox) add_dependencies(lua_circular_buffer luasandbox) add_dependencies(lua_hyperloglog luasandbox) @@ -30,16 +27,12 @@ elseif(MINGW) add_definitions(-D_MINGW) set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/liblua.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) - if (ADDRESS_MODEL EQUAL 32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set_target_properties(luasandbox PROPERTIES LINK_FLAGS "-s -m32") - endif() - install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll") + install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll") else() - if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") + if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") endif() - set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/liblua.so" ${LINK_DL} -lm) + set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}lua${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} -lm) endif() target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a7101f5..c3d376b 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -11,6 +11,11 @@ if(WIN32) ${CMAKE_BINARY_DIR}/ep_base/bin/${CMAKE_SHARED_MODULE_PREFIX}lua${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_MODULE_PREFIX}luasandbox${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) +elseif(APPLE) + add_test(NAME test_move_lua_lib COMMAND cmake -E copy_directory + ${CMAKE_BINARY_DIR}/ep_base/lib ${CMAKE_CURRENT_BINARY_DIR}/../lib) + add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy + ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_LIBRARY_PREFIX}luasandbox${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/../lib) endif() add_test(NAME test_move_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) From db9af9f150783776b773a190d7949d6db36e8426 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Wed, 4 Feb 2015 06:35:16 -0800 Subject: [PATCH 040/235] Issue #68 fix path/cpath escaping on Windows --- src/lsb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lsb.c b/src/lsb.c index 1b6e408..592fdc9 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -27,8 +27,8 @@ static const char* standard_config = "{" "memory_limit = %u," "instruction_limit = %u," "output_limit = %u," - "path = '%s'," - "cpath = '%s'," + "path = [[%s]]," + "cpath = [[%s]]," "remove_entries = {" "[''] = { 'collectgarbage', 'coroutine', 'dofile', 'load', 'loadfile'" ",'loadstring', 'print'}," @@ -194,10 +194,10 @@ lua_sandbox* lsb_create(void* parent, if (require_path) { #if defined(_WIN32) - if (expand_path(require_path, sizeof(lpath), "%s\\\\?.lua", lpath)) { + if (expand_path(require_path, sizeof(lpath), "%s\\?.lua", lpath)) { return NULL; } - if (expand_path(require_path, sizeof(cpath), "%s\\\\?.dll", cpath)) { + if (expand_path(require_path, sizeof(cpath), "%s\\?.dll", cpath)) { return NULL; } #else From d29a494c1faca8e2ad5ee97f5a0315a64b2f053a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sat, 7 Feb 2015 06:54:26 -0800 Subject: [PATCH 041/235] Disable the io module in the default sandbox - Add tests to verify the full configuration of the sandbox --- cmake/externals.cmake | 2 +- include/lsb.h | 9 ++- src/lsb.c | 10 ++- src/test/lua/sandbox_config.lua | 138 ++++++++++++++++++++++++++++++++ src/test/test_lua_sandbox.c | 17 ++++ 5 files changed, 167 insertions(+), 9 deletions(-) create mode 100644 src/test/lua/sandbox_config.lua diff --git a/cmake/externals.cmake b/cmake/externals.cmake index a47b4dd..887b273 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -58,7 +58,7 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG 63a4f245d214cdb61e7ae1ce2595e5c1ce3c7f51 + GIT_TAG d7ff33f45cf02b4b2e2c1a3cc671bda99b0056c6 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/include/lsb.h b/include/lsb.h index 5c23630..f95e9b6 100644 --- a/include/lsb.h +++ b/include/lsb.h @@ -81,10 +81,11 @@ LSB_EXPORT lua_sandbox* lsb_create(void* parent, * path = '/modules/?.lua', * cpath = '/modules/?.so', * remove_entries = { - * [''] = { 'collectgarbage', 'coroutine', 'dofile', 'load', 'loadfile', - * 'loadstring', 'module', 'print'}, - * os = {'execute', 'exit', 'remove', 'rename', 'setlocale', 'tmpname'} - * } + * [''] = {'collectgarbage','coroutine','dofile','load','loadfile'" + * ",'loadstring','newproxy','print'}, + * os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'} + * }, + * disable_modules = {io = 1} * } * * @param parent Pointer to associate the owner to this sandbox. diff --git a/src/lsb.c b/src/lsb.c index 592fdc9..b396fea 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -30,10 +30,12 @@ static const char* standard_config = "{" "path = [[%s]]," "cpath = [[%s]]," "remove_entries = {" - "[''] = { 'collectgarbage', 'coroutine', 'dofile', 'load', 'loadfile'" - ",'loadstring', 'print'}," - "os = {'execute', 'exit', 'remove', 'rename', 'setlocale', 'tmpname'}" - "}}"; + "[''] = {'collectgarbage','coroutine','dofile','load','loadfile'" + ",'loadstring','newproxy','print'}," + "os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'}" + "}," + "disable_modules = {io = 1}" + "}"; static jmp_buf g_jbuf; diff --git a/src/test/lua/sandbox_config.lua b/src/test/lua/sandbox_config.lua new file mode 100644 index 0000000..1631aaf --- /dev/null +++ b/src/test/lua/sandbox_config.lua @@ -0,0 +1,138 @@ +local ok, err = pcall(require, "io") +assert(not ok, "the io module is disabled") +assert(err == "module 'io' disabled", err) + +require "math" +require "os" +require "string" +require "table" + +local function all_there(test) + for k, v in pairs(test.values) do + assert(test.t[k], string.format("missing: %s.%s", test.name, k)) + end +end + +local function nothing_extra(test) + for k, v in pairs(test.t) do + assert(test.values[k], string.format("extra: %s.%s", test.name, k)) + end +end + +local tests = { + {name = "base", t = _G, values = { + _G=1, + _VERSION=1, + assert=1, + error=1, + gcinfo=1, + getfenv=1, + getmetatable=1, + ipairs=1, + math=1, + module=1, + next=1, + os=1, + output=1, + package=1, + pairs=1, + pcall=1, + rawequal=1, + rawget=1, + rawset=1, + require=1, + select=1, + setfenv=1, + setmetatable=1, + string=1, + table=1, + tonumber=1, + tostring=1, + type=1, + unpack=1, + xpcall=1, + } + }, + + {name = "math", t = math, values = { + abs=1, + acos=1, + asin=1, + atan2=1, + atan=1, + ceil=1, + cos=1, + cosh=1, + deg=1, + exp=1, + floor=1, + fmod=1, + frexp=1, + huge=1, + ldexp=1, + log10=1, + log=1, + max=1, + min=1, + mod=1, -- compat + modf=1, + pi=1, + pow=1, + rad=1, + random=1, + randomseed=1, + sin=1, + sinh=1, + sqrt=1, + tan=1, + tanh=1, + } + }, + + {name = "os", t = os, values = { + clock=1, + date=1, + difftime=1, + time=1, + } + }, + + {name = "package", t = package, values = {}}, + + {name = "string", t = string, values = { + byte=1, + char=1, + dump=1, + find=1, + format=1, + gfind=1, -- compat + gmatch=1, + gsub=1, + len=1, + lower=1, + match=1, + rep=1, + reverse=1, + sub=1, + upper=1, + } + }, + + {name = "table", t = table, values = { + concat=1, + foreach=1, + foreachi=1, + getn=1, + insert=1, + maxn=1, + remove=1, + setn=1, + sort=1, + } + }, +} + +for i, v in ipairs(tests) do + all_there(v) + nothing_extra(v) +end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 5293293..1b39b5f 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -1093,6 +1093,22 @@ static char* test_struct() } +static char* test_sandbox_config() +{ + lua_sandbox* sb = lsb_create(NULL, "lua/sandbox_config.lua", "modules", 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -1384,6 +1400,7 @@ static char* all_tests() mu_run_test(test_bloom_filter); mu_run_test(test_hyperloglog); mu_run_test(test_struct); + mu_run_test(test_sandbox_config); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); From 7ecb43cda1edf9a948eb9f2f6a0a994be1959ab5 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 9 Feb 2015 06:23:43 -0800 Subject: [PATCH 042/235] Update the disabled function list in the README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16a3ece..576cdf1 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ By default only the base library is loaded additional libraries must be loaded w - libraryName (string) - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) - The require() function has been modified to not expose any of the package table to the sandbox. - - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, print. + - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, newproxy, print. - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) Test whether an element is a member of a set - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) In memory time series data base and analysis - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) JSON parser with the following modifications: @@ -93,7 +93,7 @@ By default only the base library is loaded additional libraries must be loaded w - [math](http://www.lua.org/manual/5.1/manual.html#5.6) - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. - - Disabled functions (default): execute, exit, remove, rename, setlocale, tmpname. + - Disabled functions (default): getenv, execute, exit, remove, rename, setlocale, tmpname. - [strict](http://www.lua.org/extras/5.2/strict.lua) Checks for use of undeclared global variables. - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs From c40dee7c84ffcc13bc5747790e00e0a60c0dfb05 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 16 Feb 2015 15:26:19 -0800 Subject: [PATCH 043/235] Issue #70 add Heka protobuf message decoding --- include/lsb.h | 11 + src/CMakeLists.txt | 1 + src/lsb_deserialize_protobuf.c | 393 ++++++++++++++++++++++ src/lsb_serialize_protobuf.c | 272 ++++++++++----- src/test/lua/decode_message.lua | 201 +++++++++++ src/test/lua/decode_message_benchmark.lua | 6 + src/test/lua/output.lua | 26 +- src/test/lua/output_errors.lua | 30 ++ src/test/test_lua_sandbox.c | 68 ++++ 9 files changed, 930 insertions(+), 78 deletions(-) create mode 100644 src/lsb_deserialize_protobuf.c create mode 100644 src/test/lua/decode_message.lua create mode 100644 src/test/lua/decode_message_benchmark.lua diff --git a/include/lsb.h b/include/lsb.h index f95e9b6..37a5b14 100644 --- a/include/lsb.h +++ b/include/lsb.h @@ -233,4 +233,15 @@ LSB_EXPORT void lsb_pcall_teardown(lua_sandbox* lsb); */ LSB_EXPORT void lsb_terminate(lua_sandbox* lsb, const char* err); +/** + * Deserialize a Heka message protobuf string into a Lua table structure. For + * ease of reuse it is include here (since many of our sandboxes process Heka + * messages). If needed it must be manually added to the sandbox. + * + * @param lua Pointer the Lua state. + * + * @return One on success (table), two on failure (nil, error_message). + */ +LSB_EXPORT int lsb_decode_protobuf(lua_State* lua); + #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3eb3a3d..2709558 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,6 +7,7 @@ lsb.c lsb_output.c lsb_serialize.c lsb_serialize_protobuf.c +lsb_deserialize_protobuf.c ) set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c new file mode 100644 index 0000000..c39b820 --- /dev/null +++ b/src/lsb_deserialize_protobuf.c @@ -0,0 +1,393 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox Heka protobuf deserialization @file */ + +#include "lsb.h" + +#include +#include + +#define UUID_SIZE 16 +#define MAX_VARINT_BYTES 10 + +static unsigned const char* +read_key(unsigned const char* p, int* tag, int* wiretype) +{ + *wiretype = 7 & *p; + *tag = *p >> 3; + return ++p; +} + + +static unsigned const char* +read_length(unsigned const char* p, unsigned const char* e, size_t* vi) +{ + *vi = 0; + unsigned i, shift = 0; + for (i = 0; p != e && i < MAX_VARINT_BYTES; i++) { + *vi |= (p[i] & 0x7f) << shift; + shift += 7; + if ((p[i] & 0x80) == 0) break; + } + if (i == MAX_VARINT_BYTES) { + return NULL; + } + return p + i + 1; +} + + +static unsigned const char* +read_varint(unsigned const char* p, unsigned const char* e, long long* vi) +{ + *vi = 0; + unsigned i, shift = 0; + for (i = 0; p != e && i < MAX_VARINT_BYTES; i++) { + *vi |= (p[i] & 0x7f) << shift; + shift += 7; + if ((p[i] & 0x80) == 0) break; + } + if (i == MAX_VARINT_BYTES) { + return NULL; + } + return p + i + 1; +} + + +static unsigned const char* +read_string(lua_State* lua, + int wiretype, + unsigned const char* p, + unsigned const char* e) +{ + if (wiretype != 2) { + return NULL; + } + + size_t len = 0; + p = read_length(p, e, &len); + if (!p || p + len > e) { + return NULL; + } + lua_pushlstring(lua, (const char*)p, len); + p += len; + return p; +} + + +static unsigned const char* +process_varint(lua_State* lua, + const char* name, + int wiretype, + int stack_index, + unsigned const char* p, + unsigned const char* e) +{ + if (wiretype != 0) { + return NULL; + } + long long val = 0; + p = read_varint(p, e, &val); + if (!p) { + return NULL; + } + lua_pushnumber(lua, val); + lua_setfield(lua, stack_index, name); + return p; +} + + +static unsigned const char* +process_fields(lua_State* lua, + unsigned const char* p, + unsigned const char* e) +{ + int tag = 0; + int wiretype = 0; + int has_name = 0; + int value_count = 0; + size_t len = 0; + + p = read_length(p, e, &len); + if (!p || p + len > e) { + return NULL; + } + e = p + len; // only process to the end of the current field record + + lua_newtable(lua); // Table to be added to the Fields array index 4 + lua_newtable(lua); // Table to hold the value(s) index 5 + do { + p = read_key(p, &tag, &wiretype); + + switch (tag) { + case 1: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 4, "name"); + has_name = 1; + } + break; + + case 2: + p = process_varint(lua, "value_type", wiretype, 4, p, e); + break; + + case 3: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 4, "representation"); + } + break; + + case 4: // value_string + case 5: // value_bytes + p = read_string(lua, wiretype, p, e); + if (p) { + lua_rawseti(lua, 5, ++value_count); + } + break; + + case 6: // value_integer + { + long long val = 0; + switch (wiretype) { + case 0: + p = read_varint(p, p + len, &val); + if (!p) break; + lua_pushinteger(lua, val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = read_length(p, e, &len); + if (!p || p + len > e) { + p = NULL; + break; + } + do { + p = read_varint(p, p + len, &val); + if (!p) break; + lua_pushinteger(lua, val); + lua_rawseti(lua, 5, ++value_count); + } + while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + + case 7: // value_double + { + double val = 0; + switch (wiretype) { + case 1: + if (p + sizeof(double) > e) { + p = NULL; + break; + } + memcpy(&val, p, sizeof(double)); + p += sizeof(double); + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = read_length(p, e, &len); + if (!p || p + len > e || len % sizeof(double) != 0) { + p = NULL; + break; + } + do { + memcpy(&val, p, sizeof(double)); + p += sizeof(double); + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + } + while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + + case 8: // value_bool + { + long long val = 0; + switch (wiretype) { + case 0: + p = read_varint(p, p + len, &val); + if (!p) break; + lua_pushboolean(lua, val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = read_length(p, e, &len); + if (!p || p + len > e) { + p = NULL; + break; + } + do { + p = read_varint(p, p + len, &val); + if (!p) break; + lua_pushboolean(lua, val); + lua_rawseti(lua, 5, ++value_count); + } + while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + default: + p = NULL; // don't allow unknown tags + break; + } + } + while (p && p < e); + + lua_setfield(lua, 4, "value"); + + return has_name ? p : NULL; +} + + +int lsb_decode_protobuf(lua_State* lua) +{ + int n = lua_gettop(lua); + if (n != 1 || lua_type(lua, 1) != LUA_TSTRING) { + return luaL_argerror(lua, 0, "must have one string argument"); + } + + size_t len; + const char* pbstr = lua_tolstring(lua, 1, &len); + if (len < 20) { + lua_pushnil(lua); + lua_pushstring(lua, "invalid message, too short"); + return 2; + } + + unsigned const char* p = (unsigned const char*)pbstr; + unsigned const char* lp = p; + unsigned const char* e = (unsigned const char*)pbstr + len; + int wiretype = 0; + int tag = 0; + int has_uuid = 0; + int has_timestamp = 0; + int field_count = 0; + + lua_newtable(lua); // message table index 2 + do { + p = read_key(p, &tag, &wiretype); + + switch (tag) { + case 1: + p = read_string(lua, wiretype, p, e); + if (p && p - lp == 18) { + lua_setfield(lua, 2, "Uuid"); + has_uuid = 1; + } else { + p = NULL; + } + break; + + case 2: + p = process_varint(lua, "Timestamp", wiretype, 2, p, e); + if (p) { + has_timestamp = 1; + } + break; + + case 3: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Type"); + } + break; + + case 4: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Logger"); + } + break; + + case 5: + p = process_varint(lua, "Severity", wiretype, 2, p, e); + break; + + case 6: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Payload"); + } + break; + + case 7: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "EnvVersion"); + } + break; + + case 8: + p = process_varint(lua, "Pid", wiretype, 2, p, e); + break; + + case 9: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Hostname"); + } + break; + + case 10: + if (wiretype != 2) { + p = NULL; + break; + } + if (field_count == 0) { + lua_newtable(lua); // Fields table index 3 + } + p = process_fields(lua, p, e); + if (p) { + lua_rawseti(lua, 3, ++field_count); + } + break; + + default: + p = NULL; // don't allow unknown tags + break; + } + if (p) lp = p; + } + while (p && p < e); + + if (!p) { + lua_pushnil(lua); + lua_pushfstring(lua, "error in tag: %d wiretype: %d offset: %d", tag, + wiretype, (const char*)lp - pbstr); + return 2; + } + + if (!(has_uuid && has_timestamp)) { + lua_pushnil(lua); + lua_pushfstring(lua, "missing required field uuid: %s timestamp: %s", + has_uuid ? "found" : "not found", + has_timestamp ? "found" : "not found"); + return 2; + } + + if (field_count) { + lua_setfield(lua, 2, "Fields"); + } + + return 1; +} diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c index d857615..3b8a3fa 100644 --- a/src/lsb_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -15,6 +15,9 @@ #include "lsb_output.h" #include "lsb_private.h" +#define UUID_SIZE 16 +#define MAX_VARINT_BYTES 10 + /** * Writes a varint encoded number to the output buffer. * @@ -25,7 +28,7 @@ */ static int pb_write_varint(lsb_output_data* d, unsigned long long i) { - size_t needed = 10; + size_t needed = MAX_VARINT_BYTES; if (needed > d->size - d->pos) { if (lsb_realloc_output(d, needed)) return 1; } @@ -99,7 +102,8 @@ static int pb_write_bool(lsb_output_data* d, int i) * * @return int Zero on success, non-zero if out of memory. */ -static int pb_write_tag(lsb_output_data* d, char id, char wire_type) +static int pb_write_tag(lsb_output_data* d, unsigned char id, + unsigned char wire_type) { size_t needed = 1; if (needed > d->size - d->pos) { @@ -111,6 +115,41 @@ static int pb_write_tag(lsb_output_data* d, char id, char wire_type) } +/** + * Updates the field length in the output buffer once the size is known, this + * allows for single pass encoding. + * + * @param d Pointer to the output data buffer. + * @param len_pos Position in the output buffer where the length should be + * written. + * + * @return int Zero on success, non-zero if out of memory. + */ +static int update_field_length(lsb_output_data* d, size_t len_pos) +{ + size_t len = d->pos - len_pos - 1; + if (len < 128) { + d->data[len_pos] = (char)len; + return 0; + } + size_t l = len, cnt = 0; + while (l) { + l >>= 7; + ++cnt; // compute the number of bytes needed for the varint length + } + size_t needed = cnt - 1; + if (needed > d->size - d->pos) { + if (lsb_realloc_output(d, needed)) return 1; + } + size_t end_pos = d->pos + needed; + memmove(&d->data[len_pos + cnt], &d->data[len_pos + 1], len); + d->pos = len_pos; + if (pb_write_varint(d, len)) return 1; + d->pos = end_pos; + return 0; +} + + /** * Writes a string to the output buffer. * @@ -207,12 +246,13 @@ encode_int(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, * @param first Boolean indicator used to add addition protobuf data in the * correct order. * @param representation String representation of the field i.e., "ms" + * @param value_type Protobuf value type * * @return int Zero on success, non-zero if out of memory. */ static int encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, - const char* representation); + const char* representation, int value_type); /** @@ -227,9 +267,11 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, */ static int encode_field_array(lua_sandbox* lsb, lsb_output_data* d, int t, - const char* representation) + const char* representation, int value_type) { int result = 0, first = (int)lua_objlen(lsb->lua, -1); + int multiple = first > 1 ? first : 0; + size_t len_pos = 0; lua_checkstack(lsb->lua, 2); lua_pushnil(lsb->lua); while (result == 0 && lua_next(lsb->lua, -2) != 0) { @@ -237,11 +279,27 @@ encode_field_array(lua_sandbox* lsb, lsb_output_data* d, int t, snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); return 1; } - result = encode_field_value(lsb, d, first, representation); - first = 0; + result = encode_field_value(lsb, d, first, representation, value_type); + if (first) { + len_pos = d->pos; + first = 0; + } lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for // the next interation. } + if (multiple && value_type == 2) { // fix up the varint packed length + int i = len_pos - 2; + int y = 0; + while (d->data[i] != 0 && y < MAX_VARINT_BYTES) { // find the length byte + --i; + ++y; + } + if (y == MAX_VARINT_BYTES || update_field_length(d, i)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "unable set the length of the packed integer array"); + return 1; + } + } return result; } @@ -257,21 +315,26 @@ encode_field_array(lua_sandbox* lsb, lsb_output_data* d, int t, static int encode_field_object(lua_sandbox* lsb, lsb_output_data* d) { int result = 0; + int value_type = -1; const char* representation = NULL; lua_getfield(lsb->lua, -1, "representation"); if (lua_isstring(lsb->lua, -1)) { representation = lua_tostring(lsb->lua, -1); } - lua_getfield(lsb->lua, -2, "value"); - result = encode_field_value(lsb, d, 1, representation); - lua_pop(lsb->lua, 2); // remove representation and value + lua_getfield(lsb->lua, -2, "value_type"); + if (lua_isnumber(lsb->lua, -1)) { + value_type = lua_tointeger(lsb->lua, -1); + } + lua_getfield(lsb->lua, -3, "value"); + result = encode_field_value(lsb, d, 1, representation, value_type); + lua_pop(lsb->lua, 3); // remove representation, value_type and value return result; } static int encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, - const char* representation) + const char* representation, int value_type) { int result = 1; size_t len; @@ -280,20 +343,50 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, int t = lua_type(lsb->lua, -1); switch (t) { case LUA_TSTRING: + switch (value_type) { + case -1: // not specified defaults to string + value_type = 0; + case 0: + case 1: + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid string value_type: %d", value_type); + return 1; + } if (first && representation) { // this uglyness keeps the protobuf // fields in order without additional // lookups if (pb_write_string(d, 3, representation, strlen(representation))) { return 1; } + if (value_type == 1) { + if (pb_write_tag(d, 2, 0)) return 1; + if (pb_write_varint(d, value_type)) return 1; + } } s = lua_tolstring(lsb->lua, -1, &len); - result = pb_write_string(d, 4, s, len); + if (value_type == 1) { + result = pb_write_string(d, 5, s, len); + } else { + result = pb_write_string(d, 4, s, len); + } break; case LUA_TNUMBER: + switch (value_type) { + case -1: // not specified defaults to double + value_type = 3; + case 2: + case 3: + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid numeric value_type: %d", value_type); + return 1; + } if (first) { if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, 3)) return 1; + if (pb_write_varint(d, value_type)) return 1; if (representation) { if (pb_write_string(d, 3, representation, strlen(representation))) { @@ -301,15 +394,34 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, } } if (1 == first) { - if (pb_write_tag(d, 7, 1)) return 1; - } else { - if (pb_write_tag(d, 7, 2)) return 1; - if (pb_write_varint(d, first * sizeof(double))) return 1; + if (value_type == 2) { + if (pb_write_tag(d, 6, 0)) return 1; + } else { + if (pb_write_tag(d, 7, 1)) return 1; + } + } else { // pack array + if (value_type == 2) { + if (pb_write_tag(d, 6, 2)) return 1; + if (pb_write_varint(d, 0)) return 1; // length tbd later + } else { + if (pb_write_tag(d, 7, 2)) return 1; + if (pb_write_varint(d, first * sizeof(double))) return 1; + } } } - result = pb_write_double(d, lua_tonumber(lsb->lua, -1)); + if (value_type == 2) { + result = pb_write_varint(d, lua_tointeger(lsb->lua, -1)); + } else { + result = pb_write_double(d, lua_tonumber(lsb->lua, -1)); + } break; + case LUA_TBOOLEAN: + if (value_type != -1 && value_type != 4) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid boolean value_type: %d", value_type); + return 1; + } if (first) { if (pb_write_tag(d, 2, 0)) return 1; if (pb_write_varint(d, 4)) return 1; @@ -328,6 +440,7 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, } result = pb_write_bool(d, lua_toboolean(lsb->lua, -1)); break; + case LUA_TTABLE: { lua_rawgeti(lsb->lua, -1, 1); @@ -340,7 +453,7 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, case LUA_TNUMBER: case LUA_TSTRING: case LUA_TBOOLEAN: - result = encode_field_array(lsb, d, t, representation); + result = encode_field_array(lsb, d, t, representation, value_type); break; default: snprintf(lsb->error_message, LSB_ERROR_SIZE, @@ -350,6 +463,7 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, } } break; + default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", lua_typename(lsb->lua, t)); @@ -360,41 +474,6 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, } -/** - * Updates the field length in the output buffer once the size is known, this - * allows for single pass encoding. - * - * @param d Pointer to the output data buffer. - * @param len_pos Position in the output buffer where the length should be - * written. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int update_field_length(lsb_output_data* d, size_t len_pos) -{ - size_t len = d->pos - len_pos - 1; - if (len < 128) { - d->data[len_pos] = (char)len; - return 0; - } - size_t l = len, cnt = 0; - while (l) { - l >>= 7; - ++cnt; // compute the number of bytes needed for the varint length - } - size_t needed = cnt - 1; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - size_t end_pos = d->pos + needed; - memmove(&d->data[len_pos + cnt], &d->data[len_pos + 1], len); - d->pos = len_pos; - if (pb_write_varint(d, len)) return 1; - d->pos = end_pos; - return 0; -} - - /** * Iterates over the specified Lua table encoding the contents as user defined * message fields. @@ -417,25 +496,53 @@ encode_fields(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, return result; } + lua_rawgeti(lsb->lua, -1, 1); // test for the array notation size_t len_pos, len; - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - if (pb_write_tag(d, id, 2)) return 1; - len_pos = d->pos; - if (pb_write_varint(d, 0)) return 1; // length tbd later - if (lua_isstring(lsb->lua, -2)) { - const char* s = lua_tolstring(lsb->lua, -2, &len); - if (pb_write_string(d, 1, s, len)) return 1; - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "field name must be a string"); - return 1; + if (lua_istable(lsb->lua, -1)) { + int i = 1; + do { + if (pb_write_tag(d, id, 2)) return 1; + len_pos = d->pos; + if (pb_write_varint(d, 0)) return 1; // length tbd later + lua_getfield(lsb->lua, -1, "name"); + if (lua_isstring(lsb->lua, -1)) { + const char* s = lua_tolstring(lsb->lua, -1, &len); + if (pb_write_string(d, 1, s, len)) return 1; + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "field name must be a string"); + return 1; + } + lua_pop(lsb->lua, 1); // remove the name + + if (encode_field_object(lsb, d)) return 1; + if (update_field_length(d, len_pos)) return 1; + + lua_pop(lsb->lua, 1); // remove the current field object + lua_rawgeti(lsb->lua, -1, ++i); // grab the next field object + } + while (!lua_isnil(lsb->lua, -1)); + } else { + lua_pop(lsb->lua, 1); // remove the array test value + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (result == 0 && lua_next(lsb->lua, -2) != 0) { + if (pb_write_tag(d, id, 2)) return 1; + len_pos = d->pos; + if (pb_write_varint(d, 0)) return 1; // length tbd later + if (lua_isstring(lsb->lua, -2)) { + const char* s = lua_tolstring(lsb->lua, -2, &len); + if (pb_write_string(d, 1, s, len)) return 1; + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "field name must be a string"); + return 1; + } + if (encode_field_value(lsb, d, 1, NULL, -1)) return 1; + if (update_field_length(d, len_pos)) return 1; + lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for + // the next interation. } - if (encode_field_value(lsb, d, 1, NULL)) return 1; - if (update_field_length(d, len_pos)) return 1; - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. } lua_pop(lsb->lua, 1); // remove the fields table return result; @@ -451,14 +558,24 @@ int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index) if (lsb_realloc_output(d, needed)) return 1; } - // create a type 4 uuid + // use existing or create a type 4 uuid + lua_getfield(lsb->lua, index, "Uuid"); + size_t len; + const char* uuid = lua_tolstring(lsb->lua, -1, &len); + d->data[d->pos++] = 2 | (1 << 3); - d->data[d->pos++] = 16; - for (int x = 0; x < 16; ++x) { - d->data[d->pos++] = rand() % 255; + d->data[d->pos++] = UUID_SIZE; + if (uuid && len == UUID_SIZE) { + memcpy(d->data + d->pos, uuid, UUID_SIZE); + d->pos += UUID_SIZE; + } else { + for (int x = 0; x < UUID_SIZE; ++x) { + d->data[d->pos++] = rand() % 255; + } + d->data[8] = (d->data[8] & 0x0F) | 0x40; + d->data[10] = (d->data[10] & 0x0F) | 0xA0; } - d->data[8] = (d->data[8] & 0x0F) | 0x40; - d->data[10] = (d->data[10] & 0x0F) | 0xA0; + lua_pop(lsb->lua, 1); // remove uuid // use existing or create a timestamp lua_getfield(lsb->lua, index, "Timestamp"); @@ -468,7 +585,8 @@ int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index) } else { ts = (unsigned long long)(time(NULL) * 1e9); } - lua_pop(lsb->lua, 1); + lua_pop(lsb->lua, 1); // remove timestamp + if (pb_write_tag(d, 2, 0)) return 1; if (pb_write_varint(d, ts)) return 1; diff --git a/src/test/lua/decode_message.lua b/src/test/lua/decode_message.lua new file mode 100644 index 0000000..775f817 --- /dev/null +++ b/src/test/lua/decode_message.lua @@ -0,0 +1,201 @@ +-- minimal message, required fields only +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\000" +local msg = assert(decode_message(test)) +assert(msg.Timestamp == 0) + +-- message headers only +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\003\026\004type\034\006logger\040\009\050\007payload\058\011env_version\074\008hostname" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Type == "type") +assert(msg.Logger == "logger") +assert(msg.Payload == "payload") +assert(msg.EnvVersion == "env_version") +assert(msg.Hostname == "hostname") +assert(msg.Severity == 9) + +-- repeated unpacked doubles +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\027\010\005count\016\003\057\000\000\000\000\000\000\240\063\057\000\000\000\000\000\000\240\063" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 3) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated packed doubles +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\027\010\005count\016\003\058\016\000\000\000\000\000\000\240\063\000\000\000\000\000\000\240\063" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 3) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated unpacked ints +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\048\001\048\001" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 2) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated packed ints +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\050\002\001\001" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 2) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated unpacked bools +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\004\064\001\064\000" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 4) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == true) +assert(msg.Fields[1].value[2] == false) + +-- repeated packed bools +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\004\066\002\001\000" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 4) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == true) +assert(msg.Fields[1].value[2] == false) + +-- repeated strings +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\023\010\005names\016\000\034\002s1\034\002s2\026\004keys" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "names") +assert(msg.Fields[1].representation == "keys") +assert(msg.Fields[1].value_type == 0) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == "s1") +assert(msg.Fields[1].value[2] == "s2") + +-- repeated bytes +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "names") +assert(msg.Fields[1].representation == "keys") +assert(msg.Fields[1].value_type == 1) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == "s1") +assert(msg.Fields[1].value[2] == "s2") + +-- benchmark message +test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" +msg = assert(decode_message(test)) +assert(msg.Timestamp == 1e9) +assert(#msg.Fields == 6) + +assert(#msg.Fields[1].value == 1) +assert(msg.Fields[1].name == "number") +assert(msg.Fields[1].value_type == 3) +assert(msg.Fields[1].value[1] == 1) + +assert(msg.Fields[2].name == "numbers") +assert(msg.Fields[2].value_type == 3) +assert(#msg.Fields[2].value == 3) +assert(msg.Fields[2].value[1] == 1) +assert(msg.Fields[2].value[2] == 2) +assert(msg.Fields[2].value[3] == 3) +assert(msg.Fields[2].representation == "count") + +assert(#msg.Fields[3].value == 3) +assert(msg.Fields[3].name == "bools") +assert(msg.Fields[3].value[1]) +assert(msg.Fields[3].value[2] == false) +assert(msg.Fields[3].value[3] == false) + +assert(#msg.Fields[4].value == 1) +assert(msg.Fields[4].name == "bool") +assert(msg.Fields[4].value[1]) + +assert(#msg.Fields[5].value == 1) +assert(msg.Fields[5].name == "string") +assert(msg.Fields[5].value[1] == "string") + +assert(msg.Fields[6].name == "strings") +assert(#msg.Fields[6].value == 3) +assert(msg.Fields[6].value[1] =="s1") +assert(msg.Fields[6].value[2] =="s2") +assert(msg.Fields[6].value[3] =="s3") + +-- test negative varint headers +test = "\010\016\243\083\052\234\016\052\066\236\160\084\236\003\227\231\170\203\016\255\255\255\255\255\255\255\255\255\001\040\255\255\255\255\255\255\255\255\255\001\064\255\255\255\255\255\255\255\255\255\001" +msg = assert(decode_message(test)) +assert(msg.Timestamp == -1) +assert(msg.Pid == -1) +assert(msg.Severity == -1) + +-- multi byte length +test = "\010\016\104\183\062\106\120\227\073\248\173\168\122\019\057\236\124\025\016\128\148\235\220\003\082\141\001\010\006\115\116\114\105\110\103\034\130\001\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057" +msg = assert(decode_message(test)) +assert(msg.Fields[1].value[1] == "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") + + +--[[ +Parse Errors +--]] + +-- too short +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132" +msg, err = decode_message(test) +assert(not msg) +assert(err == "invalid message, too short", err) + +-- no uuid +test = "\016\128\148\235\220\003\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" +msg, err = decode_message(test) +assert(not msg) +assert(err == "missing required field uuid: not found timestamp: found", err) + +-- no timestamp +local test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" +msg, err = decode_message(test) +assert(not msg) +assert(err == "missing required field uuid: found timestamp: not found", err) + +-- missing field name +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\020\016\003\057\000\000\000\000\000\000\240\063\057\000\000\000\000\000\000\240\063" +msg, err = decode_message(test) +assert(not msg) +assert(err == "error in tag: 10 wiretype: 2 offset: 24", err) + +-- random string +test = "this is a test item over twenty bytes long" +msg, err = decode_message(test) +assert(not msg) +assert(err == "error in tag: 14 wiretype: 4 offset: 0", err) + +-- repeated packed with ints invalid length +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\050\003\001\001" +msg, err = decode_message(test) +assert(not msg) +assert(err == "error in tag: 10 wiretype: 2 offset: 24", err) + +-- invalid timestamp varint encoding +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\220\220\220\220\220\220" +msg, err = decode_message(test) +assert(not msg) +assert(err == "error in tag: 2 wiretype: 0 offset: 18", err) + +-- invalid length encoding +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\003\026\128\148\235\220\220\220\220\220\220\220type" +msg, err = decode_message(test) +assert(not msg) +assert(err == "error in tag: 3 wiretype: 2 offset: 24", err) diff --git a/src/test/lua/decode_message_benchmark.lua b/src/test/lua/decode_message_benchmark.lua new file mode 100644 index 0000000..e704640 --- /dev/null +++ b/src/test/lua/decode_message_benchmark.lua @@ -0,0 +1,6 @@ +local test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" + +function process(tc) + assert(decode_message(test)) + return 0 +end diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 9ffd331..496f5b9 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -14,7 +14,7 @@ function process(tc) elseif tc == 1 then -- cbuf write_output("annotation\n", cbuf) elseif tc == 2 then -- heka message - local hm = {Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9} + local hm = {Uuid = "ignored", Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9} write_message(hm) elseif tc == 3 then -- heka message field local hm = {Timestamp = 1e9, Fields = {count=1}} @@ -37,6 +37,30 @@ function process(tc) elseif tc == 9 then -- heka negative values local hm = {Timestamp = -1, Pid = -1, Severity = -1} write_message(hm) + elseif tc == 10 then -- heka varint encoding + local hm = {Timestamp = 1e9, Fields = { count = {value=1, value_type=2, representation="integer"}}} + write_message(hm) + elseif tc == 11 then -- heka varint packed encoding + local hm = {Timestamp = 1e9, Fields = { count = {value={1,2}, value_type=2, representation="integer"}}} + write_message(hm) + elseif tc == 12 then -- heka field array message + local hm = {Timestamp = 1e9, Fields = {{name = "count", value=1, value_type=2, representation="integer"}}} + write_message(hm) + elseif tc == 13 then -- heka field array message multiple fields + local hm = {Timestamp = 1e9, Fields = {{name = "count", value=1, value_type=2, representation="integer"}, {name = "count", value=2, value_type=2, representation="integer"}}} + write_message(hm) + elseif tc == 14 then -- heka field array message value defaults to a double + local hm = {Timestamp = 1e9, Fields = {{name = "count", value=1, representation="double"}}} + write_message(hm) + elseif tc == 15 then -- heka field array message value defaults to a double packed + local hm = {Timestamp = 1e9, Fields = {{name = "count", value={1,1}, representation="double"}}} + write_message(hm) + elseif tc == 16 then -- heka array of byte fields + local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 1, representation="list"}}} + write_message(hm) + elseif tc == 17 then -- heka array of explicit string fields + local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 0, representation="list"}}} + write_message(hm) end return 0 end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 2ccf9a2..924ba6b 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -36,6 +36,36 @@ function process(tc) cjson.encode(t) elseif tc == 8 then -- invalid type write_message("string") + elseif tc == 9 then -- invalid value_type integer + local hm = {Timestamp = 1e9, Fields = {counts={value="s", value_type=2}}} + write_message(hm) + elseif tc == 10 then -- invalid value_type double + local hm = {Timestamp = 1e9, Fields = {counts={value="s", value_type=3}}} + write_message(hm) + elseif tc == 11 then -- invalid value_type bool + local hm = {Timestamp = 1e9, Fields = {counts={value="s", value_type=4}}} + write_message(hm) + elseif tc == 12 then -- invalid value_type string + local hm = {Timestamp = 1e9, Fields = {counts={value=1, value_type=0}}} + write_message(hm) + elseif tc == 13 then -- invalid value_type bytes + local hm = {Timestamp = 1e9, Fields = {counts={value=1, value_type=1}}} + write_message(hm) + elseif tc == 14 then -- invalid value_type bool + local hm = {Timestamp = 1e9, Fields = {counts={value=1, value_type=4}}} + write_message(hm) + elseif tc == 15 then -- invalid value_type string + local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=0}}} + write_message(hm) + elseif tc == 16 then -- invalid value_type bytes + local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=1}}} + write_message(hm) + elseif tc == 17 then -- invalid value_type integer + local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}} + write_message(hm) + elseif tc == 18 then -- invalid value_type double + local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=3}}} + write_message(hm) end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 1b39b5f..a65fee2 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -430,6 +430,14 @@ static char* test_output() #endif , "\x10\x80\x94\xeb\xdc\x03\x52\x8d\x01\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x82\x01\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" , "\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x40\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" + , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01" + , "\x10\x80\x94\xeb\xdc\x03\x52\x16\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x32\x02\x01\x02" + , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01" + , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x02" + , "\x10\x80\x94\xeb\xdc\x03\x52\x1a\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x06\x64\x6f\x75\x62\x6c\x65\x39\x00\x00\x00\x00\x00\x00\xf0\x3f" + , "\x10\x80\x94\xeb\xdc\x03\x52\x23\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x06\x64\x6f\x75\x62\x6c\x65\x3a\x10\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\xf0\x3f" + , "\x10\x80\x94\xeb\xdc\x03\x52\x17\x0a\x05\x6e\x61\x6d\x65\x73\x1a\x04\x6c\x69\x73\x74\x10\x01\x2a\x02\x73\x31\x2a\x02\x73\x32" + , "\x10\x80\x94\xeb\xdc\x03\x52\x15\x0a\x05\x6e\x61\x6d\x65\x73\x1a\x04\x6c\x69\x73\x74\x22\x02\x73\x31\x22\x02\x73\x32" , NULL }; @@ -490,6 +498,16 @@ static char* test_output_errors() , "process() lua/output_errors.lua:30: write_message() could not encode protobuf - unsupported array type: table" , "process() lua/output_errors.lua:36: strbuf output_limit exceeded" , "process() lua/output_errors.lua:38: write_message() could not encode protobuf - takes a single table argument" + , "process() lua/output_errors.lua:41: write_message() could not encode protobuf - invalid string value_type: 2" + , "process() lua/output_errors.lua:44: write_message() could not encode protobuf - invalid string value_type: 3" + , "process() lua/output_errors.lua:47: write_message() could not encode protobuf - invalid string value_type: 4" + , "process() lua/output_errors.lua:50: write_message() could not encode protobuf - invalid numeric value_type: 0" + , "process() lua/output_errors.lua:53: write_message() could not encode protobuf - invalid numeric value_type: 1" + , "process() lua/output_errors.lua:56: write_message() could not encode protobuf - invalid numeric value_type: 4" + , "process() lua/output_errors.lua:59: write_message() could not encode protobuf - invalid boolean value_type: 0" + , "process() lua/output_errors.lua:62: write_message() could not encode protobuf - invalid boolean value_type: 1" + , "process() lua/output_errors.lua:65: write_message() could not encode protobuf - invalid boolean value_type: 2" + , "process() lua/output_errors.lua:68: write_message() could not encode protobuf - invalid boolean value_type: 3" , NULL }; @@ -1109,6 +1127,24 @@ static char* test_sandbox_config() } +static char* test_decode_message() +{ + lua_sandbox* sb = lsb_create(NULL, "lua/decode_message.lua", "modules", 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_add_function(sb, &lsb_decode_protobuf, "decode_message"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -1375,6 +1411,36 @@ static char* benchmark_hyperloglog_add() } +static char* benchmark_decode_message() +{ + int iter = 10000; + + lua_sandbox* sb = lsb_create(NULL, "lua/decode_message_benchmark.lua", "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + + lsb_add_function(sb, &lsb_decode_protobuf, "decode_message"); + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed + } + t = clock() - t; + report(sb, 0); + mu_assert(lsb_get_state(sb) == LSB_RUNNING, + "benchmark_decode_message() failed %s", lsb_get_error(sb)); + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + printf("benchmark_decode_message() %g seconds\n", ((float)t) + / CLOCKS_PER_SEC / iter); + + return NULL; +} + + static char* all_tests() { mu_run_test(test_create_error); @@ -1401,6 +1467,7 @@ static char* all_tests() mu_run_test(test_hyperloglog); mu_run_test(test_struct); mu_run_test(test_sandbox_config); + mu_run_test(test_decode_message); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); @@ -1412,6 +1479,7 @@ static char* all_tests() mu_run_test(benchmark_cbuf_add); mu_run_test(benchmark_bloom_filter_add); mu_run_test(benchmark_hyperloglog_add); + mu_run_test(benchmark_decode_message); return NULL; } From aefd0fb8d62f2a8dfc0e5dda3060179281d3ad58 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 17 Feb 2015 11:24:08 -0800 Subject: [PATCH 044/235] Issue #71 remove strict.lua --- README.md | 1 - src/CMakeLists.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 576cdf1..7d647b9 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,6 @@ By default only the base library is loaded additional libraries must be loaded w - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. - Disabled functions (default): getenv, execute, exit, remove, rename, setlocale, tmpname. - - [strict](http://www.lua.org/extras/5.2/strict.lua) Checks for use of undeclared global variables. - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs - [table](http://www.lua.org/manual/5.1/manual.html#5.5) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3eb3a3d..1b791c2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -39,7 +39,7 @@ target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) install(TARGETS luasandbox DESTINATION lib) install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib PATTERN "lua" EXCLUDE) -install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION modules) +install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION modules PATTERN "strict.lua" EXCLUDE) install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) From 272955d7258d4bb833352383f8871bd0bce9c381 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 20 Feb 2015 09:02:49 -0800 Subject: [PATCH 045/235] Update the lsb_output_protobuf precondition check and error message --- src/lsb_output.c | 5 ++--- src/test/test_lua_sandbox.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lsb_output.c b/src/lsb_output.c index 0738fee..7f44d29 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -213,9 +213,8 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) int lsb_output_protobuf(lua_sandbox* lsb, int index, int append) { - if (lua_gettop(lsb->lua) != 1 || lua_type(lsb->lua, 1) != LUA_TTABLE) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "takes a single table argument"); + if (lua_type(lsb->lua, index) != LUA_TTABLE) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, "takes a table argument"); return 1; } diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index a65fee2..eec7ebb 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -497,7 +497,7 @@ static char* test_output_errors() , "process() lua/output_errors.lua:27: output_limit exceeded" , "process() lua/output_errors.lua:30: write_message() could not encode protobuf - unsupported array type: table" , "process() lua/output_errors.lua:36: strbuf output_limit exceeded" - , "process() lua/output_errors.lua:38: write_message() could not encode protobuf - takes a single table argument" + , "process() lua/output_errors.lua:38: write_message() could not encode protobuf - takes a table argument" , "process() lua/output_errors.lua:41: write_message() could not encode protobuf - invalid string value_type: 2" , "process() lua/output_errors.lua:44: write_message() could not encode protobuf - invalid string value_type: 3" , "process() lua/output_errors.lua:47: write_message() could not encode protobuf - invalid string value_type: 4" From f9ad136db6d9efb8d03a7f1f68e277e9089581f4 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 25 Feb 2015 09:52:33 -0800 Subject: [PATCH 046/235] Issue #74 decode_message should throw an error instead of returning nil --- src/lsb_deserialize_protobuf.c | 18 ++---- src/test/lua/decode_message.lua | 74 +++++++++++------------ src/test/lua/decode_message_benchmark.lua | 2 +- 3 files changed, 44 insertions(+), 50 deletions(-) diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c index c39b820..4c86a51 100644 --- a/src/lsb_deserialize_protobuf.c +++ b/src/lsb_deserialize_protobuf.c @@ -269,9 +269,7 @@ int lsb_decode_protobuf(lua_State* lua) size_t len; const char* pbstr = lua_tolstring(lua, 1, &len); if (len < 20) { - lua_pushnil(lua); - lua_pushstring(lua, "invalid message, too short"); - return 2; + return luaL_error(lua, "invalid message, too short"); } unsigned const char* p = (unsigned const char*)pbstr; @@ -371,18 +369,14 @@ int lsb_decode_protobuf(lua_State* lua) while (p && p < e); if (!p) { - lua_pushnil(lua); - lua_pushfstring(lua, "error in tag: %d wiretype: %d offset: %d", tag, - wiretype, (const char*)lp - pbstr); - return 2; + return luaL_error(lua, "error in tag: %d wiretype: %d offset: %d", tag, + wiretype, (const char*)lp - pbstr); } if (!(has_uuid && has_timestamp)) { - lua_pushnil(lua); - lua_pushfstring(lua, "missing required field uuid: %s timestamp: %s", - has_uuid ? "found" : "not found", - has_timestamp ? "found" : "not found"); - return 2; + return luaL_error(lua, "missing required field uuid: %s timestamp: %s", + has_uuid ? "found" : "not found", + has_timestamp ? "found" : "not found"); } if (field_count) { diff --git a/src/test/lua/decode_message.lua b/src/test/lua/decode_message.lua index 775f817..240e870 100644 --- a/src/test/lua/decode_message.lua +++ b/src/test/lua/decode_message.lua @@ -1,11 +1,11 @@ -- minimal message, required fields only local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\000" -local msg = assert(decode_message(test)) +local msg = decode_message(test) assert(msg.Timestamp == 0) -- message headers only local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\003\026\004type\034\006logger\040\009\050\007payload\058\011env_version\074\008hostname" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Type == "type") assert(msg.Logger == "logger") @@ -16,7 +16,7 @@ assert(msg.Severity == 9) -- repeated unpacked doubles test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\027\010\005count\016\003\057\000\000\000\000\000\000\240\063\057\000\000\000\000\000\000\240\063" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "count") assert(msg.Fields[1].value_type == 3) @@ -26,7 +26,7 @@ assert(msg.Fields[1].value[2] == 1) -- repeated packed doubles test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\027\010\005count\016\003\058\016\000\000\000\000\000\000\240\063\000\000\000\000\000\000\240\063" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "count") assert(msg.Fields[1].value_type == 3) @@ -36,7 +36,7 @@ assert(msg.Fields[1].value[2] == 1) -- repeated unpacked ints test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\048\001\048\001" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "count") assert(msg.Fields[1].value_type == 2) @@ -46,7 +46,7 @@ assert(msg.Fields[1].value[2] == 1) -- repeated packed ints test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\050\002\001\001" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "count") assert(msg.Fields[1].value_type == 2) @@ -56,7 +56,7 @@ assert(msg.Fields[1].value[2] == 1) -- repeated unpacked bools test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\004\064\001\064\000" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "count") assert(msg.Fields[1].value_type == 4) @@ -66,7 +66,7 @@ assert(msg.Fields[1].value[2] == false) -- repeated packed bools test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\004\066\002\001\000" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "count") assert(msg.Fields[1].value_type == 4) @@ -76,7 +76,7 @@ assert(msg.Fields[1].value[2] == false) -- repeated strings test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\023\010\005names\016\000\034\002s1\034\002s2\026\004keys" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "names") assert(msg.Fields[1].representation == "keys") @@ -87,7 +87,7 @@ assert(msg.Fields[1].value[2] == "s2") -- repeated bytes test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(msg.Fields[1].name == "names") assert(msg.Fields[1].representation == "keys") @@ -98,7 +98,7 @@ assert(msg.Fields[1].value[2] == "s2") -- benchmark message test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == 1e9) assert(#msg.Fields == 6) @@ -137,14 +137,14 @@ assert(msg.Fields[6].value[3] =="s3") -- test negative varint headers test = "\010\016\243\083\052\234\016\052\066\236\160\084\236\003\227\231\170\203\016\255\255\255\255\255\255\255\255\255\001\040\255\255\255\255\255\255\255\255\255\001\064\255\255\255\255\255\255\255\255\255\001" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Timestamp == -1) assert(msg.Pid == -1) assert(msg.Severity == -1) -- multi byte length test = "\010\016\104\183\062\106\120\227\073\248\173\168\122\019\057\236\124\025\016\128\148\235\220\003\082\141\001\010\006\115\116\114\105\110\103\034\130\001\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057" -msg = assert(decode_message(test)) +msg = decode_message(test) assert(msg.Fields[1].value[1] == "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") @@ -154,48 +154,48 @@ Parse Errors -- too short test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132" -msg, err = decode_message(test) -assert(not msg) -assert(err == "invalid message, too short", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "invalid message, too short", msg) -- no uuid test = "\016\128\148\235\220\003\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" -msg, err = decode_message(test) -assert(not msg) -assert(err == "missing required field uuid: not found timestamp: found", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "missing required field uuid: not found timestamp: found", msg) -- no timestamp local test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" -msg, err = decode_message(test) -assert(not msg) -assert(err == "missing required field uuid: found timestamp: not found", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "missing required field uuid: found timestamp: not found", msg) -- missing field name test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\020\016\003\057\000\000\000\000\000\000\240\063\057\000\000\000\000\000\000\240\063" -msg, err = decode_message(test) -assert(not msg) -assert(err == "error in tag: 10 wiretype: 2 offset: 24", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 10 wiretype: 2 offset: 24", msg) -- random string test = "this is a test item over twenty bytes long" -msg, err = decode_message(test) -assert(not msg) -assert(err == "error in tag: 14 wiretype: 4 offset: 0", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 14 wiretype: 4 offset: 0", msg) -- repeated packed with ints invalid length test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\050\003\001\001" -msg, err = decode_message(test) -assert(not msg) -assert(err == "error in tag: 10 wiretype: 2 offset: 24", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 10 wiretype: 2 offset: 24", msg) -- invalid timestamp varint encoding local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\220\220\220\220\220\220" -msg, err = decode_message(test) -assert(not msg) -assert(err == "error in tag: 2 wiretype: 0 offset: 18", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 2 wiretype: 0 offset: 18", msg) -- invalid length encoding local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\003\026\128\148\235\220\220\220\220\220\220\220type" -msg, err = decode_message(test) -assert(not msg) -assert(err == "error in tag: 3 wiretype: 2 offset: 24", err) +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 3 wiretype: 2 offset: 24", msg) diff --git a/src/test/lua/decode_message_benchmark.lua b/src/test/lua/decode_message_benchmark.lua index e704640..fc22337 100644 --- a/src/test/lua/decode_message_benchmark.lua +++ b/src/test/lua/decode_message_benchmark.lua @@ -1,6 +1,6 @@ local test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" function process(tc) - assert(decode_message(test)) + local msg = decode_message(test) return 0 end From 12de392f859013e28d20084b33e33eebf41bd0fa Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 26 Feb 2015 10:28:28 -0800 Subject: [PATCH 047/235] Issue #79 Fix the value_type for bytes without a representation --- src/lsb_serialize_protobuf.c | 13 +++++++------ src/test/lua/output.lua | 3 +++ src/test/test_lua_sandbox.c | 3 ++- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c index 3b8a3fa..0d9ff39 100644 --- a/src/lsb_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -354,16 +354,17 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, "invalid string value_type: %d", value_type); return 1; } - if (first && representation) { // this uglyness keeps the protobuf - // fields in order without additional - // lookups - if (pb_write_string(d, 3, representation, strlen(representation))) { - return 1; - } + if (first) { // this uglyness keeps the protobuf fields in order without + // additional lookups if (value_type == 1) { if (pb_write_tag(d, 2, 0)) return 1; if (pb_write_varint(d, value_type)) return 1; } + if (representation) { + if (pb_write_string(d, 3, representation, strlen(representation))) { + return 1; + } + } } s = lua_tolstring(lsb->lua, -1, &len); if (value_type == 1) { diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 496f5b9..f737353 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -61,6 +61,9 @@ function process(tc) elseif tc == 17 then -- heka array of explicit string fields local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 0, representation="list"}}} write_message(hm) + elseif tc == 18 then -- heka array of byte fields no representation + local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 1}}} + write_message(hm) end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index eec7ebb..a950ab0 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -436,8 +436,9 @@ static char* test_output() , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x02" , "\x10\x80\x94\xeb\xdc\x03\x52\x1a\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x06\x64\x6f\x75\x62\x6c\x65\x39\x00\x00\x00\x00\x00\x00\xf0\x3f" , "\x10\x80\x94\xeb\xdc\x03\x52\x23\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x06\x64\x6f\x75\x62\x6c\x65\x3a\x10\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\xf0\x3f" - , "\x10\x80\x94\xeb\xdc\x03\x52\x17\x0a\x05\x6e\x61\x6d\x65\x73\x1a\x04\x6c\x69\x73\x74\x10\x01\x2a\x02\x73\x31\x2a\x02\x73\x32" + , "\x10\x80\x94\xeb\xdc\x03\x52\x17\x0a\x05\x6e\x61\x6d\x65\x73\x10\x01\x1a\x04\x6c\x69\x73\x74\x2a\x02\x73\x31\x2a\x02\x73\x32" , "\x10\x80\x94\xeb\xdc\x03\x52\x15\x0a\x05\x6e\x61\x6d\x65\x73\x1a\x04\x6c\x69\x73\x74\x22\x02\x73\x31\x22\x02\x73\x32" + , "\x10\x80\x94\xeb\xdc\x03\x52\x11\x0a\x05\x6e\x61\x6d\x65\x73\x10\x01\x2a\x02\x73\x31\x2a\x02\x73\x32" , NULL }; From f20a975af2abc0ad6a84742b8576f5847cd0852d Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 27 Feb 2015 07:07:25 -0800 Subject: [PATCH 048/235] Issue #78 allow user data to be added to a message structure --- cmake/externals.cmake | 2 +- src/lsb_output.c | 25 +++++++++++++------------ src/lsb_private.h | 10 ++++++++++ src/lsb_serialize_protobuf.c | 28 ++++++++++++++++++++++++++++ src/test/lua/output.lua | 9 +++++++++ src/test/lua/output_errors.lua | 6 +++++- src/test/test_lua_sandbox.c | 9 ++++++++- 7 files changed, 74 insertions(+), 15 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 887b273..2e896d0 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -112,7 +112,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG c9e3a7310b6e65a1ed54bc073ca6b836edb84cd5 + GIT_TAG 78c32961102d977cef5afac2c20a3fd9f5eae808 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/src/lsb_output.c b/src/lsb_output.c index 7f44d29..593ef5d 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -20,17 +20,6 @@ static const char* output_function = "lsb_output"; -static lua_CFunction get_output_function(lua_State* lua, int index) -{ - lua_CFunction fp = NULL; - lua_getfenv(lua, index); - lua_pushstring(lua, output_function); - lua_rawget(lua, -2); - fp = lua_tocfunction(lua, -1); - lua_pop(lua, 2); // environment and field - return fp; -} - static void update_output_stats(lua_sandbox* lsb) { @@ -51,6 +40,18 @@ void lsb_add_output_function(lua_State* lua, lua_CFunction fp) } +lua_CFunction lsb_get_output_function(lua_State* lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, output_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + + int lsb_appendc(lsb_output_data* output, char ch) { size_t needed = 2; @@ -186,7 +187,7 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) break; case LUA_TUSERDATA: { - lua_CFunction fp = get_output_function(lsb->lua, i); + lua_CFunction fp = lsb_get_output_function(lsb->lua, i); if (!fp) { luaL_argerror(lsb->lua, i, "unknown userdata type"); return; // never reaches here but the compiler doesn't know it diff --git a/src/lsb_private.h b/src/lsb_private.h index 26b1f00..344877f 100644 --- a/src/lsb_private.h +++ b/src/lsb_private.h @@ -37,4 +37,14 @@ struct lua_sandbox { char error_message[LSB_ERROR_SIZE]; }; +/** + * Utility function to retrieve a user data output function + * + * @param lua + * @param index + * + * @return lua_CFunction + */ +lua_CFunction lsb_get_output_function(lua_State* lua, int index); + #endif diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c index 0d9ff39..0548114 100644 --- a/src/lsb_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -465,6 +465,34 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, } break; + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_output_function(lsb->lua, -1); + if (!fp) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "user data object does not implement lsb_output"); + return 1; + } + if (first) { + if (pb_write_tag(d, 2, 0)) return 1; + if (pb_write_varint(d, 1)) return 1; // encode userdata as a byte array + if (representation) { + if (pb_write_string(d, 3, representation, strlen(representation))) { + return 1; + } + } + } + + if (pb_write_tag(d, 5, 2)) return 1; + size_t len_pos = d->pos; + if (pb_write_varint(d, 0)) return 1; // length tbd later + lua_pushlightuserdata(lsb->lua, d); + if (fp(lsb->lua)) return 1; + lua_pop(lsb->lua, 1); // remove output function + result = update_field_length(d, len_pos); + } + break; + default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", lua_typename(lsb->lua, t)); diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index f737353..2de5bac 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -4,6 +4,7 @@ require "circular_buffer" require "cjson" +require "hyperloglog" local cbuf = circular_buffer.new(1440, 3, 60) local benchmark = {Timestamp = 1e9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} @@ -64,6 +65,14 @@ function process(tc) elseif tc == 18 then -- heka array of byte fields no representation local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 1}}} write_message(hm) + elseif tc == 19 then + local hll = hyperloglog.new() + local hm = {Timestamp = 1e9, Fields = {hll = {value=hll, representation="hll"}}} + write_message(hm) + elseif tc == 20 then + local cb = circular_buffer.new(2, 1, 60) + local hm = {Timestamp = 1e9, Fields = {cb = {value=cb, representation="cbuf"}}} + write_message(hm) end return 0 end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 924ba6b..86a174c 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -3,7 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "cjson" -require "circular_buffer" +require "circular_buffer"; require "bloom_filter" require "lpeg" function process(tc) if tc == 0 then -- error internal reference @@ -66,6 +66,10 @@ function process(tc) elseif tc == 18 then -- invalid value_type double local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=3}}} write_message(hm) + elseif tc == 19 then -- bloom_filter doesn't implement lsb_output + local bf = bloom_filter.new(10, 0.01) + local hm = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}} + write_message(hm) end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index a950ab0..d56773e 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -439,6 +439,8 @@ static char* test_output() , "\x10\x80\x94\xeb\xdc\x03\x52\x17\x0a\x05\x6e\x61\x6d\x65\x73\x10\x01\x1a\x04\x6c\x69\x73\x74\x2a\x02\x73\x31\x2a\x02\x73\x32" , "\x10\x80\x94\xeb\xdc\x03\x52\x15\x0a\x05\x6e\x61\x6d\x65\x73\x1a\x04\x6c\x69\x73\x74\x22\x02\x73\x31\x22\x02\x73\x32" , "\x10\x80\x94\xeb\xdc\x03\x52\x11\x0a\x05\x6e\x61\x6d\x65\x73\x10\x01\x2a\x02\x73\x31\x2a\x02\x73\x32" + , "\x10\x80\x94\xeb\xdc\x03\x52\x9f\x60\x0a\x03\x68\x6c\x6c\x10\x01\x1a\x03\x68\x6c\x6c\x2a\x90\x60\x48\x59\x4c\x4c" + , "\x10\x80\x94\xeb\xdc\x03\x52\x93\x01\x0a\x02\x63\x62\x10\x01\x1a\x04\x63\x62\x75\x66\x2a\x84\x01{\"time\":0,\"rows\":2,\"columns\":1,\"seconds_per_row\":60,\"column_info\":[{\"name\":\"Column_1\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\nnan\n" , NULL }; @@ -461,9 +463,13 @@ static char* test_output() if (outputs[x][0]) { if (outputs[x][0] == 0x10) { size_t header = 18; + if (x == 19) { + mu_assert(written_data_len == 12346, "test: %d received: %zu", x, written_data_len); + written_data_len = header + 28; // just compare the protobuf before the hyperloglog + } if (memcmp(outputs[x], written_data + header, written_data_len - header) != 0) { - char hex_data[output_size + 1]; + char hex_data[output_size * 3 + 1]; size_t z = 0; for (size_t y = header; y < written_data_len; ++y, z += 3) { snprintf(hex_data + z, output_size - z, "%02x ", @@ -509,6 +515,7 @@ static char* test_output_errors() , "process() lua/output_errors.lua:62: write_message() could not encode protobuf - invalid boolean value_type: 1" , "process() lua/output_errors.lua:65: write_message() could not encode protobuf - invalid boolean value_type: 2" , "process() lua/output_errors.lua:68: write_message() could not encode protobuf - invalid boolean value_type: 3" + , "process() lua/output_errors.lua:72: write_message() could not encode protobuf - user data object does not implement lsb_output" , NULL }; From 83e72ff6f5205d0e1c5d8395cf08ac9cc35b8594 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 8 Mar 2015 22:17:02 -0700 Subject: [PATCH 049/235] Don't fail if the data restoration file does not exist --- src/lsb_serialize.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index ba0db2b..b69efc3 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -474,8 +474,21 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) } +int file_exists(const char* fn) +{ + FILE* fh = fopen(fn, "r"); + if (fh) { + fclose(fh); + return 1; + } + return 0; +} + + int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) { + if (!file_exists(data_file)) return 0; + // Clear the sandbox limits during restoration. #ifdef LUA_JIT lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); From e9e8082d247b7d8b639094624a595c43da3a1f56 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 9 Mar 2015 08:40:19 -0700 Subject: [PATCH 050/235] Correct the scope on the file_exists function --- src/lsb_serialize.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index b69efc3..adf7372 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -474,7 +474,7 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) } -int file_exists(const char* fn) +static int file_exists(const char* fn) { FILE* fh = fopen(fn, "r"); if (fh) { From 555a0ed0537710e7911c578e7f56cf73f2630eb1 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 20 Mar 2015 08:35:53 -0700 Subject: [PATCH 051/235] Pull in the new cuckoo filter and bloom filter updates --- README.md | 1 + cmake/externals.cmake | 10 +- src/test/lua/bloom_filter.lua | 6 +- src/test/lua/bloom_filter_benchmark.lua | 10 +- src/test/lua/cuckoo_filter.lua | 30 ++++++ src/test/lua/cuckoo_filter_benchmark.lua | 17 ++++ src/test/lua/output_errors.lua | 6 +- src/test/test_lua_sandbox.c | 120 ++++++++++++++++++++++- 8 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 src/test/lua/cuckoo_filter.lua create mode 100644 src/test/lua/cuckoo_filter_benchmark.lua diff --git a/README.md b/README.md index 7d647b9..762b1ba 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ By default only the base library is loaded additional libraries must be loaded w If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - The new() function has been disabled so only a single cjson parser can be created. - The encode_keep_buffer() function has been disabled (the buffer is always reused). + - [cuckoo_filter](https://github.com/mozilla-services/lua_cuckoo_filter/blob/master/README.md) Bloom filter alternative supporting deletions - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) Efficiently count the number of elements in a set - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 2e896d0..db2bb59 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -96,7 +96,7 @@ add_dependencies(lua_struct ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG ba756520a51592612c9709ebdfb99392f97eb51f + GIT_TAG 1a7d0bd9298074805c53b2c06c66defa90afaf2b CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -116,3 +116,11 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) + +externalproject_add( + lua_cuckoo_filter + GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git + GIT_TAG a171662d38cb4bfd552f37306189c3dd9791d511 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_DIR ${EP_BASE} +) diff --git a/src/test/lua/bloom_filter.lua b/src/test/lua/bloom_filter.lua index 46648a1..a187e82 100644 --- a/src/test/lua/bloom_filter.lua +++ b/src/test/lua/bloom_filter.lua @@ -5,15 +5,12 @@ require "bloom_filter" bf = bloom_filter.new(20, 0.01) -count = 0 function process(ts) if not bf:query(ts) then if not bf:add(ts) then error("key existed") end - - count = count + 1 end return 0 @@ -22,9 +19,8 @@ end function report(tc) if tc == 99 then bf:clear() - count = 0 else - write_output(count) + write_output(bf:count()) end end diff --git a/src/test/lua/bloom_filter_benchmark.lua b/src/test/lua/bloom_filter_benchmark.lua index ea05a07..8b81a3e 100644 --- a/src/test/lua/bloom_filter_benchmark.lua +++ b/src/test/lua/bloom_filter_benchmark.lua @@ -4,18 +4,14 @@ require "bloom_filter" -bf = bloom_filter.new(6e6, 0.01) -count = 0 +bf = bloom_filter.new(2e6, 0.01) function process(ts) - if bf:add(ts) then - count = count + 1 - end - + bf:add(ts) return 0 end function report(tc) - write_output(count) + write_output(bf:count()) end diff --git a/src/test/lua/cuckoo_filter.lua b/src/test/lua/cuckoo_filter.lua new file mode 100644 index 0000000..ecd48ed --- /dev/null +++ b/src/test/lua/cuckoo_filter.lua @@ -0,0 +1,30 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "cuckoo_filter" + +cf = cuckoo_filter.new(16) + +function process(ts) + if not cf:query(ts) then + if not cf:add(ts) then + error("key existed") + end + end + + return 0 +end + +function report(tc) + if tc == 98 then + cf:delete(1); + write_output(cf:count()) + elseif tc == 99 then + cf:clear() + write_output(cf:count()) + else + write_output(cf:count()) + end +end + diff --git a/src/test/lua/cuckoo_filter_benchmark.lua b/src/test/lua/cuckoo_filter_benchmark.lua new file mode 100644 index 0000000..eaa580c --- /dev/null +++ b/src/test/lua/cuckoo_filter_benchmark.lua @@ -0,0 +1,17 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "cuckoo_filter" + +cf = cuckoo_filter.new(2e6) + +function process(ts) + cf:add(ts) + return 0 +end + +function report(tc) + write_output(cf:count()) +end + diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 86a174c..4aa9f47 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -3,7 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "cjson" -require "circular_buffer"; require "bloom_filter" +require "circular_buffer"; require "bloom_filter"; require "cuckoo_filter" require "lpeg" function process(tc) if tc == 0 then -- error internal reference @@ -70,6 +70,10 @@ function process(tc) local bf = bloom_filter.new(10, 0.01) local hm = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}} write_message(hm) + elseif tc == 20 then -- cuckoo_filter doesn't implement lsb_output + local cf = cuckoo_filter.new(10) + local hm = {Timestamp = 1e9, Fields = {cf = {value=cf, representation="cf"}}} + write_message(hm) end return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index d56773e..daac1ef 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -516,6 +516,7 @@ static char* test_output_errors() , "process() lua/output_errors.lua:65: write_message() could not encode protobuf - invalid boolean value_type: 2" , "process() lua/output_errors.lua:68: write_message() could not encode protobuf - invalid boolean value_type: 3" , "process() lua/output_errors.lua:72: write_message() could not encode protobuf - user data object does not implement lsb_output" + , "process() lua/output_errors.lua:76: write_message() could not encode protobuf - user data object does not implement lsb_output" , NULL }; @@ -1004,6 +1005,10 @@ static char* test_bloom_filter() lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); + report(sb, 0); + mu_assert(strcmp("3", written_data) == 0, "test: count received: %s", + written_data); + for (int i = 0; tests[i]; ++i) { result = process(sb, i); mu_assert(result == 0, "process() received: %d %s", result, @@ -1153,6 +1158,86 @@ static char* test_decode_message() } +static char* test_cuckoo_filter() +{ + const char* output_file = "cuckoo_filter.preserve"; + const char* tests[] = { + "1" + , "2" + , "3" + , NULL + }; + + lua_sandbox* sb = lsb_create(NULL, "lua/cuckoo_filter.lua", "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + lsb_add_function(sb, &write_output, "write_output"); + + int i = 0; + for (; tests[i]; ++i) { + result = process(sb, i); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + result = report(sb, 0); + mu_assert(result == 0, "report() received: %d", result); + mu_assert(strcmp(tests[i], written_data) == 0, "test: %d received: %s", i, + written_data); + } + + result = process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + result = report(sb, 0); + mu_assert(result == 0, "report() received: %d", result); + mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, + written_data); // count should remain the same + + e = lsb_destroy(sb, output_file); + mu_assert(!e, "lsb_destroy() received: %s", e); + + // re-load to test the preserved data + sb = lsb_create(NULL, "lua/cuckoo_filter.lua", "modules", 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + result = lsb_init(sb, output_file); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + lsb_add_function(sb, &write_output, "write_output"); + + report(sb, 0); + mu_assert(strcmp("3", written_data) == 0, "test: count received: %s", + written_data); + + for (int i = 0; tests[i]; ++i) { + result = process(sb, i); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + } + result = report(sb, 0); + mu_assert(result == 0, "report() received: %d", result); + mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, + written_data); // count should remain the same + + // test deletion + report(sb, 98); + mu_assert(strcmp("2", written_data) == 0, "test: delete received: %s", + written_data); + // tst clear + report(sb, 99); + mu_assert(strcmp("0", written_data) == 0, "test: clear received: %s", + written_data); + + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -1377,7 +1462,7 @@ static char* benchmark_bloom_filter_add() } t = clock() - t; report(sb, 0); - mu_assert(strcmp("1000000", written_data) == 0, "received: %s", written_data); + mu_assert(strcmp("999970", written_data) == 0, "received: %s", written_data); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_bloom_filter_add() failed %s", lsb_get_error(sb)); e = lsb_destroy(sb, NULL); @@ -1449,6 +1534,37 @@ static char* benchmark_decode_message() } +static char* benchmark_cuckoo_filter_add() +{ + int iter = 1000000; + + lua_sandbox* sb = lsb_create(NULL, + "lua/cuckoo_filter_benchmark.lua", "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + lsb_add_function(sb, &write_output, "write_output"); + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed + } + t = clock() - t; + report(sb, 0); + mu_assert(strcmp("999985", written_data) == 0, "received: %s", written_data); + mu_assert(lsb_get_state(sb) == LSB_RUNNING, + "benchmark_cuckoo_filter_add() failed %s", lsb_get_error(sb)); + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + printf("benchmark_cuckoo_filter_add() %g seconds\n", ((float)t) + / CLOCKS_PER_SEC / iter); + + return NULL; +} + + static char* all_tests() { mu_run_test(test_create_error); @@ -1476,6 +1592,7 @@ static char* all_tests() mu_run_test(test_struct); mu_run_test(test_sandbox_config); mu_run_test(test_decode_message); + mu_run_test(test_cuckoo_filter); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); @@ -1488,6 +1605,7 @@ static char* all_tests() mu_run_test(benchmark_bloom_filter_add); mu_run_test(benchmark_hyperloglog_add); mu_run_test(benchmark_decode_message); + mu_run_test(benchmark_cuckoo_filter_add); return NULL; } From 7d0e4ed65d73b158c61ebeaf9a84f58ba6cf6a15 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 27 Mar 2015 13:20:46 -0700 Subject: [PATCH 052/235] Avoid conflicts with installed Lua libraries and headers --- CMakeLists.txt | 2 +- cmake/FindLua.cmake | 4 +-- cmake/externals.cmake | 47 +++++++++++++++++----------------- include/lsb.h | 2 +- include/lsb_output.h | 2 +- include/lsb_serialize.h | 2 +- src/CMakeLists.txt | 15 ++++++++--- src/lsb.c | 6 ++--- src/lsb_deserialize_protobuf.c | 6 ++--- src/lsb_output.c | 2 +- src/lsb_private.h | 3 +-- src/lsb_serialize.c | 4 +-- src/test/test_lua_sandbox.c | 4 +-- 13 files changed, 53 insertions(+), 46 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f634cbf..c0459e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(mozsvc) include(externals) -include_directories("${CMAKE_SOURCE_DIR}/include") +include_directories("${EP_BASE}/include") install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION modules) add_subdirectory(src) diff --git a/cmake/FindLua.cmake b/cmake/FindLua.cmake index e47f1cb..d9bc7b8 100644 --- a/cmake/FindLua.cmake +++ b/cmake/FindLua.cmake @@ -33,9 +33,9 @@ if (LUA_SANDBOX_INCLUDE) - set(LUA_INCLUDE_DIR ${EP_BASE}/include) + set(LUA_INCLUDE_DIR ${EP_BASE}/include/lsb) set(LIB_PATH ${EP_BASE}/lib) - find_library(LUA_LIBRARY lua PATHS ${LIB_PATH} NO_DEFAULT_PATH) + find_library(LUA_LIBRARY luasb PATHS ${LIB_PATH} NO_DEFAULT_PATH) else() # Always search for non-versioned lua first (recommended) SET(_POSSIBLE_LUA_INCLUDE include include/lua) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index db2bb59..bd4dfdb 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -13,9 +13,10 @@ endif() set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include --no-warn-unused-cli) -set(LUA_INCLUDE_DIR "${EP_BASE}/include") +set(LUA_INCLUDE_DIR "${EP_BASE}/include/lsb") if (LUA_JIT) + message(FATAL_ERROR, "LuaJIT support has not been added back in yet, issue #66") set(LUA_PROJECT "luajit-2_0_2") if(MSVC) externalproject_add( @@ -25,14 +26,14 @@ if (LUA_JIT) URL_MD5 112dfb82548b03377fbefbba2e0e3a5b PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/luajit-2_0_2.patch CONFIGURE_COMMAND "" - BUILD_COMMAND cmake -E chdir src msvcbuild.bat - INSTALL_COMMAND cmake -E copy src/lua.dll ${EP_BASE}/lib/lua.dll - COMMAND cmake -E copy src/lua.lib ${EP_BASE}/lib/lua.lib - COMMAND cmake -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" - COMMAND cmake -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" - COMMAND cmake -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" - COMMAND cmake -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" - COMMAND cmake -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" + BUILD_COMMAND ${CMAKE_COMMAND} -E chdir src msvcbuild.bat + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy src/lua.dll ${EP_BASE}/lib/lua.dll + COMMAND ${CMAKE_COMMAND} -E copy src/lua.lib ${EP_BASE}/lib/lua.lib + COMMAND ${CMAKE_COMMAND} -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" + COMMAND ${CMAKE_COMMAND} -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" + COMMAND ${CMAKE_COMMAND} -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" + COMMAND ${CMAKE_COMMAND} -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" + COMMAND ${CMAKE_COMMAND} -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" ) elseif(UNIX) externalproject_add( @@ -43,12 +44,12 @@ if (LUA_JIT) PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/luajit-2_0_2.patch CONFIGURE_COMMAND "" BUILD_COMMAND make - INSTALL_COMMAND cmake -E copy src/libluajit.a ${EP_BASE}/lib/liblua.a - COMMAND cmake -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" - COMMAND cmake -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" - COMMAND cmake -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" - COMMAND cmake -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" - COMMAND cmake -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" + INSTALL_COMMAND ${CMAKE_COMMAND} -E copy src/libluajit.a ${EP_BASE}/lib/liblua.a + COMMAND ${CMAKE_COMMAND} -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" + COMMAND ${CMAKE_COMMAND} -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" + COMMAND ${CMAKE_COMMAND} -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" + COMMAND ${CMAKE_COMMAND} -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" + COMMAND ${CMAKE_COMMAND} -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" ) else() message(FATAL_ERROR "Cannot use LuaJIT with ${CMAKE_GENERATOR}") @@ -58,17 +59,17 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG d7ff33f45cf02b4b2e2c1a3cc671bda99b0056c6 + GIT_TAG b0f8c615c58ec4daf6f5d86e740cfedf4b76b53c CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) endif() -include_directories(${LUA_INCLUDE_DIR}) externalproject_add( lua_lpeg GIT_REPOSITORY https://github.com/LuaDist/lpeg.git GIT_TAG baf0dc90b9278360be719dbfb8e56d34ce3c61bd + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -77,7 +78,7 @@ add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG af793dbfd83d9be69835faf8b88702ebb74547ed + GIT_TAG d7a112639d8c84ee9498fce7f664d896d53eec4e CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -87,7 +88,7 @@ externalproject_add( lua_struct GIT_REPOSITORY https://github.com/trink/struct.git GIT_TAG 5cf31819bee0d829d058cb5219e95ef0b1dd43a8 - UPDATE_COMMAND cmake -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -96,7 +97,7 @@ add_dependencies(lua_struct ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG 1a7d0bd9298074805c53b2c06c66defa90afaf2b + GIT_TAG 6358d0ead2b1b5a96d8b66a46505ee083d1ac644 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -104,7 +105,7 @@ externalproject_add( externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG ca4b0271bd741b857527f8315387142beac9cef9 + GIT_TAG 9c484bda208132a349cb706f8c37c6e0d8f87e36 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -112,7 +113,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 78c32961102d977cef5afac2c20a3fd9f5eae808 + GIT_TAG 2baf9e0e8da958f7675acfae1e8157470d8a5ab8 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) @@ -120,7 +121,7 @@ externalproject_add( externalproject_add( lua_cuckoo_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG a171662d38cb4bfd552f37306189c3dd9791d511 + GIT_TAG bd092808481260be373a85334514c55b1ef99ba2 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} ) diff --git a/include/lsb.h b/include/lsb.h index 37a5b14..4e9df16 100644 --- a/include/lsb.h +++ b/include/lsb.h @@ -9,7 +9,7 @@ #ifndef lsb_h_ #define lsb_h_ -#include +#include "lsb/lua.h" #ifdef _WIN32 #ifdef luasandbox_EXPORTS diff --git a/include/lsb_output.h b/include/lsb_output.h index de13e17..6fa2bce 100644 --- a/include/lsb_output.h +++ b/include/lsb_output.h @@ -9,10 +9,10 @@ #ifndef lsb_output_h_ #define lsb_output_h_ -#include #include #include "lsb.h" +#include "lsb/lua.h" /** * Add a output function to the environment table. The environment table must be diff --git a/include/lsb_serialize.h b/include/lsb_serialize.h index 4ceff6b..747793a 100644 --- a/include/lsb_serialize.h +++ b/include/lsb_serialize.h @@ -9,9 +9,9 @@ #ifndef lsb_serialize_h_ #define lsb_serialize_h_ -#include #include +#include "lsb/lua.h" #include "lsb_output.h" /** diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f8eef8e..74d8617 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,31 +16,38 @@ set ( CMAKE_INSTALL_RPATH "$ORIGIN/../lib" CACHE STRING "" FORCE ) set ( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE STRING "" FORCE ) set ( CMAKE_INSTALL_NAME_DIR "@executable_path/../lib" CACHE STRING "" FORCE ) +add_custom_target(copy_includes +COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/lsb.h ${EP_BASE}/include +COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/lsb_output.h ${EP_BASE}/include +COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/lsb_serialize.h ${EP_BASE}/include) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) +add_dependencies(luasandbox copy_includes) + add_dependencies(lua_bloom_filter luasandbox) add_dependencies(lua_circular_buffer luasandbox) add_dependencies(lua_hyperloglog luasandbox) +add_dependencies(lua_cuckoo_filter luasandbox) if(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) - set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/lua.lib") + set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/luasb.lib") elseif(MINGW) add_definitions(-D_MINGW) - set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/liblua.dll") + set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/libluasb.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll") else() if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") endif() - set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}lua${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} -lm) + set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} -lm) endif() target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) install(TARGETS luasandbox DESTINATION lib) install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib PATTERN "lua" EXCLUDE) -install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION modules PATTERN "strict.lua" EXCLUDE) +install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION modules) install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) diff --git a/src/lsb.c b/src/lsb.c index b396fea..56ca230 100644 --- a/src/lsb.c +++ b/src/lsb.c @@ -9,15 +9,15 @@ #include "lsb.h" #include -#include -#include -#include #include #include #include #include #include +#include "lsb/lauxlib.h" +#include "lsb/lua.h" +#include "lsb/lualib.h" #include "lsb_output.h" #include "lsb_private.h" #include "lsb_serialize.h" diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c index 4c86a51..1825fa3 100644 --- a/src/lsb_deserialize_protobuf.c +++ b/src/lsb_deserialize_protobuf.c @@ -6,11 +6,11 @@ /** @brief Lua sandbox Heka protobuf deserialization @file */ -#include "lsb.h" - -#include #include +#include "lsb/lauxlib.h" +#include "lsb.h" + #define UUID_SIZE 16 #define MAX_VARINT_BYTES 10 diff --git a/src/lsb_output.c b/src/lsb_output.c index 593ef5d..4e894ca 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -8,12 +8,12 @@ #include "lsb_output.h" -#include #include #include #include #include +#include "lsb/lauxlib.h" #include "lsb_private.h" #include "lsb_serialize.h" #include "lsb_serialize_protobuf.h" diff --git a/src/lsb_private.h b/src/lsb_private.h index 344877f..7b323be 100644 --- a/src/lsb_private.h +++ b/src/lsb_private.h @@ -9,9 +9,8 @@ #ifndef lsb_private_h_ #define lsb_private_h_ -#include - #include "lsb.h" +#include "lsb/lua.h" #ifdef _WIN32 #define snprintf _snprintf diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index adf7372..040d792 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -8,12 +8,12 @@ #include "lsb_serialize.h" -#include -#include #include #include #include +#include "lsb/lauxlib.h" +#include "lsb/lualib.h" #include "lsb_private.h" typedef struct diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index daac1ef..26a166b 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -7,14 +7,14 @@ /** @brief Lua sandbox unit tests @file */ #include -#include -#include #include #include #include #include #include "lsb.h" +#include "lsb/lauxlib.h" +#include "lsb/lua.h" #include "lsb_output.h" #ifdef _WIN32 From 445f0445ca955927b70f1ac2783fb40bee3bfca2 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 7 Apr 2015 09:17:18 -0700 Subject: [PATCH 053/235] Complete the renaming and package cleanup --- CMakeLists.txt | 13 ++++++++-- cmake/FindLua.cmake | 2 +- cmake/externals.cmake | 25 +++++++++++++------ cmake/mozsvc.cmake | 4 ++- include/{lsb.h => luasandbox.h} | 2 +- include/{lsb_output.h => luasandbox_output.h} | 4 +-- ...lsb_serialize.h => luasandbox_serialize.h} | 4 +-- src/CMakeLists.txt | 24 ++++++++---------- src/lsb_deserialize_protobuf.c | 4 +-- src/lsb_output.c | 6 ++--- src/lsb_private.h | 4 +-- src/lsb_serialize.c | 6 ++--- src/lsb_serialize_protobuf.c | 2 +- src/lsb_serialize_protobuf.h | 2 +- src/{lsb.c => luasandbox.c} | 12 ++++----- src/test/test_lua_sandbox.c | 8 +++--- 16 files changed, 70 insertions(+), 52 deletions(-) rename include/{lsb.h => luasandbox.h} (99%) rename include/{lsb_output.h => luasandbox_output.h} (98%) rename include/{lsb_serialize.h => luasandbox_serialize.h} (97%) rename src/{lsb.c => luasandbox.c} (98%) diff --git a/CMakeLists.txt b/CMakeLists.txt index c0459e8..5a3a8f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,11 +3,19 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -project(lua_sandbox C) +project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 8) set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") + +set(CPACK_DEB_COMPONENT_INSTALL ON) + +set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") +set(CPACK_RPM_COMPONENT_INSTALL ON) + +set(CPACK_COMPONENTS_ALL core) option(LUA_JIT "Enable LuaJIT" off) if(LUA_JIT) @@ -21,6 +29,7 @@ include(mozsvc) include(externals) include_directories("${EP_BASE}/include") -install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION modules) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) +install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION share/doc/${PROJECT_NAME} COMPONENT core) add_subdirectory(src) diff --git a/cmake/FindLua.cmake b/cmake/FindLua.cmake index d9bc7b8..2043318 100644 --- a/cmake/FindLua.cmake +++ b/cmake/FindLua.cmake @@ -33,7 +33,7 @@ if (LUA_SANDBOX_INCLUDE) - set(LUA_INCLUDE_DIR ${EP_BASE}/include/lsb) + set(LUA_INCLUDE_DIR ${EP_BASE}/include/luasandbox) set(LIB_PATH ${EP_BASE}/lib) find_library(LUA_LIBRARY luasb PATHS ${LIB_PATH} NO_DEFAULT_PATH) else() diff --git a/cmake/externals.cmake b/cmake/externals.cmake index bd4dfdb..686532b 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. include(ExternalProject) +string(REPLACE ")$" "|INSTALL_ARGS)$" _ep_keywords_ExternalProject_Add ${_ep_keywords_ExternalProject_Add}) get_filename_component(GIT_PATH ${GIT_EXECUTABLE} PATH) find_program(PATCH_EXECUTABLE patch HINTS "${GIT_PATH}" "${GIT_PATH}/../bin") @@ -12,8 +13,8 @@ endif() set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) -set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include --no-warn-unused-cli) -set(LUA_INCLUDE_DIR "${EP_BASE}/include/lsb") +set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include -DUSE_RPATH=false --no-warn-unused-cli) +set(LUA_INCLUDE_DIR "${EP_BASE}/include/${PROJECT_NAME}") if (LUA_JIT) message(FATAL_ERROR, "LuaJIT support has not been added back in yet, issue #66") @@ -59,9 +60,10 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG b0f8c615c58ec4daf6f5d86e740cfedf4b76b53c + GIT_TAG 477c0b5ae0932953ee41ab7cd24098c23e3e33c3 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) endif() @@ -72,15 +74,17 @@ externalproject_add( UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG d7a112639d8c84ee9498fce7f664d896d53eec4e + GIT_TAG 3c8321cbfc6bea944974e046da82deca38e66587 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) add_dependencies(lua_cjson ${LUA_PROJECT}) @@ -91,37 +95,42 @@ externalproject_add( UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) add_dependencies(lua_struct ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG 6358d0ead2b1b5a96d8b66a46505ee083d1ac644 + GIT_TAG 1afb5fe85adec5750238c5ebd7d6d95e7993f122 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG 9c484bda208132a349cb706f8c37c6e0d8f87e36 + GIT_TAG e87bcc0a3ae29c5a7acaabbb7b1f2b56700769c6 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 2baf9e0e8da958f7675acfae1e8157470d8a5ab8 + GIT_TAG 5deeaf7c24088dada8d80529951502aa4a55c8ab CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) externalproject_add( lua_cuckoo_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG bd092808481260be373a85334514c55b1ef99ba2 + GIT_TAG cfb311a433ed7e4c8224ca99ab58b54302037b6d CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_DIR ${EP_BASE} + INSTALL_ARGS "install/strip" ) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 6025911..2818048 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -20,7 +20,7 @@ if(MSVC) set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2 /DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) - set(CPACK_GENERATOR "NSIS") + set(CPACK_GENERATOR "ZIP") else() # Predefined Macros: clang|gcc -dM -E -x c /dev/null # Compiler options: http://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html#Invoking-GCC @@ -47,6 +47,8 @@ endif() set(CPACK_PACKAGE_VENDOR "Mozilla Services") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE.txt") +set(CPACK_STRIP_FILES TRUE) + include(CPack) include(CTest) include(doxygen) diff --git a/include/lsb.h b/include/luasandbox.h similarity index 99% rename from include/lsb.h rename to include/luasandbox.h index 4e9df16..16985b2 100644 --- a/include/lsb.h +++ b/include/luasandbox.h @@ -9,7 +9,7 @@ #ifndef lsb_h_ #define lsb_h_ -#include "lsb/lua.h" +#include "luasandbox/lua.h" #ifdef _WIN32 #ifdef luasandbox_EXPORTS diff --git a/include/lsb_output.h b/include/luasandbox_output.h similarity index 98% rename from include/lsb_output.h rename to include/luasandbox_output.h index 6fa2bce..1bfee87 100644 --- a/include/lsb_output.h +++ b/include/luasandbox_output.h @@ -11,8 +11,8 @@ #include -#include "lsb.h" -#include "lsb/lua.h" +#include "luasandbox.h" +#include "luasandbox/lua.h" /** * Add a output function to the environment table. The environment table must be diff --git a/include/lsb_serialize.h b/include/luasandbox_serialize.h similarity index 97% rename from include/lsb_serialize.h rename to include/luasandbox_serialize.h index 747793a..f696c9b 100644 --- a/include/lsb_serialize.h +++ b/include/luasandbox_serialize.h @@ -11,8 +11,8 @@ #include -#include "lsb/lua.h" -#include "lsb_output.h" +#include "luasandbox/lua.h" +#include "luasandbox_output.h" /** * Serialize all user global data to disk. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 74d8617..9694bb0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. set(LUA_SANDBOX_SRC -lsb.c +luasandbox.c lsb_output.c lsb_serialize.c lsb_serialize_protobuf.c @@ -12,14 +12,11 @@ lsb_deserialize_protobuf.c set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) -set ( CMAKE_INSTALL_RPATH "$ORIGIN/../lib" CACHE STRING "" FORCE ) -set ( CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE CACHE STRING "" FORCE ) -set ( CMAKE_INSTALL_NAME_DIR "@executable_path/../lib" CACHE STRING "" FORCE ) add_custom_target(copy_includes -COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/lsb.h ${EP_BASE}/include -COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/lsb_output.h ${EP_BASE}/include -COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/lsb_serialize.h ${EP_BASE}/include) +COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/luasandbox.h ${EP_BASE}/include +COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/luasandbox_output.h ${EP_BASE}/include +COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/luasandbox_serialize.h ${EP_BASE}/include) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) add_dependencies(luasandbox copy_includes) @@ -35,7 +32,7 @@ elseif(MINGW) add_definitions(-D_MINGW) set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/libluasb.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) - install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll") + install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll" COMPONENT core) else() if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") @@ -43,12 +40,13 @@ else() set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} -lm) endif() +set_target_properties(luasandbox PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) + target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) -install(TARGETS luasandbox DESTINATION lib) -install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib PATTERN "lua" EXCLUDE) -install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION modules) -install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) -install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) +install(TARGETS luasandbox DESTINATION lib COMPONENT core) +install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib COMPONENT core PATTERN "lua" EXCLUDE) +install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) +install(DIRECTORY "${EP_BASE}/include/" DESTINATION include COMPONENT core) add_subdirectory(test) diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c index 1825fa3..843acaa 100644 --- a/src/lsb_deserialize_protobuf.c +++ b/src/lsb_deserialize_protobuf.c @@ -8,8 +8,8 @@ #include -#include "lsb/lauxlib.h" -#include "lsb.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox.h" #define UUID_SIZE 16 #define MAX_VARINT_BYTES 10 diff --git a/src/lsb_output.c b/src/lsb_output.c index 4e894ca..d7e7c50 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -6,16 +6,16 @@ /** @brief Lua sandbox output buffer implementation @file */ -#include "lsb_output.h" +#include "luasandbox_output.h" #include #include #include #include -#include "lsb/lauxlib.h" +#include "luasandbox/lauxlib.h" #include "lsb_private.h" -#include "lsb_serialize.h" +#include "luasandbox_serialize.h" #include "lsb_serialize_protobuf.h" static const char* output_function = "lsb_output"; diff --git a/src/lsb_private.h b/src/lsb_private.h index 7b323be..083c21d 100644 --- a/src/lsb_private.h +++ b/src/lsb_private.h @@ -9,8 +9,8 @@ #ifndef lsb_private_h_ #define lsb_private_h_ -#include "lsb.h" -#include "lsb/lua.h" +#include "luasandbox.h" +#include "luasandbox/lua.h" #ifdef _WIN32 #define snprintf _snprintf diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index 040d792..b3e13ab 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -6,14 +6,14 @@ /** @brief Sandbox serialization implementation @file */ -#include "lsb_serialize.h" +#include "luasandbox_serialize.h" #include #include #include -#include "lsb/lauxlib.h" -#include "lsb/lualib.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lualib.h" #include "lsb_private.h" typedef struct diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c index 0548114..21f8571 100644 --- a/src/lsb_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -12,7 +12,7 @@ #include #include -#include "lsb_output.h" +#include "luasandbox_output.h" #include "lsb_private.h" #define UUID_SIZE 16 diff --git a/src/lsb_serialize_protobuf.h b/src/lsb_serialize_protobuf.h index d2f27d9..1c6a709 100644 --- a/src/lsb_serialize_protobuf.h +++ b/src/lsb_serialize_protobuf.h @@ -9,7 +9,7 @@ #ifndef lsb_serialize_protobuf_h_ #define lsb_serialize_protobuf_h_ -#include "lsb.h" +#include "luasandbox.h" /** * Serialize a specific Lua table structure as Protobuf diff --git a/src/lsb.c b/src/luasandbox.c similarity index 98% rename from src/lsb.c rename to src/luasandbox.c index 56ca230..51e5c01 100644 --- a/src/lsb.c +++ b/src/luasandbox.c @@ -6,7 +6,7 @@ /** @brief Lua sandboxed implementation @file */ -#include "lsb.h" +#include "luasandbox.h" #include #include @@ -15,12 +15,12 @@ #include #include -#include "lsb/lauxlib.h" -#include "lsb/lua.h" -#include "lsb/lualib.h" -#include "lsb_output.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox/lualib.h" +#include "luasandbox_output.h" #include "lsb_private.h" -#include "lsb_serialize.h" +#include "luasandbox_serialize.h" #include "lsb_serialize_protobuf.h" static const char* standard_config = "{" diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 26a166b..fbbdff2 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -12,10 +12,10 @@ #include #include -#include "lsb.h" -#include "lsb/lauxlib.h" -#include "lsb/lua.h" -#include "lsb_output.h" +#include "luasandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox_output.h" #ifdef _WIN32 #define snprintf _snprintf From 2a5e0dc52e344e9c56598495c0fe4fa81741611f Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 8 Apr 2015 13:48:52 -0700 Subject: [PATCH 054/235] Fix the varint decoding so the value is not shifted off the left --- src/lsb_deserialize_protobuf.c | 4 ++-- src/test/lua/decode_message.lua | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c index 843acaa..2c69021 100644 --- a/src/lsb_deserialize_protobuf.c +++ b/src/lsb_deserialize_protobuf.c @@ -29,7 +29,7 @@ read_length(unsigned const char* p, unsigned const char* e, size_t* vi) *vi = 0; unsigned i, shift = 0; for (i = 0; p != e && i < MAX_VARINT_BYTES; i++) { - *vi |= (p[i] & 0x7f) << shift; + *vi |= ((unsigned long long)p[i] & 0x7f) << shift; shift += 7; if ((p[i] & 0x80) == 0) break; } @@ -46,7 +46,7 @@ read_varint(unsigned const char* p, unsigned const char* e, long long* vi) *vi = 0; unsigned i, shift = 0; for (i = 0; p != e && i < MAX_VARINT_BYTES; i++) { - *vi |= (p[i] & 0x7f) << shift; + *vi |= ((unsigned long long)p[i] & 0x7f) << shift; shift += 7; if ((p[i] & 0x80) == 0) break; } diff --git a/src/test/lua/decode_message.lua b/src/test/lua/decode_message.lua index 240e870..0595730 100644 --- a/src/test/lua/decode_message.lua +++ b/src/test/lua/decode_message.lua @@ -96,6 +96,11 @@ assert(#msg.Fields[1].value == 2) assert(msg.Fields[1].value[1] == "s1") assert(msg.Fields[1].value[2] == "s2") +-- recent timestamp +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\152\141\135\236\222\200\233\19" +msg = decode_message(test) +assert(msg.Timestamp == 1428523950000000000) + -- benchmark message test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" msg = decode_message(test) From c0c62b8c0553e0d12e4c10523e02c0fbbc46e79a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 3 May 2015 20:03:21 -0700 Subject: [PATCH 055/235] Add the luasocket module --- cmake/externals.cmake | 20 ++++++++++++-------- src/CMakeLists.txt | 1 + 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 686532b..c91f778 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -62,7 +62,6 @@ else() GIT_REPOSITORY https://github.com/trink/lua.git GIT_TAG 477c0b5ae0932953ee41ab7cd24098c23e3e33c3 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) endif() @@ -73,7 +72,6 @@ externalproject_add( GIT_TAG baf0dc90b9278360be719dbfb8e56d34ce3c61bd UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) add_dependencies(lua_lpeg ${LUA_PROJECT}) @@ -83,7 +81,6 @@ externalproject_add( GIT_REPOSITORY https://github.com/trink/lua-cjson.git GIT_TAG 3c8321cbfc6bea944974e046da82deca38e66587 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) add_dependencies(lua_cjson ${LUA_PROJECT}) @@ -94,17 +91,27 @@ externalproject_add( GIT_TAG 5cf31819bee0d829d058cb5219e95ef0b1dd43a8 UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) add_dependencies(lua_struct ${LUA_PROJECT}) +externalproject_add( + lua_socket + GIT_REPOSITORY https://github.com/LuaDist/luasocket.git + GIT_TAG b97ed47e7a01e0d2523809efe11676333a85dcab + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + INSTALL_ARGS "install/strip" +) +add_dependencies(lua_socket ${LUA_PROJECT}) + +# sandbox enhanced modules + externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git GIT_TAG 1afb5fe85adec5750238c5ebd7d6d95e7993f122 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) @@ -113,7 +120,6 @@ externalproject_add( GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git GIT_TAG e87bcc0a3ae29c5a7acaabbb7b1f2b56700769c6 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) @@ -122,7 +128,6 @@ externalproject_add( GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git GIT_TAG 5deeaf7c24088dada8d80529951502aa4a55c8ab CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) @@ -131,6 +136,5 @@ externalproject_add( GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git GIT_TAG cfb311a433ed7e4c8224ca99ab58b54302037b6d CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} INSTALL_ARGS "install/strip" ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9694bb0..a3f6979 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,6 +47,7 @@ target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) install(TARGETS luasandbox DESTINATION lib COMPONENT core) install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib COMPONENT core PATTERN "lua" EXCLUDE) install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) +install(DIRECTORY "${EP_BASE}/io/lib/lua/" DESTINATION lib/${PROJECT_NAME}/io_modules COMPONENT core) install(DIRECTORY "${EP_BASE}/include/" DESTINATION include COMPONENT core) add_subdirectory(test) From 6cadd80be3883dd82c058fd23d756fae05a92f19 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 8 May 2015 15:18:19 -0700 Subject: [PATCH 056/235] Use a better seed for srand --- src/lsb_serialize_protobuf.c | 2 +- src/luasandbox.c | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c index 21f8571..23a3e36 100644 --- a/src/lsb_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -599,7 +599,7 @@ int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index) d->pos += UUID_SIZE; } else { for (int x = 0; x < UUID_SIZE; ++x) { - d->data[d->pos++] = rand() % 255; + d->data[d->pos++] = rand() % 256; } d->data[8] = (d->data[8] & 0x0F) | 0x40; d->data[10] = (d->data[10] & 0x0F) | 0xA0; diff --git a/src/luasandbox.c b/src/luasandbox.c index 51e5c01..3b8ccad 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -156,7 +157,7 @@ static int unprotected_panic(lua_State* lua) static unsigned get_usage_config(lua_State* lua, int idx, const char* item) { lua_getfield(lua, idx, item); - unsigned u = (unsigned)lua_tonumber(lua, -1); // defaults to zero + unsigned u = (unsigned)lua_tonumber(lua, -1); // defaults to zero lua_pop(lua, 1); return u; } @@ -231,15 +232,36 @@ lua_sandbox* lsb_create_custom(void* parent, return NULL; } + bool seeded = false; #if _WIN32 if (_putenv("TZ=UTC") != 0) { return NULL; } + // todo use CryptGenRandom to seed srand #else if (setenv("TZ", "UTC", 1) != 0) { return NULL; } + + FILE* fh = fopen("/dev/urandom", "r"); + if (fh) { + unsigned seed; + unsigned char advance; + if (fread(&seed, sizeof(unsigned), 1, fh) == 1 && + fread(&advance, sizeof(char), 1, fh) == 1) { + srand(seed); + // advance the sequence a random amount + for (unsigned i = 0; i < advance; ++i) { + rand(); + } + seeded = true; + } + fclose(fh); + } #endif + if (!seeded) { + srand((unsigned)time(NULL)); + } lua_sandbox* lsb = malloc(sizeof(lua_sandbox)); memset(lsb->usage, 0, sizeof(lsb->usage)); @@ -297,7 +319,6 @@ lua_sandbox* lsb_create_custom(void* parent, return NULL; } strcpy(lsb->lua_file, lua_file); - srand((unsigned int)time(NULL)); return lsb; } From ecd1b406d51cb805b893c18df9541a6007741289 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 11 May 2015 09:53:41 -0700 Subject: [PATCH 057/235] Add initial code coverage numbers --- covfn.txt | 152 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 covfn.txt diff --git a/covfn.txt b/covfn.txt new file mode 100644 index 0000000..9bb5f77 --- /dev/null +++ b/covfn.txt @@ -0,0 +1,152 @@ +Function Source Line FnCov C/D Coverage +----------------------------------------------------------------------------- --------------------------------------------------- ----- --------------------- +circular_buffer_mannwhitneyu(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 662 0 / 1 0 / 30 = 0% +rank_data(double*[],size_t) .../lua_circular_buffer/lua_circular_buffer.c 618 0 / 1 0 / 16 = 0% +double_pp_compare(const void*,const void*) .../lua_circular_buffer/lua_circular_buffer.c 647 0 / 1 0 / 14 = 0% +circular_buffer_compute(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 552 0 / 1 0 / 13 = 0% +compute_min(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 498 0 / 1 0 / 10 = 0% +compute_max(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 525 0 / 1 0 / 10 = 0% +compute_variance(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 463 0 / 1 0 / 8 = 0% +compute_sum(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 415 0 / 1 0 / 6 = 0% +compute_avg(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 438 0 / 1 0 / 6 = 0% +append_values(circular_buffer*,unsigned,unsigned,unsigned,double[]) .../lua_circular_buffer/lua_circular_buffer.c 602 0 / 1 0 / 4 = 0% +lsb_get_parent(lua_sandbox*) ../src/luasandbox.c 457 0 / 1 0 / 0 +bloom_filter_version(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 186 0 / 1 0 / 0 +circular_buffer_get_configuration(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 315 0 / 1 0 / 0 +circular_buffer_get_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 402 0 / 1 0 / 0 +compute_sd(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 490 0 / 1 0 / 0 +circular_buffer_current_time(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 754 0 / 1 0 / 0 +circular_buffer_version(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 858 0 / 1 0 / 0 +cf_version(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 330 0 / 1 0 / 0 +hyperloglog_version(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 152 0 / 1 0 / 0 +lsb_restore_global_data(lua_sandbox*,const char*) ../src/lsb_serialize.c 488 1 / 1 2 / 12 = 16% +add_table_ref(table_ref_array*,const void*,size_t) ../src/lsb_serialize.c 168 1 / 1 1 / 4 = 25% +pb_write_double(lsb_output_data*,double) ../src/lsb_serialize_protobuf.c 58 1 / 1 1 / 4 = 25% +pb_write_tag(lsb_output_data*,unsigned char,unsigned char) ../src/lsb_serialize_protobuf.c 105 1 / 1 1 / 4 = 25% +MurmurHash64A(const void*,int,unsigned int) ...Source/lua_hyperloglog/redis_hyperloglog.c 264 1 / 1 3 / 10 = 30% +pb_write_string(lsb_output_data*,char,const char*,size_t) ../src/lsb_serialize_protobuf.c 164 1 / 1 3 / 8 = 37% +lsb_appendf(lsb_output_data*,const char*,...) ../src/lsb_output.c 67 1 / 1 9 / 22 = 40% +cf_delete(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 262 1 / 1 3 / 7 = 42% +serialize_data(lua_sandbox*,int,lsb_output_data*) ../src/lsb_serialize.c 227 1 / 1 10 / 22 = 45% +serialize_bloom_filter(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 195 1 / 1 8 / 16 = 50% +serialize_cuckoo_filter(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 339 1 / 1 8 / 16 = 50% +serialize_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 160 1 / 1 8 / 16 = 50% +pb_write_bool(lsb_output_data*,int) ../src/lsb_serialize_protobuf.c 80 1 / 1 3 / 6 = 50% +hyperloglog_fromstring(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 132 1 / 1 3 / 6 = 50% +output_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 183 1 / 1 3 / 6 = 50% +bloom_filter_fromstring(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 164 1 / 1 2 / 4 = 50% +file_exists(const char*) ../src/lsb_serialize.c 477 1 / 1 1 / 2 = 50% +instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 123 1 / 1 1 / 2 = 50% +circular_buffer_get(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 298 1 / 1 1 / 2 = 50% +cf_fromstring(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 316 1 / 1 1 / 2 = 50% +lsb_serialize_table_as_pb(lua_sandbox*,int) ../src/lsb_serialize_protobuf.c 581 1 / 1 20 / 38 = 52% +expand_path(const char*,int,const char*,char*) ../src/luasandbox.c 165 1 / 1 10 / 18 = 55% +circular_buffer_set(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 326 1 / 1 14 / 25 = 56% +ignore_value_type(lua_sandbox*,serialization_data*,int,lua_CFunction*) ../src/lsb_serialize.c 109 1 / 1 8 / 14 = 57% +lsb_serialize_binary(const void*,size_t,lsb_output_data*) ../src/lsb_serialize.c 535 1 / 1 10 / 17 = 58% +update_field_length(lsb_output_data*,size_t) ../src/lsb_serialize_protobuf.c 128 1 / 1 6 / 10 = 60% +cf_add(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 195 1 / 1 3 / 5 = 60% +cf_query(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 228 1 / 1 3 / 5 = 60% +hyperloglog_add(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 59 1 / 1 3 / 5 = 60% +lsb_create_custom(void*,const char*,const char*) ../src/luasandbox.c 226 1 / 1 16 / 26 = 61% +pb_write_varint(lsb_output_data*,unsigned long long) ../src/lsb_serialize_protobuf.c 29 1 / 1 5 / 8 = 62% +lsb_pcall_setup(lua_sandbox*,const char*) ../src/luasandbox.c 493 1 / 1 5 / 8 = 62% +serialize_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 1017 1 / 1 19 / 30 = 63% +output_circular_buffer_cbufd(lua_State*,circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 889 1 / 1 14 / 22 = 63% +lsb_create(void*,const char*,const char*,unsigned,unsigned,unsigned) ../src/luasandbox.c 182 1 / 1 9 / 14 = 64% +serialize_circular_buffer_delta(lua_State*,circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 974 1 / 1 13 / 20 = 65% +encode_fields(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 519 1 / 1 24 / 36 = 66% +encode_field_array(lua_sandbox*,lsb_output_data*,int,const char*,int) ../src/lsb_serialize_protobuf.c 269 1 / 1 20 / 30 = 66% +check_row(circular_buffer*,double,int) .../lua_circular_buffer/lua_circular_buffer.c 187 1 / 1 8 / 12 = 66% +hllCount(hyperloglog*) ...Source/lua_hyperloglog/redis_hyperloglog.c 454 1 / 1 11 / 16 = 68% +process_fields(lua_State*,unsigned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 104 1 / 1 51 / 74 = 68% +nlz(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 70 1 / 1 7 / 10 = 70% +output_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 933 1 / 1 17 / 24 = 70% +lsb_serialize_double(lsb_output_data*,double) ../src/lsb_serialize.c 561 1 / 1 27 / 38 = 71% +encode_field_value(lua_sandbox*,lsb_output_data*,int,const char*,int) ../src/lsb_serialize_protobuf.c 336 1 / 1 75 / 105 = 71% +lsb_init(lua_sandbox*,const char*) ../src/luasandbox.c 305 1 / 1 20 / 28 = 71% +circular_buffer_set_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 373 1 / 1 10 / 14 = 71% +read_double(char**,double*) .../lua_circular_buffer/lua_circular_buffer.c 781 1 / 1 16 / 22 = 72% +lsb_preserve_global_data(lua_sandbox*,const char*) ../src/lsb_serialize.c 366 1 / 1 19 / 26 = 73% +serialize_kvp(lua_sandbox*,serialization_data*,size_t) ../src/lsb_serialize.c 287 1 / 1 18 / 24 = 75% +circular_buffer_fromstring(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 830 1 / 1 9 / 12 = 75% +read_string(lua_State*,int,unsigned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 61 1 / 1 6 / 8 = 75% +lsb_usage(lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 405 1 / 1 6 / 8 = 75% +circular_buffer_add_delta(lua_State*,circular_buffer*,double,int,double) .../lua_circular_buffer/lua_circular_buffer.c 219 1 / 1 6 / 8 = 75% +process_varint(lua_State*,const char*...ned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 82 1 / 1 3 / 4 = 75% +lsb_appendc(lsb_output_data*,char) ../src/lsb_output.c 55 1 / 1 3 / 4 = 75% +encode_int(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 225 1 / 1 3 / 4 = 75% +output(lua_State*) ../src/luasandbox.c 131 1 / 1 3 / 4 = 75% +lsb_set_error(lua_sandbox*,const char*) ../src/luasandbox.c 438 1 / 1 3 / 4 = 75% +lsb_get_output(lua_sandbox*,size_t*) ../src/luasandbox.c 481 1 / 1 3 / 4 = 75% +bucket_delete(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 136 1 / 1 3 / 4 = 75% +bloom_filter_add(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 76 1 / 1 7 / 9 = 77% +bloom_filter_query(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 115 1 / 1 7 / 9 = 77% +lsb_realloc_output(lsb_output_data*,size_t) ../src/lsb_output.c 132 1 / 1 13 / 16 = 81% +bucket_insert(cuckoo_filter*,unsigned,unsigned,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 160 1 / 1 13 / 16 = 81% +lsb_decode_protobuf(lua_State*) ../src/lsb_deserialize_protobuf.c 262 1 / 1 54 / 65 = 83% +lsb_output(lua_sandbox*,int,int,int) ../src/lsb_output.c 153 1 / 1 25 / 30 = 83% +lsb_output_protobuf(lua_sandbox*,int,int) ../src/lsb_output.c 215 1 / 1 5 / 6 = 83% +lsb_destroy(lua_sandbox*,const char*) ../src/luasandbox.c 381 1 / 1 12 / 14 = 85% +output_circular_buffer_full(circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 867 1 / 1 12 / 14 = 85% +circular_buffer_delta_fromstring(lua_State*,circular_buffer*,char**) .../lua_circular_buffer/lua_circular_buffer.c 804 1 / 1 7 / 8 = 87% +hllDenseSum(uint8_t*,double*,int*) ...Source/lua_hyperloglog/redis_hyperloglog.c 356 1 / 1 34 / 38 = 89% +read_length(unsigned const char*,unsigned const char*,size_t*) ../src/lsb_deserialize_protobuf.c 27 1 / 1 9 / 10 = 90% +read_varint(unsigned const char*,unsigned const char*,long long*) ../src/lsb_deserialize_protobuf.c 44 1 / 1 9 / 10 = 90% +memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 86 1 / 1 11 / 12 = 91% +circular_buffer_add(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 267 1 / 1 11 / 12 = 91% +read_key(unsigned const char*,int*,int*) ../src/lsb_deserialize_protobuf.c 18 1 / 1 0 / 0 +update_output_stats(lua_sandbox*) ../src/lsb_output.c 24 1 / 1 2 / 2 = 100% +lsb_add_output_function(lua_State*,lua_CFunction) ../src/lsb_output.c 35 1 / 1 0 / 0 +lsb_get_output_function(lua_State*,int) ../src/lsb_output.c 43 1 / 1 0 / 0 +lsb_appends(lsb_output_data*,const char*,size_t) ../src/lsb_output.c 119 1 / 1 4 / 4 = 100% +get_serialize_function(lua_State*,int) ../src/lsb_serialize.c 87 1 / 1 0 / 0 +find_table_ref(table_ref_array*,const void*) ../src/lsb_serialize.c 147 1 / 1 4 / 4 = 100% +get_preservation_version(lua_State*) ../src/lsb_serialize.c 193 1 / 1 4 / 4 = 100% +serialize_table(lua_sandbox*,serialization_data*,size_t) ../src/lsb_serialize.c 212 1 / 1 6 / 6 = 100% +lsb_add_serialize_function(lua_State*,lua_CFunction) ../src/lsb_serialize.c 527 1 / 1 0 / 0 +encode_string(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 196 1 / 1 2 / 2 = 100% +encode_field_object(lua_sandbox*,lsb_output_data*) ../src/lsb_serialize_protobuf.c 315 1 / 1 4 / 4 = 100% +libsize(const luaL_Reg*) ../src/luasandbox.c 52 1 / 1 2 / 2 = 100% +preload_modules(lua_State*) ../src/luasandbox.c 59 1 / 1 2 / 2 = 100% +instruction_usage(lua_sandbox*) ../src/luasandbox.c 117 1 / 1 0 / 0 +unprotected_panic(lua_State*) ../src/luasandbox.c 148 1 / 1 0 / 0 +get_usage_config(lua_State*,int,const char*) ../src/luasandbox.c 156 1 / 1 0 / 0 +lsb_get_error(lua_sandbox*) ../src/luasandbox.c 429 1 / 1 2 / 2 = 100% +lsb_get_lua(lua_sandbox*) ../src/luasandbox.c 451 1 / 1 0 / 0 +lsb_get_state(lua_sandbox*) ../src/luasandbox.c 463 1 / 1 2 / 2 = 100% +lsb_add_function(lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 472 1 / 1 0 / 0 +lsb_pcall_teardown(lua_sandbox*) ../src/luasandbox.c 512 1 / 1 2 / 2 = 100% +lsb_terminate(lua_sandbox*,const char*) ../src/luasandbox.c 524 1 / 1 4 / 4 = 100% +bloom_filter_new(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 36 1 / 1 0 / 0 +check_bloom_filter(lua_State*,int) ...Source/lua_bloom_filter/lua_bloom_filter.c 66 1 / 1 0 / 0 +bloom_filter_count(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 147 1 / 1 0 / 0 +bloom_filter_clear(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 155 1 / 1 0 / 0 +luaopen_bloom_filter(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 242 1 / 1 0 / 0 +get_start_time(circular_buffer*) .../lua_circular_buffer/lua_circular_buffer.c 81 1 / 1 0 / 0 +copy_cleared_row(circular_buffer*,double*,size_t) .../lua_circular_buffer/lua_circular_buffer.c 87 1 / 1 4 / 4 = 100% +clear_rows(circular_buffer*,unsigned) .../lua_circular_buffer/lua_circular_buffer.c 106 1 / 1 10 / 10 = 100% +circular_buffer_new(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 130 1 / 1 4 / 4 = 100% +check_circular_buffer(lua_State*,int) .../lua_circular_buffer/lua_circular_buffer.c 177 1 / 1 0 / 0 +check_column(lua_State*,circular_buffer*,int) .../lua_circular_buffer/lua_circular_buffer.c 209 1 / 1 0 / 0 +circular_buffer_format(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 761 1 / 1 0 / 0 +read_time_row(char**,circular_buffer*) .../lua_circular_buffer/lua_circular_buffer.c 774 1 / 1 0 / 0 +luaopen_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 1102 1 / 1 0 / 0 +clp2(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 51 1 / 1 0 / 0 +cf_new(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 85 1 / 1 0 / 0 +check_cuckoo_filter(lua_State*,int) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 110 1 / 1 0 / 0 +fingerprint(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 120 1 / 1 2 / 2 = 100% +bucket_lookup(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 127 1 / 1 4 / 4 = 100% +bucket_add(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 148 1 / 1 4 / 4 = 100% +cf_count(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 299 1 / 1 0 / 0 +cf_clear(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 307 1 / 1 0 / 0 +luaopen_cuckoo_filter(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 387 1 / 1 0 / 0 +hyperloglog_new(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 29 1 / 1 0 / 0 +check_hyperloglog(lua_State*,int) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 49 1 / 1 0 / 0 +hyperloglog_count(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 90 1 / 1 2 / 2 = 100% +hyperloglog_clear(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 123 1 / 1 0 / 0 +luaopen_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 213 1 / 1 0 / 0 +hllPatLen(unsigned char*,size_t,long*) ...Source/lua_hyperloglog/redis_hyperloglog.c 323 1 / 1 2 / 2 = 100% +hllDenseAdd(uint8_t*,unsigned char*,size_t) ...Source/lua_hyperloglog/redis_hyperloglog.c 426 1 / 1 2 / 2 = 100% +----------------------------------------------------------------------------- --------------------------------------------------- ----- --------------------- +Total 87% 904 / 1419 = 63% From 3b310428c313e178e49feaa71b1298ad53acff18 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 11 May 2015 10:41:25 -0700 Subject: [PATCH 058/235] Run the core tests associated with our Lua modules --- cmake/externals.cmake | 2 +- covfn.txt | 56 ++++++++++++++++++------------------- src/test/test_lua_sandbox.c | 54 +++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 29 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index c91f778..25cf053 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -126,7 +126,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 5deeaf7c24088dada8d80529951502aa4a55c8ab + GIT_TAG 92f8a59e9e6f4ee381482f449b6cee78bb55460a CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS "install/strip" ) diff --git a/covfn.txt b/covfn.txt index 9bb5f77..fce484b 100644 --- a/covfn.txt +++ b/covfn.txt @@ -1,21 +1,6 @@ Function Source Line FnCov C/D Coverage ----------------------------------------------------------------------------- --------------------------------------------------- ----- --------------------- -circular_buffer_mannwhitneyu(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 662 0 / 1 0 / 30 = 0% -rank_data(double*[],size_t) .../lua_circular_buffer/lua_circular_buffer.c 618 0 / 1 0 / 16 = 0% -double_pp_compare(const void*,const void*) .../lua_circular_buffer/lua_circular_buffer.c 647 0 / 1 0 / 14 = 0% -circular_buffer_compute(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 552 0 / 1 0 / 13 = 0% -compute_min(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 498 0 / 1 0 / 10 = 0% -compute_max(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 525 0 / 1 0 / 10 = 0% -compute_variance(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 463 0 / 1 0 / 8 = 0% -compute_sum(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 415 0 / 1 0 / 6 = 0% -compute_avg(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 438 0 / 1 0 / 6 = 0% -append_values(circular_buffer*,unsigned,unsigned,unsigned,double[]) .../lua_circular_buffer/lua_circular_buffer.c 602 0 / 1 0 / 4 = 0% lsb_get_parent(lua_sandbox*) ../src/luasandbox.c 457 0 / 1 0 / 0 -bloom_filter_version(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 186 0 / 1 0 / 0 -circular_buffer_get_configuration(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 315 0 / 1 0 / 0 -circular_buffer_get_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 402 0 / 1 0 / 0 -compute_sd(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 490 0 / 1 0 / 0 -circular_buffer_current_time(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 754 0 / 1 0 / 0 circular_buffer_version(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 858 0 / 1 0 / 0 cf_version(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 330 0 / 1 0 / 0 hyperloglog_version(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 152 0 / 1 0 / 0 @@ -23,8 +8,8 @@ lsb_restore_global_data(lua_sandbox*,const char*) . add_table_ref(table_ref_array*,const void*,size_t) ../src/lsb_serialize.c 168 1 / 1 1 / 4 = 25% pb_write_double(lsb_output_data*,double) ../src/lsb_serialize_protobuf.c 58 1 / 1 1 / 4 = 25% pb_write_tag(lsb_output_data*,unsigned char,unsigned char) ../src/lsb_serialize_protobuf.c 105 1 / 1 1 / 4 = 25% -MurmurHash64A(const void*,int,unsigned int) ...Source/lua_hyperloglog/redis_hyperloglog.c 264 1 / 1 3 / 10 = 30% pb_write_string(lsb_output_data*,char,const char*,size_t) ../src/lsb_serialize_protobuf.c 164 1 / 1 3 / 8 = 37% +MurmurHash64A(const void*,int,unsigned int) ...Source/lua_hyperloglog/redis_hyperloglog.c 264 1 / 1 4 / 10 = 40% lsb_appendf(lsb_output_data*,const char*,...) ../src/lsb_output.c 67 1 / 1 9 / 22 = 40% cf_delete(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 262 1 / 1 3 / 7 = 42% serialize_data(lua_sandbox*,int,lsb_output_data*) ../src/lsb_serialize.c 227 1 / 1 10 / 22 = 45% @@ -32,12 +17,9 @@ serialize_bloom_filter(lua_State*) . serialize_cuckoo_filter(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 339 1 / 1 8 / 16 = 50% serialize_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 160 1 / 1 8 / 16 = 50% pb_write_bool(lsb_output_data*,int) ../src/lsb_serialize_protobuf.c 80 1 / 1 3 / 6 = 50% -hyperloglog_fromstring(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 132 1 / 1 3 / 6 = 50% output_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 183 1 / 1 3 / 6 = 50% -bloom_filter_fromstring(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 164 1 / 1 2 / 4 = 50% file_exists(const char*) ../src/lsb_serialize.c 477 1 / 1 1 / 2 = 50% instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 123 1 / 1 1 / 2 = 50% -circular_buffer_get(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 298 1 / 1 1 / 2 = 50% cf_fromstring(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 316 1 / 1 1 / 2 = 50% lsb_serialize_table_as_pb(lua_sandbox*,int) ../src/lsb_serialize_protobuf.c 581 1 / 1 20 / 38 = 52% expand_path(const char*,int,const char*,char*) ../src/luasandbox.c 165 1 / 1 10 / 18 = 55% @@ -47,7 +29,6 @@ lsb_serialize_binary(const void*,size_t,lsb_output_data*) . update_field_length(lsb_output_data*,size_t) ../src/lsb_serialize_protobuf.c 128 1 / 1 6 / 10 = 60% cf_add(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 195 1 / 1 3 / 5 = 60% cf_query(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 228 1 / 1 3 / 5 = 60% -hyperloglog_add(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 59 1 / 1 3 / 5 = 60% lsb_create_custom(void*,const char*,const char*) ../src/luasandbox.c 226 1 / 1 16 / 26 = 61% pb_write_varint(lsb_output_data*,unsigned long long) ../src/lsb_serialize_protobuf.c 29 1 / 1 5 / 8 = 62% lsb_pcall_setup(lua_sandbox*,const char*) ../src/luasandbox.c 493 1 / 1 5 / 8 = 62% @@ -57,8 +38,7 @@ lsb_create(void*,const char*,const char*,unsigned,unsigned,unsigned) . serialize_circular_buffer_delta(lua_State*,circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 974 1 / 1 13 / 20 = 65% encode_fields(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 519 1 / 1 24 / 36 = 66% encode_field_array(lua_sandbox*,lsb_output_data*,int,const char*,int) ../src/lsb_serialize_protobuf.c 269 1 / 1 20 / 30 = 66% -check_row(circular_buffer*,double,int) .../lua_circular_buffer/lua_circular_buffer.c 187 1 / 1 8 / 12 = 66% -hllCount(hyperloglog*) ...Source/lua_hyperloglog/redis_hyperloglog.c 454 1 / 1 11 / 16 = 68% +hyperloglog_fromstring(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 132 1 / 1 4 / 6 = 66% process_fields(lua_State*,unsigned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 104 1 / 1 51 / 74 = 68% nlz(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 70 1 / 1 7 / 10 = 70% output_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 933 1 / 1 17 / 24 = 70% @@ -66,10 +46,9 @@ lsb_serialize_double(lsb_output_data*,double) . encode_field_value(lua_sandbox*,lsb_output_data*,int,const char*,int) ../src/lsb_serialize_protobuf.c 336 1 / 1 75 / 105 = 71% lsb_init(lua_sandbox*,const char*) ../src/luasandbox.c 305 1 / 1 20 / 28 = 71% circular_buffer_set_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 373 1 / 1 10 / 14 = 71% -read_double(char**,double*) .../lua_circular_buffer/lua_circular_buffer.c 781 1 / 1 16 / 22 = 72% lsb_preserve_global_data(lua_sandbox*,const char*) ../src/lsb_serialize.c 366 1 / 1 19 / 26 = 73% serialize_kvp(lua_sandbox*,serialization_data*,size_t) ../src/lsb_serialize.c 287 1 / 1 18 / 24 = 75% -circular_buffer_fromstring(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 830 1 / 1 9 / 12 = 75% +hllCount(hyperloglog*) ...Source/lua_hyperloglog/redis_hyperloglog.c 454 1 / 1 12 / 16 = 75% read_string(lua_State*,int,unsigned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 61 1 / 1 6 / 8 = 75% lsb_usage(lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 405 1 / 1 6 / 8 = 75% circular_buffer_add_delta(lua_State*,circular_buffer*,double,int,double) .../lua_circular_buffer/lua_circular_buffer.c 219 1 / 1 6 / 8 = 75% @@ -80,21 +59,27 @@ output(lua_State*) . lsb_set_error(lua_sandbox*,const char*) ../src/luasandbox.c 438 1 / 1 3 / 4 = 75% lsb_get_output(lua_sandbox*,size_t*) ../src/luasandbox.c 481 1 / 1 3 / 4 = 75% bucket_delete(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 136 1 / 1 3 / 4 = 75% -bloom_filter_add(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 76 1 / 1 7 / 9 = 77% -bloom_filter_query(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 115 1 / 1 7 / 9 = 77% +circular_buffer_mannwhitneyu(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 662 1 / 1 23 / 30 = 76% +compute_max(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 525 1 / 1 8 / 10 = 80% lsb_realloc_output(lsb_output_data*,size_t) ../src/lsb_output.c 132 1 / 1 13 / 16 = 81% bucket_insert(cuckoo_filter*,unsigned,unsigned,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 160 1 / 1 13 / 16 = 81% +read_double(char**,double*) .../lua_circular_buffer/lua_circular_buffer.c 781 1 / 1 18 / 22 = 81% lsb_decode_protobuf(lua_State*) ../src/lsb_deserialize_protobuf.c 262 1 / 1 54 / 65 = 83% lsb_output(lua_sandbox*,int,int,int) ../src/lsb_output.c 153 1 / 1 25 / 30 = 83% lsb_output_protobuf(lua_sandbox*,int,int) ../src/lsb_output.c 215 1 / 1 5 / 6 = 83% -lsb_destroy(lua_sandbox*,const char*) ../src/luasandbox.c 381 1 / 1 12 / 14 = 85% +compute_sum(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 415 1 / 1 5 / 6 = 83% +compute_avg(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 438 1 / 1 5 / 6 = 83% +circular_buffer_compute(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 552 1 / 1 11 / 13 = 84% output_circular_buffer_full(circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 867 1 / 1 12 / 14 = 85% +compute_variance(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 463 1 / 1 7 / 8 = 87% circular_buffer_delta_fromstring(lua_State*,circular_buffer*,char**) .../lua_circular_buffer/lua_circular_buffer.c 804 1 / 1 7 / 8 = 87% hllDenseSum(uint8_t*,double*,int*) ...Source/lua_hyperloglog/redis_hyperloglog.c 356 1 / 1 34 / 38 = 89% read_length(unsigned const char*,unsigned const char*,size_t*) ../src/lsb_deserialize_protobuf.c 27 1 / 1 9 / 10 = 90% read_varint(unsigned const char*,unsigned const char*,long long*) ../src/lsb_deserialize_protobuf.c 44 1 / 1 9 / 10 = 90% +compute_min(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 498 1 / 1 9 / 10 = 90% memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 86 1 / 1 11 / 12 = 91% circular_buffer_add(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 267 1 / 1 11 / 12 = 91% +lsb_destroy(lua_sandbox*,const char*) ../src/luasandbox.c 381 1 / 1 13 / 14 = 92% read_key(unsigned const char*,int*,int*) ../src/lsb_deserialize_protobuf.c 18 1 / 1 0 / 0 update_output_stats(lua_sandbox*) ../src/lsb_output.c 24 1 / 1 2 / 2 = 100% lsb_add_output_function(lua_State*,lua_CFunction) ../src/lsb_output.c 35 1 / 1 0 / 0 @@ -120,17 +105,31 @@ lsb_pcall_teardown(lua_sandbox*) . lsb_terminate(lua_sandbox*,const char*) ../src/luasandbox.c 524 1 / 1 4 / 4 = 100% bloom_filter_new(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 36 1 / 1 0 / 0 check_bloom_filter(lua_State*,int) ...Source/lua_bloom_filter/lua_bloom_filter.c 66 1 / 1 0 / 0 +bloom_filter_add(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 76 1 / 1 9 / 9 = 100% +bloom_filter_query(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 115 1 / 1 9 / 9 = 100% bloom_filter_count(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 147 1 / 1 0 / 0 bloom_filter_clear(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 155 1 / 1 0 / 0 +bloom_filter_fromstring(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 164 1 / 1 4 / 4 = 100% +bloom_filter_version(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 186 1 / 1 0 / 0 luaopen_bloom_filter(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 242 1 / 1 0 / 0 get_start_time(circular_buffer*) .../lua_circular_buffer/lua_circular_buffer.c 81 1 / 1 0 / 0 copy_cleared_row(circular_buffer*,double*,size_t) .../lua_circular_buffer/lua_circular_buffer.c 87 1 / 1 4 / 4 = 100% clear_rows(circular_buffer*,unsigned) .../lua_circular_buffer/lua_circular_buffer.c 106 1 / 1 10 / 10 = 100% circular_buffer_new(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 130 1 / 1 4 / 4 = 100% check_circular_buffer(lua_State*,int) .../lua_circular_buffer/lua_circular_buffer.c 177 1 / 1 0 / 0 +check_row(circular_buffer*,double,int) .../lua_circular_buffer/lua_circular_buffer.c 187 1 / 1 12 / 12 = 100% check_column(lua_State*,circular_buffer*,int) .../lua_circular_buffer/lua_circular_buffer.c 209 1 / 1 0 / 0 +circular_buffer_get(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 298 1 / 1 2 / 2 = 100% +circular_buffer_get_configuration(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 315 1 / 1 0 / 0 +circular_buffer_get_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 402 1 / 1 0 / 0 +compute_sd(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 490 1 / 1 0 / 0 +append_values(circular_buffer*,unsigned,unsigned,unsigned,double[]) .../lua_circular_buffer/lua_circular_buffer.c 602 1 / 1 4 / 4 = 100% +rank_data(double*[],size_t) .../lua_circular_buffer/lua_circular_buffer.c 618 1 / 1 16 / 16 = 100% +double_pp_compare(const void*,const void*) .../lua_circular_buffer/lua_circular_buffer.c 647 1 / 1 14 / 14 = 100% +circular_buffer_current_time(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 754 1 / 1 0 / 0 circular_buffer_format(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 761 1 / 1 0 / 0 read_time_row(char**,circular_buffer*) .../lua_circular_buffer/lua_circular_buffer.c 774 1 / 1 0 / 0 +circular_buffer_fromstring(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 830 1 / 1 12 / 12 = 100% luaopen_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 1102 1 / 1 0 / 0 clp2(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 51 1 / 1 0 / 0 cf_new(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 85 1 / 1 0 / 0 @@ -143,10 +142,11 @@ cf_clear(lua_State*) . luaopen_cuckoo_filter(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 387 1 / 1 0 / 0 hyperloglog_new(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 29 1 / 1 0 / 0 check_hyperloglog(lua_State*,int) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 49 1 / 1 0 / 0 +hyperloglog_add(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 59 1 / 1 5 / 5 = 100% hyperloglog_count(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 90 1 / 1 2 / 2 = 100% hyperloglog_clear(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 123 1 / 1 0 / 0 luaopen_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 213 1 / 1 0 / 0 hllPatLen(unsigned char*,size_t,long*) ...Source/lua_hyperloglog/redis_hyperloglog.c 323 1 / 1 2 / 2 = 100% hllDenseAdd(uint8_t*,unsigned char*,size_t) ...Source/lua_hyperloglog/redis_hyperloglog.c 426 1 / 1 2 / 2 = 100% ----------------------------------------------------------------------------- --------------------------------------------------- ----- --------------------- -Total 87% 904 / 1419 = 63% +Total 97% 1028 / 1419 = 72% diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index fbbdff2..222b4ba 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -578,6 +578,23 @@ static char* test_cbuf_errors() } +static char* test_cbuf_core() +{ + lua_sandbox* sb = lsb_create(NULL, "../../ep_base/Source/lua_circular_buffer/test.lua", + "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + e = lsb_destroy(sb, ""); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* test_cbuf() { const char* outputs[] = { @@ -955,6 +972,23 @@ static char* test_serialize_failure() } +static char* test_bloom_filter_core() +{ + lua_sandbox* sb = lsb_create(NULL, "../../ep_base/Source/lua_bloom_filter/test.lua", + "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + e = lsb_destroy(sb, ""); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* test_bloom_filter() { const char* output_file = "bloom_filter.preserve"; @@ -1033,6 +1067,23 @@ static char* test_bloom_filter() } +static char* test_hyperloglog_core() +{ + lua_sandbox* sb = lsb_create(NULL, "../../ep_base/Source/lua_hyperloglog/test.lua", + "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + e = lsb_destroy(sb, ""); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* test_hyperloglog() { const char* output_file = "hyperloglog.preserve"; @@ -1577,6 +1628,7 @@ static char* all_tests() mu_run_test(test_output); mu_run_test(test_output_errors); mu_run_test(test_cbuf_errors); + mu_run_test(test_cbuf_core); mu_run_test(test_cbuf); mu_run_test(test_cbuf_delta); mu_run_test(test_cjson); @@ -1587,7 +1639,9 @@ static char* all_tests() mu_run_test(test_serialize); mu_run_test(test_restore); mu_run_test(test_serialize_failure); + mu_run_test(test_bloom_filter_core); mu_run_test(test_bloom_filter); + mu_run_test(test_hyperloglog_core); mu_run_test(test_hyperloglog); mu_run_test(test_struct); mu_run_test(test_sandbox_config); From 96c0ecf243141790f49577fe65937eb47f9c3778 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 25 May 2015 16:22:08 -0400 Subject: [PATCH 059/235] Fix the RPM install --- CMakeLists.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a3a8f9..b392d81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,8 @@ if(LUA_JIT) endif() find_package(Git REQUIRED) +set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") +set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(mozsvc) From 3bdd565dcdb4b0727444a4acc9464a792c14a4da Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Tue, 21 Jul 2015 13:46:14 -0700 Subject: [PATCH 060/235] Fix up the Visual Studio build --- cmake/externals.cmake | 27 ++++++++++++++++----------- src/CMakeLists.txt | 2 +- src/lsb_deserialize_protobuf.c | 6 +++--- src/lsb_serialize_protobuf.c | 7 ++++--- src/test/CMakeLists.txt | 2 +- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 25cf053..49b3009 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -16,6 +16,11 @@ set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include -DUSE_RPATH=false --no-warn-unused-cli) set(LUA_INCLUDE_DIR "${EP_BASE}/include/${PROJECT_NAME}") +set(INST_ARGS "install/strip") +if(MSVC) + set(INST_ARGS "install") +endif() + if (LUA_JIT) message(FATAL_ERROR, "LuaJIT support has not been added back in yet, issue #66") set(LUA_PROJECT "luajit-2_0_2") @@ -62,7 +67,7 @@ else() GIT_REPOSITORY https://github.com/trink/lua.git GIT_TAG 477c0b5ae0932953ee41ab7cd24098c23e3e33c3 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) endif() @@ -72,7 +77,7 @@ externalproject_add( GIT_TAG baf0dc90b9278360be719dbfb8e56d34ce3c61bd UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) add_dependencies(lua_lpeg ${LUA_PROJECT}) @@ -81,7 +86,7 @@ externalproject_add( GIT_REPOSITORY https://github.com/trink/lua-cjson.git GIT_TAG 3c8321cbfc6bea944974e046da82deca38e66587 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) add_dependencies(lua_cjson ${LUA_PROJECT}) @@ -91,7 +96,7 @@ externalproject_add( GIT_TAG 5cf31819bee0d829d058cb5219e95ef0b1dd43a8 UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) add_dependencies(lua_struct ${LUA_PROJECT}) @@ -101,7 +106,7 @@ externalproject_add( GIT_TAG b97ed47e7a01e0d2523809efe11676333a85dcab UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) add_dependencies(lua_socket ${LUA_PROJECT}) @@ -110,9 +115,9 @@ add_dependencies(lua_socket ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG 1afb5fe85adec5750238c5ebd7d6d95e7993f122 + GIT_TAG 4151be752c3dd2f74d1c3487d8352ca54055eb81 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) externalproject_add( @@ -120,7 +125,7 @@ externalproject_add( GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git GIT_TAG e87bcc0a3ae29c5a7acaabbb7b1f2b56700769c6 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) externalproject_add( @@ -128,13 +133,13 @@ externalproject_add( GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git GIT_TAG 92f8a59e9e6f4ee381482f449b6cee78bb55460a CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) externalproject_add( lua_cuckoo_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG cfb311a433ed7e4c8224ca99ab58b54302037b6d + GIT_TAG 8343207aa7656d0324d23f79a5732fcb0bdb5917 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS "install/strip" + INSTALL_ARGS ${INST_ARGS} ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a3f6979..c71b3e1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -32,7 +32,7 @@ elseif(MINGW) add_definitions(-D_MINGW) set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/libluasb.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) - install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib PATTERN "*.dll" COMPONENT core) + install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib COMPONENT core PATTERN "*.dll") else() if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c index 2c69021..f1741ed 100644 --- a/src/lsb_deserialize_protobuf.c +++ b/src/lsb_deserialize_protobuf.c @@ -94,7 +94,7 @@ process_varint(lua_State* lua, if (!p) { return NULL; } - lua_pushnumber(lua, val); + lua_pushnumber(lua, (lua_Number)val); lua_setfield(lua, stack_index, name); return p; } @@ -223,7 +223,7 @@ process_fields(lua_State* lua, case 0: p = read_varint(p, p + len, &val); if (!p) break; - lua_pushboolean(lua, val); + lua_pushboolean(lua, (int)val); lua_rawseti(lua, 5, ++value_count); break; case 2: @@ -235,7 +235,7 @@ process_fields(lua_State* lua, do { p = read_varint(p, p + len, &val); if (!p) break; - lua_pushboolean(lua, val); + lua_pushboolean(lua, (int)val); lua_rawseti(lua, 5, ++value_count); } while (p < e); diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c index 23a3e36..e66657f 100644 --- a/src/lsb_serialize_protobuf.c +++ b/src/lsb_serialize_protobuf.c @@ -288,7 +288,7 @@ encode_field_array(lua_sandbox* lsb, lsb_output_data* d, int t, // the next interation. } if (multiple && value_type == 2) { // fix up the varint packed length - int i = len_pos - 2; + size_t i = len_pos - 2; int y = 0; while (d->data[i] != 0 && y < MAX_VARINT_BYTES) { // find the length byte --i; @@ -323,7 +323,7 @@ static int encode_field_object(lua_sandbox* lsb, lsb_output_data* d) } lua_getfield(lsb->lua, -2, "value_type"); if (lua_isnumber(lsb->lua, -1)) { - value_type = lua_tointeger(lsb->lua, -1); + value_type = (int)lua_tointeger(lsb->lua, -1); } lua_getfield(lsb->lua, -3, "value"); result = encode_field_value(lsb, d, 1, representation, value_type); @@ -468,6 +468,7 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, case LUA_TUSERDATA: { lua_CFunction fp = lsb_get_output_function(lsb->lua, -1); + size_t len_pos = 0; if (!fp) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "user data object does not implement lsb_output"); @@ -484,7 +485,7 @@ encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, } if (pb_write_tag(d, 5, 2)) return 1; - size_t len_pos = d->pos; + len_pos = d->pos; if (pb_write_varint(d, 0)) return 1; // length tbd later lua_pushlightuserdata(lsb->lua, d); if (fp(lsb->lua)) return 1; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index c3d376b..b167b79 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -8,7 +8,7 @@ target_link_libraries(test_lua_sandbox luasandbox) add_test(NAME test_move_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) if(WIN32) add_test(NAME test_move_lua_lib COMMAND cmake -E copy - ${CMAKE_BINARY_DIR}/ep_base/bin/${CMAKE_SHARED_MODULE_PREFIX}lua${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) + ${CMAKE_BINARY_DIR}/ep_base/bin/${CMAKE_SHARED_MODULE_PREFIX}luasb${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_MODULE_PREFIX}luasandbox${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) elseif(APPLE) From 82ae00d85f3111ee77bf15dbcb97264f177a66ec Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Wed, 22 Jul 2015 09:25:59 -0700 Subject: [PATCH 061/235] Fix the packaging of the Visual studio build - remove wlua - install the luasb.dll --- cmake/externals.cmake | 2 +- src/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 49b3009..0d3e4e7 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -65,7 +65,7 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG 477c0b5ae0932953ee41ab7cd24098c23e3e33c3 + GIT_TAG 57d3e82ef270b20dc976f3a6001439589c807793 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c71b3e1..a603a26 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,6 +28,7 @@ add_dependencies(lua_cuckoo_filter luasandbox) if(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/luasb.lib") + install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib COMPONENT core PATTERN "*.dll") elseif(MINGW) add_definitions(-D_MINGW) set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/libluasb.dll") From 459cc2759ac6bc3b6772b8dc29563be0d329fce7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 31 Jul 2015 07:49:15 -0700 Subject: [PATCH 062/235] Switch usage stats from unsigned to size_t --- include/luasandbox.h | 4 ++-- src/lsb_output.c | 2 +- src/lsb_private.h | 2 +- src/lsb_serialize.c | 4 ++-- src/luasandbox.c | 23 +++++++++++---------- src/test/test_lua_sandbox.c | 40 ++++++++++++++++++------------------- 6 files changed, 38 insertions(+), 37 deletions(-) diff --git a/include/luasandbox.h b/include/luasandbox.h index 16985b2..59c7e2d 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -135,9 +135,9 @@ LSB_EXPORT char* lsb_destroy(lua_sandbox* lsb, const char* state_file); * @param lsb_usage_type Type of statistic to retrieve i.e. memory. * @param lsb_usage_stat Type of statistic to retrieve i.e. current. * - * @return unsigned Count or number of bytes depending on the statistic. + * @return size_t Count or number of bytes depending on the statistic. */ -LSB_EXPORT unsigned +LSB_EXPORT size_t lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, lsb_usage_stat ustat); /** * Retrieve the current sandbox status. diff --git a/src/lsb_output.c b/src/lsb_output.c index d7e7c50..f0b1bcd 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -23,7 +23,7 @@ static const char* output_function = "lsb_output"; static void update_output_stats(lua_sandbox* lsb) { - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = (unsigned)lsb->output.pos; + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = lsb->output.pos; if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = diff --git a/src/lsb_private.h b/src/lsb_private.h index 083c21d..7b482c4 100644 --- a/src/lsb_private.h +++ b/src/lsb_private.h @@ -32,7 +32,7 @@ struct lua_sandbox { lsb_state state; lsb_output_data output; char* lua_file; - unsigned usage[LSB_UT_MAX][LSB_US_MAX]; + size_t usage[LSB_UT_MAX][LSB_US_MAX]; char error_message[LSB_ERROR_SIZE]; }; diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index b3e13ab..ecab416 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -401,7 +401,7 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) #ifdef LUA_JIT lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); #else -// unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; +// size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif lua_sethook(lsb->lua, NULL, 0, 0); @@ -493,7 +493,7 @@ int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) #ifdef LUA_JIT lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); #else - unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; + size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif lua_sethook(lsb->lua, NULL, 0, 0); diff --git a/src/luasandbox.c b/src/luasandbox.c index 3b8ccad..459f163 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -91,10 +91,10 @@ static void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize) void* nptr = NULL; if (nsize == 0) { free(ptr); - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= (unsigned)osize; + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= osize; } else { - unsigned new_state_memory = - (unsigned)(lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize); + size_t new_state_memory = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize; if (0 == lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] || new_state_memory <= lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]) { @@ -154,10 +154,10 @@ static int unprotected_panic(lua_State* lua) } -static unsigned get_usage_config(lua_State* lua, int idx, const char* item) +static size_t get_usage_config(lua_State* lua, int idx, const char* item) { lua_getfield(lua, idx, item); - unsigned u = (unsigned)lua_tonumber(lua, -1); // defaults to zero + size_t u = (size_t)lua_tointeger(lua, -1); // defaults to zero lua_pop(lua, 1); return u; } @@ -288,9 +288,9 @@ lua_sandbox* lsb_create_custom(void* parent, free(lsb); return NULL; } - unsigned memory_limit = get_usage_config(lsb->lua, -1, "memory_limit"); - unsigned instruction_limit = get_usage_config(lsb->lua, -1, "instruction_limit"); - unsigned output_limit = get_usage_config(lsb->lua, -1, "output_limit"); + size_t memory_limit = get_usage_config(lsb->lua, -1, "memory_limit"); + size_t instruction_limit = get_usage_config(lsb->lua, -1, "instruction_limit"); + size_t output_limit = get_usage_config(lsb->lua, -1, "output_limit"); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, "lsb_config"); lua_pop(lsb->lua, 1); // remove configuration string @@ -329,7 +329,7 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) return 0; } #ifndef LUA_JIT - unsigned mem_limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; + size_t mem_limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif @@ -365,6 +365,7 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); #ifdef LUA_JIT + // todo limit lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, (int)lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); #else @@ -423,8 +424,8 @@ char* lsb_destroy(lua_sandbox* lsb, const char* data_file) } -unsigned lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, - lsb_usage_stat ustat) +size_t lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, + lsb_usage_stat ustat) { if (!lsb || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { return 0; diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 222b4ba..3dfa5d4 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -289,22 +289,22 @@ static char* test_destroy_error() static char* test_usage_error() { - unsigned u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "NULL sandbox memory usage received: %u", u); + size_t u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); + mu_assert(u == 0, "NULL sandbox memory usage received: %zu", u); lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); u = lsb_usage(NULL, LSB_UT_MAX + 1, LSB_US_CURRENT); - mu_assert(u == 0, "Invalid usage type received: %u", u); + mu_assert(u == 0, "Invalid usage type received: %zu", u); u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_MAX + 1); - mu_assert(u == 0, "Invalid usage stat received: %u", u); + mu_assert(u == 0, "Invalid usage stat received: %zu", u); mu_assert(sb, "lsb_create() received: NULL"); lsb_terminate(sb, "forced termination"); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "Terminated memory usage received: %u", u); + mu_assert(u == 0, "Terminated memory usage received: %zu", u); e = lsb_destroy(sb, NULL); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -335,39 +335,39 @@ static char* test_simple() mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - unsigned u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u > 0, "Current memory usage received: %u", u); - printf("cur_mem %u\n", u); + size_t u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); + mu_assert(u > 0, "Current memory usage received: %zu", u); + printf("cur_mem %zu\n", u); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_MAXIMUM); - mu_assert(u > 0, "Maximum memory usage received: %u", u); - printf("max_mem %u\n", u); + mu_assert(u > 0, "Maximum memory usage received: %zu", u); + printf("max_mem %zu\n", u); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_LIMIT); - mu_assert(u == 65765, "Memory limit received: %u", u); + mu_assert(u == 65765, "Memory limit received: %zu", u); u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_CURRENT); - mu_assert(u == 7, "Current instructions received: %u", u); + mu_assert(u == 7, "Current instructions received: %zu", u); u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_MAXIMUM); - mu_assert(u == 7, "Maximum instructions received: %u", u); - printf("max_ins %u\n", u); + mu_assert(u == 7, "Maximum instructions received: %zu", u); + printf("max_ins %zu\n", u); u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_LIMIT); - mu_assert(u == 1000, "Instruction limit received: %u", u); + mu_assert(u == 1000, "Instruction limit received: %zu", u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_CURRENT); - mu_assert(u == 0, "Current output received: %u", u); + mu_assert(u == 0, "Current output received: %zu", u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_MAXIMUM); - mu_assert(u == 0, "Maximum output received: %u", u); - printf("max_out %u\n", u); + mu_assert(u == 0, "Maximum output received: %zu", u); + printf("max_out %zu\n", u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); - mu_assert(u == 1024, "Output limit received: %u", u); + mu_assert(u == 1024, "Output limit received: %zu", u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); - mu_assert(u == 1024, "Output limit received: %u", u); + mu_assert(u == 1024, "Output limit received: %zu", u); lsb_state s = lsb_get_state(sb); mu_assert(s == LSB_RUNNING, "lsb_get_state() received: %d", s); From f3a2221afdfe5bd3a2cbda1e9090c0c59f1111a7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 31 Jul 2015 07:52:06 -0700 Subject: [PATCH 063/235] Clear the hook if the instruction limit is set to unlimited (0) - prevents the hook count from underflowing --- src/luasandbox.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/luasandbox.c b/src/luasandbox.c index 459f163..4bd2015 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -362,8 +362,12 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) lua_pushcclosure(lsb->lua, &output, 1); lua_setglobal(lsb->lua, "output"); - lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { + lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, + (int)lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); + } else { + lua_sethook(lsb->lua, NULL, 0, 0); + } #ifdef LUA_JIT // todo limit lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, @@ -514,9 +518,12 @@ const char* lsb_get_output(lua_sandbox* lsb, size_t* len) int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name) { - - lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { + lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, + (int)lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); + } else { + lua_sethook(lsb->lua, NULL, 0, 0); + } lua_getglobal(lsb->lua, func_name); if (!lua_isfunction(lsb->lua, -1)) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, From 4d52f11d20703d6cd811f52576efa1cb5d3de1a4 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Wed, 29 Jul 2015 00:01:21 +0200 Subject: [PATCH 064/235] syslog.lua: parse and return RFC 5424 structured data --- modules/syslog.lua | 24 ++++++++++++++++++------ src/test/lua/lpeg_syslog.lua | 29 +++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/modules/syslog.lua b/modules/syslog.lua index 9e986dc..2bb4314 100644 --- a/modules/syslog.lua +++ b/modules/syslog.lua @@ -13,10 +13,19 @@ local tonumber = tonumber local error = error local type = type local ipairs = ipairs +local rawset = rawset local M = {} setfenv(1, M) -- Remove external access to contain everything in the module +local function unescape_param_value(param_value) + return string.gsub(param_value, '\\([]"\\])', '%1') +end + +local function prefix_param_name(param_name) + return '_' .. param_name +end + -- http://tools.ietf.org/html/rfc5424#page-8 local octet = l.P(1) local utf8_string = octet^0 @@ -30,12 +39,15 @@ local msg_any = octet^0 local msg_utf8 = bom * utf8_string local msg = msg_utf8 + msg_any local hostname = nilvalue + printusascii^-255 -local sd_name = printusascii^-32 -local param_name = sd_name -local param_value = utf8_string -local sd_param = param_name * '="' * param_value * '"' -local sd_id = sd_name -local sd_element = l.P"[" * sd_id * (sp * sd_param)^0 * "]" +local sd_name = (printusascii - l.S'=]" ')^-32 +local param_name = sd_name / prefix_param_name +local esc = l.P'\\' +local param_value_esc = l.S'"\\]' +local param_value = ((octet - param_value_esc) + (esc * octet))^0 / unescape_param_value +local sd_id = l.Cg(l.Cc"id" * l.C(sd_name)) +local sd_param = l.Cg(param_name * '="' * param_value * '"') +local sd_params = l.Cf(l.Ct"" * sd_id * (sp * sd_param)^0, rawset) +local sd_element = l.P"[" * sd_params * "]" local syslog_facility = digit^-3 / tonumber local syslog_severity = digit / tonumber diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua index f2b5616..48d6d6d 100644 --- a/src/test/lua/lpeg_syslog.lua +++ b/src/test/lua/lpeg_syslog.lua @@ -98,6 +98,34 @@ local function no_colon_in_tag() assert(fields.hostname == "example.com", fields.hostname) end +local function structured_data() + local grammar = syslog.build_rsyslog_grammar("%STRUCTURED-DATA%") + local log = '[origin@123 name1="first value" name2="sec\\ond\\\\v[a\\]l\\"ue"]' + local fields = grammar:match(log) + local sd = fields["structured-data"] + assert(sd["name1"] == "first value", sd["name1"]) + assert(sd["name2"] == "sec\\ond\\v[a]l\"ue", sd["name2"]) + assert(sd["id"] == "origin@123", sd["id"]) + + log = '[origin@123 name="value\\\\"]' + fields = grammar:match(log) + sd = fields["structured-data"] + assert(sd["name"] == "value\\", sd["name"]) + assert(sd["id"] == "origin@123", sd["id"]) + + log = '[origin@123 name="value\\\""]' + fields = grammar:match(log) + sd = fields["structured-data"] + assert(sd["name"] == 'value"', sd["name"]) + assert(sd["id"] == "origin@123", sd["id"]) + + log = '[origin@123 name=""]' + fields = grammar:match(log) + sd = fields["structured-data"] + assert(sd["name"] == "", sd["name"]) + assert(sd["id"] == "origin@123", sd["id"]) +end + function process() traditional_file_format() file_format() @@ -105,6 +133,7 @@ function process() traditional_forward_format() kitchen_sink() no_colon_in_tag() + structured_data() return 0 end From b4929c9f0b25fa71261625fe96c14a61f1235099 Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Mon, 3 Aug 2015 15:35:20 +0200 Subject: [PATCH 065/235] syslog.lua: allow nil values for structured RFC 5424 data --- modules/syslog.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/syslog.lua b/modules/syslog.lua index 2bb4314..4f4ced1 100644 --- a/modules/syslog.lua +++ b/modules/syslog.lua @@ -162,7 +162,7 @@ local rsyslog_properties = { timereported = lookup_time_format, timestamp = lookup_time_format, ["protocol-version"] = digit^1, - ["structured-data"] = nilvalue + sd_element, + ["structured-data"] = (l.Ct"" * nilvalue) + sd_element, ["app-name"] = nilvalue + printusascii^-48, procid = nilvalue + printusascii^-128, msgid = nilvalue + printusascii^-32, From 67580f8bdcddadf697969d17155e6e30fef7f56f Mon Sep 17 00:00:00 2001 From: Frank Denis Date: Mon, 3 Aug 2015 17:58:38 +0200 Subject: [PATCH 066/235] syslog.lua: update tests after property name changes --- src/test/lua/lpeg_syslog.lua | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua index 48d6d6d..799e1dc 100644 --- a/src/test/lua/lpeg_syslog.lua +++ b/src/test/lua/lpeg_syslog.lua @@ -71,7 +71,7 @@ local function kitchen_sink() -- time reported is mapped to timestamp assert(fields.timestamp == 1392052853559934000, fields.timestamp) assert(fields["protocol-version"] == "0", fields["protocol-version"]) - assert(fields["structured-data"] == "-", fields["structured-data"]) + assert(type(fields["structured-data"]) == "table", type(fields["structured-data"])) assert(fields["app-name"] == "kernel", fields["app-name"]) assert(fields.procid == "", fields.procid) assert(fields.msgid == "-", fields.msgid) @@ -103,26 +103,26 @@ local function structured_data() local log = '[origin@123 name1="first value" name2="sec\\ond\\\\v[a\\]l\\"ue"]' local fields = grammar:match(log) local sd = fields["structured-data"] - assert(sd["name1"] == "first value", sd["name1"]) - assert(sd["name2"] == "sec\\ond\\v[a]l\"ue", sd["name2"]) + assert(sd["_name1"] == "first value", sd["_name1"]) + assert(sd["_name2"] == "sec\\ond\\v[a]l\"ue", sd["_name2"]) assert(sd["id"] == "origin@123", sd["id"]) log = '[origin@123 name="value\\\\"]' fields = grammar:match(log) sd = fields["structured-data"] - assert(sd["name"] == "value\\", sd["name"]) + assert(sd["_name"] == "value\\", sd["_name"]) assert(sd["id"] == "origin@123", sd["id"]) log = '[origin@123 name="value\\\""]' fields = grammar:match(log) sd = fields["structured-data"] - assert(sd["name"] == 'value"', sd["name"]) + assert(sd["_name"] == 'value"', sd["_name"]) assert(sd["id"] == "origin@123", sd["id"]) log = '[origin@123 name=""]' fields = grammar:match(log) sd = fields["structured-data"] - assert(sd["name"] == "", sd["name"]) + assert(sd["_name"] == "", sd["_name"]) assert(sd["id"] == "origin@123", sd["id"]) end From ef72f4f810231e6013893ae9e7898892afa0c9b7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 26 Aug 2015 06:40:50 -0700 Subject: [PATCH 067/235] Pull in lua-cjson with a max encode_number_precision of 17 --- cmake/externals.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 0d3e4e7..a88110e 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -84,7 +84,7 @@ add_dependencies(lua_lpeg ${LUA_PROJECT}) externalproject_add( lua_cjson GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG 3c8321cbfc6bea944974e046da82deca38e66587 + GIT_TAG b24f724a4982531e392a88679b3783fdc5361c5d CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From 39bd64efb314ff5da4b93c524668616e677c51aa Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Mon, 13 Jul 2015 16:47:21 +0200 Subject: [PATCH 068/235] Postfix log parser --- README.md | 1 + modules/postfix.lua | 524 ++++++++++++++++ src/test/lua/lpeg_postfix.lua | 1100 +++++++++++++++++++++++++++++++++ src/test/test_lua_sandbox.c | 1 + 4 files changed, 1626 insertions(+) create mode 100644 modules/postfix.lua create mode 100644 src/test/lua/lpeg_postfix.lua diff --git a/README.md b/README.md index 762b1ba..75f0748 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,7 @@ By default only the base library is loaded additional libraries must be loaded w - data_time - RFC3339, RFC3164, strftime, common log format, MySQL and Postgres timestamps - ip_address - IPv4 and IPv6 address - mysql - MySQL and MariaDB slow query and short slow query parsers + - postfix - Postfix messages - syslog - Rsyslog meta grammar generator (creates a grammar based on the template configuration) - _user provided_ (lua, so/dll) diff --git a/modules/postfix.lua b/modules/postfix.lua new file mode 100644 index 0000000..07d12dd --- /dev/null +++ b/modules/postfix.lua @@ -0,0 +1,524 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- Copyright 2015 Mathieu Parent + +local string = require 'string' +local ip = require 'ip_address' +local l = require 'lpeg' +l.locale(l) +local ipairs = ipairs +local pairs = pairs +local rawset = rawset +local tonumber = tonumber +local type = type + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +-- Basic types +local notspace = l.P(1) - l.space +local integer = l.digit^1 / tonumber +local number = (l.digit^1 * (l.P'.' * l.digit^1)^-1) / tonumber + +-- keyvalue_data +local kv_key = l.C(l.alpha^1) +local kv_pair_end = (l.P', ' * l.alpha^1 * l.P'=') + (l.P',' * -1) + -1 +local kv_value = l.P'<' * l.C((l.P(1) - (l.P'>' * kv_pair_end))^0) * l.P'>' + + l.C((l.P(1) - kv_pair_end)^0) +local kv_sep = l.P', ' + (l.P',' * -1) + -1 +local kv_pair = l.Cg(kv_key * '=' * kv_value) * kv_sep^-1 +local kv_grammar = l.Cf(l.Ct'' * kv_pair^0, rawset) + +-- Hostname and IPs +local ipv46 = l.Ct(l.Cg(ip.v4, 'value') * l.Cg(l.Cc'ipv4', 'representation')) + + l.Ct(l.Cg(ip.v6, 'value') * l.Cg(l.Cc'ipv6', 'representation')) +local hostname = l.Ct(l.Cg((l.P(1) - l.S'[] ')^1, 'value') * l.Cg(l.Cc'hostname', 'representation')) +local host = ipv46 + hostname +local port = l.Ct(l.Cg(integer, 'value') * l.Cg(l.Cc'2', 'value_type')) + +-- Postfix grammar +-- Inspired by https://github.com/whyscream/postfix-grok-patterns +-- 2015-05-26 8fdd562d159845b97bf8d9c273e5357f3a143a94 +-- Original under BSD-3-clause license +-- Adaptation 2015 Mathieu Parent +local postfix_queueid = l.xdigit ^ 6 + l.alnum ^ 15 + l.P'NOQUEUE' +local postfix_queueid_cg = l.Cg(postfix_queueid, 'postfix_queueid') + +local postfix_client_hostname_cg = l.Cg(host, 'postfix_client_hostname') +local postfix_client_ip_cg = l.Cg(ipv46, 'postfix_client_ip') +local postfix_client_port_cg = l.Cg(port, 'postfix_client_port') +local postfix_client_info_cg = postfix_client_hostname_cg^-1 + * l.P'[' * postfix_client_ip_cg * l.P']' + * (l.P':' * postfix_client_port_cg)^-1 +local postfix_client_info_ct = l.Ct(postfix_client_info_cg) + +local postfix_relay_hostname_cg = l.Cg(host, 'postfix_relay_hostname') +local postfix_relay_ip_cg = l.Cg(ipv46, 'postfix_relay_ip') +local postfix_relay_service_cg = l.Cg((l.P(1)-l.P']')^1, 'postfix_relay_service') +local postfix_relay_port_cg = l.Cg(port, 'postfix_relay_port') +local postfix_relay_info_cg = postfix_relay_hostname_cg^-1 + * l.P'[' * (postfix_relay_ip_cg + postfix_relay_service_cg) * l.P']' + * (l.P':' * postfix_relay_port_cg)^-1 +local postfix_relay_info_ct = l.Ct(postfix_relay_info_cg) + +local postfix_smtp_stage = l.P'CONNECT' + + l.P'HELO' + + l.P'EHLO' + + l.P'STARTTLS' + + l.P'AUTH' + + l.P'MAIL' + + l.P'RCPT' + + l.P'DATA' + + l.P'RSET' + + l.P'UNKNOWN' + + l.P'END-OF-MESSAGE' + + l.P'VRFY' + + l.P'.' +local postfix_smtp_stage_cg = l.Cg(postfix_smtp_stage, 'postfix_smtp_stage') +local postfix_proxy_smtp_stage_cg = l.Cg(postfix_smtp_stage, 'postfix_proxy_smtp_stage') + +local postfix_action = l.P'reject' + + l.P'defer' + + l.P'accept' + + l.P'header-redirect' +local postfix_action_cg = l.Cg(postfix_action, 'postfix_action') +local postfix_proxy_result_cg = l.Cg(postfix_action, 'postfix_proxy_result') + +local postfix_status_code =( l.digit * l.digit * l.digit) / tonumber +local postfix_status_code_cg = l.Cg(postfix_status_code, 'postfix_status_code') +local postfix_status_code_enhanced = l.digit * l.P'.' * l.digit * l.P'.' * l.digit +local postfix_status_code_enhanced_cg = l.Cg(postfix_status_code_enhanced, 'postfix_status_code_enhanced') +local postfix_dnsbl_message_cg = l.P'Service unavailable; ' + * (l.P(1) - l.P'[')^0 + * l.P'[' + * l.Cg((l.P(1) - l.P']')^0, 'postfix_status_data') + * l.P'] ' + * l.Cg((l.P(1)-(l.P'; ' * l.alpha^1 * l.P'='))^1, 'postfix_status_message') + * l.P';' +local postfix_ps_access_action = l.P'DISCONNECT' + + l.P'BLACKLISTED' + + l.P'WHITELISTED' + + l.P'WHITELIST VETO' + + l.P'PASS NEW' + + l.P'PASS OLD' +local postfix_ps_violation_cg = l.Cg(l.P'BARE NEWLINE' + + l.P'COMMAND TIME LIMIT' + + l.P'COMMAND COUNT LIMIT' + + l.P'COMMAND LENGTH LIMIT' + + l.P'COMMAND PIPELINING' + + l.P'DNSBL' + + l.P'HANGUP' + + l.P'NON-SMTP COMMAND' + + l.P'PREGREET', + 'postfix_postscreen_violation') +local postfix_time_unit = l.digit^1 * l.S'smhd' +local postfix_keyvalue_greedy_cg = postfix_queueid_cg + * l.P': ' + * l.Cg(l.P(1)^1, 'postfix_keyvalue_data') +local postfix_warning_cg = (l.P'warning' + l.P'fatal') * ': ' * l.Cg(l.P(1)^1, 'postfix_warning') +local postfix_tlsconn_cg = (l.P'Anonymous' + l.P'Trusted' + l.P'Untrusted' + l.P'Verified') + * l.P' TLS connection established ' + * ((l.P'to ' * postfix_relay_info_cg) + (l.P'from ' * postfix_client_info_cg)) * l.P': ' + * l.Cg(notspace^0, 'postfix_tls_version') + * l.P' with cipher ' + * l.Cg(notspace^0, 'postfix_tls_cipher') + * l.P' (' * l.Cg(notspace^0, 'postfix_tls_cipher_size') * l.P' bits)' +local postfix_delays_ct = l.Ct(l.Cg(number, 'postfix_delay_before_qmgr') + * l.P'/' + * l.Cg(number, 'postfix_delay_in_qmgr') + * l.P'/' + * l.Cg(number, 'postfix_delay_conn_setup') + * l.P'/' + * l.Cg(number, 'postfix_delay_transmission')) +local postfix_lostconn = l.P'lost connection' + + l.P'timeout' + + l.P'Connection timed out' +local postfix_smtpd_lostconn_data_cg = l.Cg(postfix_lostconn, 'postfix_smtpd_lostconn_data') +local postfix_proxy_message_cg = (l.Cg(postfix_status_code, 'postfix_proxy_status_code') * l.P' ')^-1 * (l.Cg(postfix_status_code_enhanced, 'postfix_proxy_status_code_enhanced') * l.P' ')^-1 * (l.P(1) - l.P';')^0 + +-- smtpd patterns +local postfix_smtpd_connect_cg = l.P'connect from ' + * postfix_client_info_cg +local postfix_smtpd_disconnect_cg = l.P'disconnect from ' + * postfix_client_info_cg +local postfix_smtpd_lostconn_cg = (postfix_smtpd_lostconn_data_cg + * l.P' after ' + * postfix_smtp_stage_cg + * (l.P' (' * l.digit^1 * l.P' bytes)') ^-1 + * l.P' from ' + * postfix_client_info_cg) + + (l.Cg((l.P(1) - l.P' from ')^1, 'postfix_action') + * l.P' from ' + * postfix_client_info_cg + * l.P': ' + * postfix_smtpd_lostconn_data_cg) +local postfix_smtpd_noqueue_cg = l.P'NOQUEUE: ' + * postfix_action_cg + * l.P': ' + * postfix_smtp_stage_cg + * l.P' from ' + * postfix_client_info_cg + * l.P': ' + * postfix_status_code_cg + * l.P' ' + * postfix_status_code_enhanced_cg + * (l.P' <' * l.Cg((l.P(1) - l.P'>')^1, 'postfix_status_data') * l.P'>:')^-1 + * l.P' ' + * (postfix_dnsbl_message_cg + (l.Cg((l.P(1)-l.P';')^0, 'postfix_status_message') * l.P';')) + * l.P' ' + * l.Cg(l.P(1)^1, 'postfix_keyvalue_data') +local postfix_smtpd_pipelining_cg = l.P'improper command pipelining after ' * postfix_smtp_stage_cg * l.P' from ' * postfix_client_info_cg * l.P':' +local postfix_smtpd_proxy_cg = l.P'proxy-' + * postfix_proxy_result_cg + * l.P': ' + * postfix_proxy_smtp_stage_cg + * l.P': ' + * postfix_proxy_message_cg -- FIXME this should fill postfix_proxy_message + * l.P'; ' + * l.Cg(l.P(1)^1, 'postfix_keyvalue_data') + +-- cleanup patterns +local postfix_cleanup_milter_cg = postfix_queueid_cg + * l.P': ' + * l.P'milter-' + * l.Cg(postfix_action, 'postfix_milter_result') + * l.P': ' + * l.Cg((l.P(1)-l.P';')^1, 'postfix_milter_message') + * l.P'; ' + * l.Cg((l.P(1)-l.P':')^1, 'postfix_keyvalue_data') + *(l.P': ' * l.Cg(l.P(1)^1, 'postfix_milter_data'))^-1 + +-- qmgr patterns +local postfix_qmgr_removed_cg = postfix_queueid_cg * l.P': removed' +local postfix_qmgr_active_cg = postfix_queueid_cg + * l.P': ' + * l.Cg((l.P(1) - l.P' (')^1, 'postfix_keyvalue_data') + * l.P' (queue active)' + +-- pipe patterns +local postfix_pipe_delivered_cg = postfix_queueid_cg + * l.P': ' + * l.Cg((l.P(1) - l.P' (')^1, 'postfix_keyvalue_data') + * l.P' (delivered via ' + * l.Cg(l.alnum^1, 'postfix_pipe_service') +local postfix_pipe_forward_cg = postfix_queueid_cg + * l.P': ' + * l.Cg((l.P(1) - l.P' (')^1, 'postfix_keyvalue_data') + * l.P' (mail forwarding loop for ' + * l.Cg((l.P(1) - l.P')')^1, 'postfix_to') + +-- postscreen patterns +local postfix_ps_connect_cg = l.P'CONNECT from ' + * postfix_client_info_cg + * l.P' to [' + * l.Cg(ipv46, 'postfix_server_ip') + * l.P']:' + * l.Cg(number, 'postfix_server_port') +local postfix_ps_access_cg = l.Cg(postfix_ps_access_action, 'postfix_postscreen_access') + * l.P' ' + * postfix_client_info_cg +local postfix_ps_noqueue_cg =postfix_smtpd_noqueue_cg +local postfix_ps_toobusy_cg = l.P'NOQUEUE: reject: CONNECT from ' + * postfix_client_info_cg + * l.P': ' + * l.Cg(l.P(1)^1, 'postfix_postscreen_toobusy_data') +local postfix_ps_dnsbl_cg = postfix_ps_violation_cg + * l.P' rank ' + * l.Cg(integer, 'postfix_postscreen_dnsbl_rank') + * l.P' for ' + * postfix_client_info_cg +local postfix_ps_cache_cg = l.P'cache ' + * (l.P(1)-l.P' ')^1 + * l.P' full cleanup: retained=' + * l.Cg(integer, 'postfix_postscreen_cache_retained') + * l.P' dropped=' + * l.Cg(integer, 'postfix_postscreen_cache_dropped') + * l.P' entries' +local postfix_ps_violations_cg = postfix_ps_violation_cg + * (l.P' ' * l.digit^1)^-1 + * (l.P' after ' * l.Cg(number, 'postfix_postscreen_violation_time'))^-1 + * l.P' from ' + * postfix_client_info_cg + * (l.P' after ' * postfix_smtp_stage_cg)^-1 + +-- dnsblog patterns +local postfix_dnsblog_listing_cg = l.P'addr ' + * l.Cg(ipv46, 'postfix_client_ip') + * l.P' listed by domain ' + * l.Cg(host, 'postfix_dnsbl_domain') + * l.P' as ' + * l.Cg(ipv46, 'postfix_dnsbl_result') + +-- tlsproxy patterns +local postfix_tlsproxy_conn_cg = (l.P'DISCONNECT' + + l.P'CONNECT') + * (l.P' from')^-1 + * l.P' ' + * postfix_client_info_cg + +-- anvil patterns +local postfix_service_cg = (l.Cg((l.P(1)-l.S':')^1, 'postfix_service') + * l.P':' + * l.Cg(ipv46, 'postfix_client_ip')) + + (l.Cg((l.P(1)-l.S':')^1 * l.P':' * l.digit^1, 'postfix_service') + * l.P':' + * l.Cg(ipv46, 'postfix_client_ip')) + +local postfix_anvil_conn_rate_cg = l.P'statistics: max connection rate ' + * l.Cg(integer, 'postfix_anvil_conn_rate') + * l.P'/' + * l.Cg(postfix_time_unit, 'postfix_anvil_conn_period') + * l.P' for (' + * postfix_service_cg + * l.P') at ' + * l.Cg(l.P(1)^1, 'postfix_anvil_timestamp') +local postfix_anvil_conn_cache_cg = l.P'statistics: max cache size ' + * l.Cg(integer, 'postfix_anvil_cache_size') + * l.P' at ' + * l.Cg(l.P(1)^1, 'postfix_anvil_timestamp') +local postfix_anvil_conn_count_cg = l.P'statistics: max connection count ' + * l.Cg(integer, 'postfix_anvil_conn_count') + * l.P' for (' + * postfix_service_cg + * l.P') at ' + * l.Cg(l.P(1)^1, 'postfix_anvil_timestamp') + +-- smtp patterns +local postfix_smtp_delivery_cg = postfix_queueid_cg + * l.P': ' + * l.Cg((l.P(1) - l.P' status=')^1, 'postfix_keyvalue_data') + * l.P' status=' + * l.Cg(l.alpha^1, 'postfix_status') + * (l.P' (' * l.Cg((l.P(1) - (l.P')' * -1))^1, 'postfix_smtp_response') * l.P')')^-1 +local postfix_smtp_connerr_cg = l.P'connect to ' + * postfix_relay_info_cg + * l.P': ' + * (l.P'Connection timed out' + l.P'No route to host' + l.P'Connection refused') +local postfix_smtp_lostconn_cg = postfix_queueid_cg + * l.P': ' + * postfix_lostconn + * l.P' with ' + * postfix_relay_info_cg + +-- master patterns +local postfix_master_start_cg = (l.P'daemon started' + l.P'reload') + * l.P' -- version ' + * l.Cg((l.P(1)-l.P',')^1, 'postfix_version') + * l.P', configuration ' + * l.Cg(l.P(1)^1, 'postfix_config_path') +local postfix_master_exit_cg = l.P'terminating on signal ' + * l.Cg(integer, 'postfix_termination_signal') + +-- bounce patterns +local postfix_bounce_notification_cg = postfix_queueid_cg + * l.P': ' + * l.P'sender ' + * (l.P'non-delivery' + l.P'delivery status' + l.P'delay') + * l.P' notification: ' + * l.Cg(postfix_queueid, 'postfix_bounce_queueid') + +-- scache patterns +local postfix_scache_lookups_cg = l.P'statistics: ' + * (l.P'address' + l.P'domain') + * l.P' lookup hits=' + * l.Cg(integer, 'postfix_scache_hits') + * l.P' miss=' + * l.Cg(integer, 'postfix_scache_miss') + * l.P' success=' + * l.Cg(integer, 'postfix_scache_success') +local postfix_scache_simultaneous_cg = l.P'statistics: max simultaneous domains=' + * l.Cg(integer, 'postfix_scache_domains') + * l.P' addresses=' + * l.Cg(integer, 'postfix_scache_addresses') + * l.P' connection=' + * l.Cg(integer, 'postfix_scache_connection') +local postfix_scache_timestamp_cg = l.P'statistics: start interval ' + * l.Cg(l.P(1)^1, 'postfix_scache_timestamp') + +-- aggregate all patterns +local postfix_patterns = { + smtpd = l.Ct(postfix_smtpd_connect_cg + + postfix_smtpd_disconnect_cg + + postfix_smtpd_lostconn_cg + + postfix_smtpd_noqueue_cg + + postfix_smtpd_pipelining_cg + + postfix_tlsconn_cg + + postfix_warning_cg + + postfix_smtpd_proxy_cg + + postfix_keyvalue_greedy_cg), + cleanup = l.Ct(postfix_cleanup_milter_cg + + postfix_warning_cg + + postfix_keyvalue_greedy_cg), + qmgr = l.Ct(postfix_qmgr_removed_cg + + postfix_qmgr_active_cg + + postfix_warning_cg), + pipe = l.Ct(postfix_pipe_delivered_cg + + postfix_pipe_forward_cg), + postscreen = l.Ct(postfix_ps_connect_cg + + postfix_ps_access_cg + + postfix_ps_noqueue_cg + + postfix_ps_toobusy_cg + + postfix_ps_cache_cg + + postfix_ps_dnsbl_cg + + postfix_ps_violations_cg + + postfix_warning_cg), + dnsblog = l.Ct(postfix_dnsblog_listing_cg), + anvil = l.Ct(postfix_anvil_conn_rate_cg + + postfix_anvil_conn_cache_cg + + postfix_anvil_conn_count_cg), + smtp = l.Ct(postfix_smtp_delivery_cg + + postfix_smtp_connerr_cg + + postfix_smtp_lostconn_cg + + postfix_tlsconn_cg + + postfix_warning_cg), + discard = l.Ct(postfix_queueid_cg + * l.P': ' + * l.Cg((l.P(1) - l.P' status=')^1, 'postfix_keyvalue_data') + * l.P' status=' + * l.Cg(l.alnum^1, 'postfix_status')), + -- lmtp = smtp, + pickup = l.Ct(postfix_queueid_cg + * l.P': uid=' + * l.Cg((l.P(1) - l.P' from=')^1, 'postfix_uid') + * l.P' from=<' + * l.Cg((l.P(1) - l.P'>')^1, 'postfix_from') + * (l.P'> orig_id=' + * l.Cg(l.P(1)^1, 'postfix_orig_id'))^-1 + ), + tlsproxy = l.Ct(postfix_tlsproxy_conn_cg), + master = l.Ct(postfix_master_start_cg + + postfix_master_exit_cg), + bounce = l.Ct(postfix_bounce_notification_cg), + sendmail = l.Ct(postfix_warning_cg), + postdrop = l.Ct(postfix_warning_cg), + scache = l.Ct(postfix_scache_lookups_cg + + postfix_scache_simultaneous_cg + + postfix_scache_timestamp_cg), + trivial_rewrite = l.Ct(postfix_warning_cg), + tlsmgr = l.Ct(postfix_warning_cg), + ['local'] = l.Ct(postfix_keyvalue_greedy_cg), +} +postfix_patterns['lmtp'] = postfix_patterns['smtp'] + +--local name = l.C(l.alpha^1) * space +--local value = l.C((l.P(1) - l.P',')^1) +--local sep = l.S(',;') * space +--local pair = l.Cg(name * '=' * space * value) * sep^-1 +--local queueid = lpeg.Cf(lpeg.Ct'' * l.Cg(lpeg.Cc'queueid' * l.C(l.xdigit ^ 1)) * l.P': ', rawset) +--local postfix_grammar = l.Cf(queueid * pair^0, rawset) + + + +local postfix_programname = l.P'postfix' + * (l.P(1) - l.P'/')^0 + * '/' + * l.Cg(l.P(1)^1) + +local integer_fields = { + 'postfix_anvil_cache_size', + 'postfix_anvil_conn_count', + 'postfix_anvil_conn_rate', + 'postfix_client_port', + 'postfix_nrcpt', + 'postfix_postscreen_cache_dropped', + 'postfix_postscreen_cache_retained', + 'postfix_postscreen_dnsbl_rank', + 'postfix_relay_port', + 'postfix_server_port', + 'postfix_size', + 'postfix_status_code', + 'postfix_termination_signal', + 'postfix_uid', +} +local float_fields = { + 'postfix_delay', + --'postfix_delay_before_qmgr', + --'postfix_delay_conn_setup', + --'postfix_delay_in_qmgr', + --'postfix_delay_transmission', + --'postfix_postscreen_violation_time', +} + + +function postfix_match(programname, message, extract_keyvalue_data) + local daemon_name = postfix_programname:match(programname) + local ret = nil + if not daemon_name then + return nil + end + if postfix_patterns[daemon_name] then + ret = postfix_patterns[daemon_name]:match(message) + -- below are fake patterns used by tests + elseif daemon_name == 'relay_info' then + ret = postfix_relay_info_ct:match(message) + elseif daemon_name == 'delays' then + ret = postfix_delays_ct:match(message) + elseif daemon_name == 'keyvalue_data' then + ret = kv_grammar:match(message) + end + -- extract postfix_keyvalue_data + if extract_keyvalue_data and ret and ret.postfix_keyvalue_data then + local kv = kv_grammar:match(ret.postfix_keyvalue_data) + if kv then + ret.postfix_keyvalue_data = nil + for k,v in pairs(kv) do + ret[string.format('postfix_%s', k)] = v + end + if ret.postfix_client then + local cl = postfix_client_info_ct:match(ret.postfix_client) + if cl then + ret.postfix_client = nil + for k2,v2 in pairs(cl) do + ret[k2] = v2 + end + end + end + if ret.postfix_relay then + local relay = postfix_relay_info_ct:match(ret.postfix_relay) + if relay then + ret.postfix_relay = nil + for k2,v2 in pairs(relay) do + ret[k2] = v2 + end + end + end + if ret.postfix_delays then + local delays = postfix_delays_ct:match(ret.postfix_delays) + if delays then + ret.postfix_delays = nil + for k2,v2 in pairs(delays) do + ret[k2] = v2 + end + end + end + end + end + if ret then + for i, v in ipairs(integer_fields) do + if ret[v] then + local value = ret[v] + if type(value) == 'table' then + value = value.value + end + ret[v] = { + value = tonumber(value), + value_type = 2, + } + end + end + for i, v in ipairs(float_fields) do + if ret[v] then + local value = ret[v] + if type(value) == 'table' then + value = value.value + end + ret[v] = tonumber(value) + end + end + end + return ret +end + +return M diff --git a/src/test/lua/lpeg_postfix.lua b/src/test/lua/lpeg_postfix.lua new file mode 100644 index 0000000..08ced5a --- /dev/null +++ b/src/test/lua/lpeg_postfix.lua @@ -0,0 +1,1100 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require 'string' +require 'table' +local pf = require 'postfix' + +-- From https://github.com/whyscream/postfix-grok-patterns +-- 2015-05-26 8fdd562d159845b97bf8d9c273e5357f3a143a94 +-- Original under BSD-3-clause license +local tests = { + ["anvil_0001.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max cache size 1 at Oct 26 18:13:50", + { + postfix_anvil_cache_size = 1, + postfix_anvil_timestamp = "Oct 26 18:13:50" + } + }, + ["anvil_0002.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max connection count 1 for (smtpd:61.238.241.86) at Oct 26 18:13:50", + { + postfix_anvil_conn_count = 1, + postfix_anvil_timestamp = "Oct 26 18:13:50", + postfix_client_ip = "61.238.241.86", + postfix_service = "smtpd" + } + }, + ["anvil_0003.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max connection rate 1/60s for (smtpd:61.238.241.86) at Oct 26 18:13:50", + { + postfix_anvil_conn_period = "60s", + postfix_anvil_conn_rate = 1, + postfix_anvil_timestamp = "Oct 26 18:13:50", + postfix_client_ip = "61.238.241.86", + postfix_service = "smtpd" + } + }, + ["anvil_0004.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max connection count 1 for (smtpd:2604:8d00:0:1::3) at Oct 26 17:46:59", + { + postfix_anvil_conn_count = 1, + postfix_anvil_timestamp = "Oct 26 17:46:59", + postfix_client_ip = "2604:8d00:0:1::3", + postfix_service = "smtpd" + } + }, + ["anvil_0005.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max connection rate 1/60s for (smtpd:2604:8d00:0:1::3) at Oct 26 17:46:59", + { + postfix_anvil_conn_period = "60s", + postfix_anvil_conn_rate = 1, + postfix_anvil_timestamp = "Oct 26 17:46:59", + postfix_client_ip = "2604:8d00:0:1::3", + postfix_service = "smtpd" + } + }, + ["anvil_0006.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max connection count 1 for (127.0.0.1:2525:127.0.0.1) at Oct 26 19:46:00", + { + postfix_anvil_conn_count = 1, + postfix_anvil_timestamp = "Oct 26 19:46:00", + postfix_client_ip = "127.0.0.1", + postfix_service = "127.0.0.1:2525" + } + }, + ["anvil_0007.yaml"] = { + "POSTFIX_ANVIL", + "statistics: max connection rate 1/60s for (127.0.0.1:2525:127.0.0.1) at Oct 26 18:13:50", + { + postfix_anvil_conn_period = "60s", + postfix_anvil_conn_rate = 1, + postfix_anvil_timestamp = "Oct 26 18:13:50", + postfix_client_ip = "127.0.0.1", + postfix_service = "127.0.0.1:2525" + } + }, + ["bounce_0001.yaml"] = { + "POSTFIX_BOUNCE", + "17CCA8044: sender non-delivery notification: ADD2F8052", + { + postfix_bounce_queueid = "ADD2F8052", + postfix_queueid = "17CCA8044" + } + }, + ["bounce_0002.yaml"] = { + "POSTFIX_BOUNCE", + "65AFB8B5: sender delivery status notification: 1DF35917", + { + postfix_bounce_queueid = "1DF35917", + postfix_queueid = "65AFB8B5" + } + }, + ["bounce_0003.yaml"] = { + "POSTFIX_BOUNCE", + "264FE1A18: sender delay notification: 0A87A1A08", + { + postfix_bounce_queueid = "0A87A1A08", + postfix_queueid = "264FE1A18" + } + }, + ["cleanup_0001.yaml"] = { + "POSTFIX_CLEANUP", + "0F5383D: message-id=<544D2523.30609@rhsoft.net>", + { + postfix_keyvalue_data = "message-id=<544D2523.30609@rhsoft.net>", + postfix_queueid = "0F5383D" + } + }, + ["cleanup_0002.yaml"] = { + "POSTFIX_CLEANUP", + "4104E3D: message-id=<>", + { + postfix_keyvalue_data = "message-id=<>", + postfix_queueid = "4104E3D" + } + }, + ["cleanup_0003.yaml"] = { + "POSTFIX_CLEANUP", + "4104E3D: milter-header-redirect: header X-Spam-Status: Yes, score=39.3 required=5.0 tests=ADVANCE_FE from 061238241086.static.ctinets.com[61.238.241.86]; from= to= proto=ESMTP helo=: tom@example.com", + { + postfix_keyvalue_data = "from= to= proto=ESMTP helo=", + postfix_milter_data = "tom@example.com", + postfix_milter_message = "header X-Spam-Status: Yes, score=39.3 required=5.0 tests=ADVANCE_FE from 061238241086.static.ctinets.com[61.238.241.86]", + postfix_milter_result = "header-redirect", + postfix_queueid = "4104E3D" + } + }, + ["cleanup_0004.yaml"] = { + "POSTFIX_CLEANUP", + "4104E3D: milter-reject: END-OF-MESSAGE from 061238241086.static.ctinets.com[61.238.241.86]: 5.7.1 Blocked by SpamAssassin; from= to= proto=ESMTP helo=", + { + postfix_keyvalue_data = "from= to= proto=ESMTP helo=", + postfix_milter_message = "END-OF-MESSAGE from 061238241086.static.ctinets.com[61.238.241.86]: 5.7.1 Blocked by SpamAssassin", + postfix_milter_result = "reject", + postfix_queueid = "4104E3D" + } + }, + ["cleanup_0005.yaml"] = { + "POSTFIX_CLEANUP", + "warning: dict_ldap_get_values[1]: DN uid=mguiraud,ou=people,dc=neotion,dc=com not found, skipping ", + { + postfix_warning = "dict_ldap_get_values[1]: DN uid=mguiraud,ou=people,dc=neotion,dc=com not found, skipping " + } + }, + ["delays_0001.yaml"] = { + "POSTFIX_DELAYS", + "0.2/0.02/0.04/3.3", + { + postfix_delay_before_qmgr = 0.20000000000000001, + postfix_delay_conn_setup = 0.040000000000000001, + postfix_delay_in_qmgr = 0.02, + postfix_delay_transmission = 3.2999999999999998 + } + }, + ["discard_0001.yaml"] = { + "POSTFIX_DISCARD", + "25F2E5E061: to=, relay=none, delay=0.05, delays=0.05/0/0/0, dsn=2.0.0, status=sent (test.example.com)", + { + postfix_keyvalue_data = "to=, relay=none, delay=0.05, delays=0.05/0/0/0, dsn=2.0.0,", + postfix_queueid = "25F2E5E061", + postfix_status = "sent" + } + }, + ["dnsblog_0001.yaml"] = { + "POSTFIX_DNSBLOG", + "addr 61.238.241.86 listed by domain psbl.surriel.com as 127.0.0.2", + { + postfix_client_ip = "61.238.241.86", + postfix_dnsbl_domain = "psbl.surriel.com", + postfix_dnsbl_result = "127.0.0.2" + } + }, + ["dnsblog_0002.yaml"] = { + "POSTFIX_DNSBLOG", + "addr 2607:f8b0:4003:c01::23a listed by domain list.dnswl.org as 127.0.5.1", + { + postfix_client_ip = "2607:f8b0:4003:c01::23a", + postfix_dnsbl_domain = "list.dnswl.org", + postfix_dnsbl_result = "127.0.5.1" + } + }, + ["local_0001.yaml"] = { + "POSTFIX_LOCAL", + "2A22C263F6: to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0, status=sent (delivered to command: procmail -a \"$EXTENSION\")", + { + postfix_keyvalue_data = "to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0, status=sent (delivered to command: procmail -a \"$EXTENSION\")", + postfix_queueid = "2A22C263F6" + } + }, + ["local_0002.yaml"] = { + "POSTFIX_LOCAL", + "892A0205B6: to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1, status=bounced (unknown user: \"ghdsgfhdslfh\")", + { + postfix_keyvalue_data = "to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1, status=bounced (unknown user: \"ghdsgfhdslfh\")", + postfix_queueid = "892A0205B6" + } + }, + ["master_0001.yaml"] = { + "POSTFIX_MASTER", + "daemon started -- version 2.11.0, configuration /etc/postfix", + { + postfix_config_path = "/etc/postfix", + postfix_version = "2.11.0" + } + }, + ["master_0002.yaml"] = { + "POSTFIX_MASTER", + "terminating on signal 15", + { + postfix_termination_signal = 15 + } + }, + ["master_0003.yaml"] = { + "POSTFIX_MASTER", + "reload -- version 2.11.0, configuration /etc/postfix", + { + postfix_config_path = "/etc/postfix", + postfix_version = "2.11.0" + } + }, + ["pickup_0001.yaml"] = { + "POSTFIX_PICKUP", + "D2FAE20586: uid=0 from=", + { + postfix_queueid = "D2FAE20586", + -- from postfix_keyvalue_data = "uid=0 from=", + postfix_uid = 0, + postfix_from = 'fail2ban' + } + }, + ["pipe_0001.yaml"] = { + "POSTFIX_PIPE", + "0F5383D: to=, relay=dovecot, delay=4.3, delays=4.1/0.01/0/0.15, dsn=2.0.0, status=sent (delivered via dovecot service)", + { + postfix_keyvalue_data = "to=, relay=dovecot, delay=4.3, delays=4.1/0.01/0/0.15, dsn=2.0.0, status=sent", + postfix_pipe_service = "dovecot", + postfix_queueid = "0F5383D" + } + }, + ["pipe_0002.yaml"] = { + "POSTFIX_PIPE", + "153053D: to=, orig_to=, relay=dovecot, delay=3.4, delays=3.3/0.03/0/0.12, dsn=2.0.0, status=sent (delivered via dovecot service)", + { + postfix_keyvalue_data = "to=, orig_to=, relay=dovecot, delay=3.4, delays=3.3/0.03/0/0.12, dsn=2.0.0, status=sent", + postfix_pipe_service = "dovecot", + postfix_queueid = "153053D" + } + }, + ["pipe_0003.yaml"] = { + "POSTFIX_PIPE", + "95ECE24E0: to=, relay=dovecot, delay=0.12, delays=0.03/0/0/0.08, dsn=5.4.6, status=bounced (mail forwarding loop for tom@example.com)", + { + postfix_keyvalue_data = "to=, relay=dovecot, delay=0.12, delays=0.03/0/0/0.08, dsn=5.4.6, status=bounced", + postfix_queueid = "95ECE24E0", + postfix_to = "tom@example.com" + } + }, + ["postdrop_0001.yaml"] = { + "POSTFIX_POSTDROP", + "warning: uid=0: File too large", + { + postfix_warning = "uid=0: File too large" + } + }, + ["postscreen_0001.yaml"] = { + "POSTFIX_POSTSCREEN", + "cache btree:/var/lib/postfix-in/postscreen_cache full cleanup: retained=242 dropped=7 entries", + { + postfix_postscreen_cache_dropped = 7, + postfix_postscreen_cache_retained = 242 + } + }, + ["postscreen_0002.yaml"] = { + "POSTFIX_POSTSCREEN", + "CONNECT from [61.238.241.86]:53024 to [89.105.204.244]:25", + { + postfix_client_ip = "61.238.241.86", + postfix_client_port = 53024, + postfix_server_ip = "89.105.204.244", + postfix_server_port = 25 + } + }, + ["postscreen_0003.yaml"] = { + "POSTFIX_POSTSCREEN", + "DISCONNECT [216.81.72.72]:43421", + { + postfix_client_ip = "216.81.72.72", + postfix_client_port = 43421 + } + }, + ["postscreen_0004.yaml"] = { + "POSTFIX_POSTSCREEN", + "PASS OLD [61.238.241.86]:53024", + { + postfix_client_ip = "61.238.241.86", + postfix_client_port = 53024, + postfix_postscreen_access = "PASS OLD" + } + }, + ["postscreen_0005.yaml"] = { + "POSTFIX_POSTSCREEN", + "PASS OLD [2604:8d00:0:1::3]:52237", + { + postfix_client_ip = "2604:8d00:0:1::3", + postfix_client_port = 52237, + postfix_postscreen_access = "PASS OLD" + } + }, + ["postscreen_0006.yaml"] = { + "POSTFIX_POSTSCREEN", + "PASS NEW [2607:f8b0:4003:c01::23a]:36051", + { + postfix_client_ip = "2607:f8b0:4003:c01::23a", + postfix_client_port = 36051, + postfix_postscreen_access = "PASS NEW" + } + }, + ["postscreen_0007.yaml"] = { + "POSTFIX_POSTSCREEN", + "WHITELISTED [61.238.241.86]:53024", + { + postfix_client_ip = "61.238.241.86", + postfix_client_port = 53024, + postfix_postscreen_access = "WHITELISTED" + } + }, + ["postscreen_0008.yaml"] = { + "POSTFIX_POSTSCREEN", + "BLACKLISTED [61.238.241.86]:53024", + { + postfix_client_ip = "61.238.241.86", + postfix_client_port = 53024, + postfix_postscreen_access = "BLACKLISTED" + } + }, + ["postscreen_0009.yaml"] = { + "POSTFIX_POSTSCREEN", + "WHITELIST VETO [61.238.241.86]:53024", + { + postfix_client_ip = "61.238.241.86", + postfix_client_port = 53024, + postfix_postscreen_access = "WHITELIST VETO" + } + }, + ["postscreen_0010.yaml"] = { + "POSTFIX_POSTSCREEN", + "PREGREET 6 after 1.2 from [216.81.72.72]:43421: RSET\r\n", + { + postfix_client_ip = "216.81.72.72", + postfix_client_port = 43421, + postfix_postscreen_violation = "PREGREET", + postfix_postscreen_violation_time = 1.2 + } + }, + ["postscreen_0011.yaml"] = { + "POSTFIX_POSTSCREEN", + "DNSBL rank 3 for [111.73.45.149]:1038", + { + postfix_client_ip = "111.73.45.149", + postfix_client_port = 1038, + postfix_postscreen_dnsbl_rank = 3, + postfix_postscreen_violation = "DNSBL" + } + }, + ["postscreen_0012.yaml"] = { + "POSTFIX_POSTSCREEN", + "HANGUP after 63 from [111.73.45.149]:1038 in tests after SMTP handshake", + { + postfix_client_ip = "111.73.45.149", + postfix_client_port = 1038, + postfix_postscreen_violation = "HANGUP", + postfix_postscreen_violation_time = 63 + } + }, + ["postscreen_0013.yaml"] = { + "POSTFIX_POSTSCREEN", + "HANGUP after 0 from [66.55.85.58]:54017 in tests before SMTP handshake", + { + postfix_client_ip = "66.55.85.58", + postfix_client_port = 54017, + postfix_postscreen_violation = "HANGUP", + postfix_postscreen_violation_time = 0 + } + }, + ["postscreen_0014.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND TIME LIMIT from [182.246.250.243]:51799 after CONNECT", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51799, + postfix_postscreen_violation = "COMMAND TIME LIMIT", + postfix_smtp_stage = "CONNECT" + } + }, + ["postscreen_0015.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND TIME LIMIT from [182.246.250.243]:51799 after HELO", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51799, + postfix_postscreen_violation = "COMMAND TIME LIMIT", + postfix_smtp_stage = "HELO" + } + }, + ["postscreen_0016.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND TIME LIMIT from [182.246.250.243]:51799 after EHLO", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51799, + postfix_postscreen_violation = "COMMAND TIME LIMIT", + postfix_smtp_stage = "EHLO" + } + }, + ["postscreen_0017.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND TIME LIMIT from [182.246.250.243]:51799 after AUTH", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51799, + postfix_postscreen_violation = "COMMAND TIME LIMIT", + postfix_smtp_stage = "AUTH" + } + }, + ["postscreen_0018.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND TIME LIMIT from [182.246.250.243]:51799 after MAIL", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51799, + postfix_postscreen_violation = "COMMAND TIME LIMIT", + postfix_smtp_stage = "MAIL" + } + }, + ["postscreen_0019.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND TIME LIMIT from [182.246.250.243]:51796 after RCPT", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51796, + postfix_postscreen_violation = "COMMAND TIME LIMIT", + postfix_smtp_stage = "RCPT" + } + }, + ["postscreen_0020.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND COUNT LIMIT from [182.246.250.243]:51796 after RCPT", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51796, + postfix_postscreen_violation = "COMMAND COUNT LIMIT", + postfix_smtp_stage = "RCPT" + } + }, + ["postscreen_0021.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND LENGTH LIMIT from [182.246.250.243]:51796 after RCPT", + { + postfix_client_ip = "182.246.250.243", + postfix_client_port = 51796, + postfix_postscreen_violation = "COMMAND LENGTH LIMIT", + postfix_smtp_stage = "RCPT" + } + }, + ["postscreen_0022.yaml"] = { + "POSTFIX_POSTSCREEN", + "NON-SMTP COMMAND from [93.174.93.51]:45284 after CONNECT: GET http://ipv4scan.com/hello/check.txt HTTP/1.1", + { + postfix_client_ip = "93.174.93.51", + postfix_client_port = 45284, + postfix_postscreen_violation = "NON-SMTP COMMAND", + postfix_smtp_stage = "CONNECT" + } + }, + ["postscreen_0023.yaml"] = { + "POSTFIX_POSTSCREEN", + "NOQUEUE: reject: CONNECT from [1.2.3.4]:1337: too many connections", + { + postfix_client_ip = "1.2.3.4", + postfix_client_port = 1337, + postfix_postscreen_toobusy_data = "too many connections" + } + }, + ["postscreen_0024.yaml"] = { + "POSTFIX_POSTSCREEN", + "NOQUEUE: reject: CONNECT from [5.6.7.8]:1337: all server ports busy", + { + postfix_client_ip = "5.6.7.8", + postfix_client_port = 1337, + postfix_postscreen_toobusy_data = "all server ports busy" + } + }, + ["postscreen_0025.yaml"] = { + "POSTFIX_POSTSCREEN", + "COMMAND PIPELINING from [88.208.233.4]:42606 after .: \n", + { + postfix_client_ip = "88.208.233.4", + postfix_client_port = 42606, + postfix_postscreen_violation = "COMMAND PIPELINING", + postfix_smtp_stage = "." + } + }, + ["postscreen_0026.yaml"] = { + "POSTFIX_POSTSCREEN", + "BARE NEWLINE from [88.208.233.4]:42606 after .", + { + postfix_client_ip = "88.208.233.4", + postfix_client_port = 42606, + postfix_postscreen_violation = "BARE NEWLINE", + postfix_smtp_stage = "." + } + }, + ["postscreen_0027.yaml"] = { + "POSTFIX_POSTSCREEN", + "NOQUEUE: reject: RCPT from [182.98.255.184]:2413: 550 5.5.1 Protocol error; from=, to=, proto=SMTP, helo=", + { + postfix_action = "reject", + postfix_client_ip = "182.98.255.184", + postfix_client_port = 2413, + postfix_keyvalue_data = "from=, to=, proto=SMTP, helo=", + postfix_smtp_stage = "RCPT", + postfix_status_code = 550, + postfix_status_code_enhanced = "5.5.1", + postfix_status_message = "Protocol error" + } + }, + ["postscreen_0028.yaml"] = { + "POSTFIX_POSTSCREEN", + "NOQUEUE: reject: RCPT from [27.157.200.233]:4984: 550 5.7.1 Service unavailable; client [27.157.200.233] blocked using zen.spamhaus.org; from=, to=<4ECEA00F.9040306@example.com>, proto=ESMTP, helo=", + { + postfix_action = "reject", + postfix_client_ip = "27.157.200.233", + postfix_client_port = 4984, + postfix_keyvalue_data = "from=, to=<4ECEA00F.9040306@example.com>, proto=ESMTP, helo=", + postfix_smtp_stage = "RCPT", + postfix_status_code = 550, + postfix_status_code_enhanced = "5.7.1", + postfix_status_data = "27.157.200.233", + postfix_status_message = "blocked using zen.spamhaus.org" + } + }, + ["postscreen_0029.yaml"] = { + "POSTFIX_POSTSCREEN", + "warning: getpeername: Transport endpoint is not connected -- dropping this connection", + { + postfix_warning = "getpeername: Transport endpoint is not connected -- dropping this connection" + } + }, + ["qmgr_0001.yaml"] = { + "POSTFIX_QMGR", + "0F5383D: removed", + { + postfix_queueid = "0F5383D" + } + }, + ["qmgr_0002.yaml"] = { + "POSTFIX_QMGR", + "0F5383D: from=, size=5360, nrcpt=1 (queue active)", + { + postfix_keyvalue_data = "from=, size=5360, nrcpt=1", + postfix_queueid = "0F5383D" + } + }, + ["qmgr_0003.yaml"] = { + "POSTFIX_QMGR", + "warning: bounce_queue_lifetime is larger than maximal_queue_lifetime - adjusting bounce_queue_lifetime", + { + postfix_warning = "bounce_queue_lifetime is larger than maximal_queue_lifetime - adjusting bounce_queue_lifetime" + } + }, + ["relay_info_0001.yaml"] = { + "POSTFIX_RELAY_INFO", + "1.example.com.si[private/dovecot-lmtp]", + { + postfix_relay_hostname = "1.example.com.si", + postfix_relay_service = "private/dovecot-lmtp" + } + }, + ["scache_0001.yaml"] = { + "POSTFIX_SCACHE", + "statistics: domain lookup hits=0 miss=1 success=0%", + { + postfix_scache_hits = 0, + postfix_scache_miss = 1, + postfix_scache_success = 0 + } + }, + ["scache_0002.yaml"] = { + "POSTFIX_SCACHE", + "statistics: start interval Dec 6 21:20:35", + { + postfix_scache_timestamp = "Dec 6 21:20:35" + } + }, + ["scache_0003.yaml"] = { + "POSTFIX_SCACHE", + "statistics: address lookup hits=0 miss=1 success=0%", + { + postfix_scache_hits = 0, + postfix_scache_miss = 1, + postfix_scache_success = 0 + } + }, + ["scache_0004.yaml"] = { + "POSTFIX_SCACHE", + "statistics: max simultaneous domains=1 addresses=1 connection=1", + { + postfix_scache_addresses = 1, + postfix_scache_connection = 1, + postfix_scache_domains = 1 + } + }, + ["sendmail_0001.yaml"] = { + "POSTFIX_SENDMAIL", + "fatal: root(0): message file too big", + { + postfix_warning = "root(0): message file too big" + } + }, + ["smtp_0001.yaml"] = { + "POSTFIX_SMTP", + "7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)", + { + postfix_keyvalue_data = "to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0,", + postfix_queueid = "7EE668039", + postfix_smtp_response = "250 2.0.0 Ok: queued as 153053D", + postfix_status = "sent" + } + }, + ["smtp_0002.yaml"] = { + "POSTFIX_SMTP", + "0D9633D: to=, orig_to=, relay=mail1.example.net[10.74.103.11]:25, delay=11, delays=5.4/0.06/5.1/0.03, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as F12DF84004)", + { + postfix_keyvalue_data = "to=, orig_to=, relay=mail1.example.net[10.74.103.11]:25, delay=11, delays=5.4/0.06/5.1/0.03, dsn=2.0.0,", + postfix_queueid = "0D9633D", + postfix_smtp_response = "250 2.0.0 Ok: queued as F12DF84004", + postfix_status = "sent" + } + }, + ["smtp_0003.yaml"] = { + "POSTFIX_SMTP", + "Untrusted TLS connection established to mx4.hotmail.com[65.55.92.136]:25: TLSv1.2 with cipher ECDHE-RSA-AES256-SHA384 (256/256 bits)", + { + postfix_relay_hostname = "mx4.hotmail.com", + postfix_relay_ip = "65.55.92.136", + postfix_relay_port = 25, + postfix_tls_cipher = "ECDHE-RSA-AES256-SHA384", + postfix_tls_cipher_size = "256/256", + postfix_tls_version = "TLSv1.2" + } + }, + ["smtp_0004.yaml"] = { + "POSTFIX_SMTP", + "Untrusted TLS connection established to 127.0.0.1[127.0.0.1]:2525: TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)", + { + postfix_relay_hostname = "127.0.0.1", + postfix_relay_ip = "127.0.0.1", + postfix_relay_port = 2525, + postfix_tls_cipher = "AECDH-AES256-SHA", + postfix_tls_cipher_size = "256/256", + postfix_tls_version = "TLSv1.2" + } + }, + ["smtp_0005.yaml"] = { + "POSTFIX_SMTP", + "connect to intern.nl[194.109.6.98]:25: Connection timed out", + { + postfix_relay_hostname = "intern.nl", + postfix_relay_ip = "194.109.6.98", + postfix_relay_port = 25 + } + }, + ["smtp_0006.yaml"] = { + "POSTFIX_SMTP", + "B99FE3D: lost connection with mx3.hotmail.com[65.55.37.72] while receiving the initial server greeting", + { + postfix_queueid = "B99FE3D", + postfix_relay_hostname = "mx3.hotmail.com", + postfix_relay_ip = "65.55.37.72" + } + }, + ["smtp_0007.yaml"] = { + "POSTFIX_SMTP", + "connect to mail2.crabruzzo.it[2.115.207.21]:25: No route to host", + { + postfix_relay_hostname = "mail2.crabruzzo.it", + postfix_relay_ip = "2.115.207.21", + postfix_relay_port = 25 + } + }, + ["smtp_0015.yaml"] = { + "POSTFIX_SMTP", + "Trusted TLS connection established to gmail-smtp-in.l.google.com[74.125.136.26]:25: TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)", + { + postfix_relay_hostname = "gmail-smtp-in.l.google.com", + postfix_relay_ip = "74.125.136.26", + postfix_relay_port = 25, + postfix_tls_cipher = "ECDHE-RSA-AES128-GCM-SHA256", + postfix_tls_cipher_size = "128/128", + postfix_tls_version = "TLSv1.2" + } + }, + ["smtp_0016.yaml"] = { + "POSTFIX_SMTP", + "Verified TLS connection established to mail.sys4.de[2001:1578:400:111::7]:25: TLSv1 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)", + { + postfix_relay_hostname = "mail.sys4.de", + postfix_relay_ip = "2001:1578:400:111::7", + postfix_relay_port = 25, + postfix_tls_cipher = "ECDHE-RSA-AES256-SHA", + postfix_tls_cipher_size = "256/256", + postfix_tls_version = "TLSv1" + } + }, + ["smtp_0017.yaml"] = { + "POSTFIX_SMTP", + "AD6E41CDC05C: to=, relay=sub1.example.com[private/dovecot-lmtp], delay=0.15, delays=0.04/0/0.02/0.08, dsn=2.0.0, status=sent (250 2.0.0 qYyyBMsl9FRuZQAAA15QOA Saved)", + { + postfix_keyvalue_data = "to=, relay=sub1.example.com[private/dovecot-lmtp], delay=0.15, delays=0.04/0/0.02/0.08, dsn=2.0.0,", + postfix_queueid = "AD6E41CDC05C", + postfix_smtp_response = "250 2.0.0 qYyyBMsl9FRuZQAAA15QOA Saved", + postfix_status = "sent" + } + }, + ["smtp_0018.yaml"] = { + "POSTFIX_SMTP", + "4E74613FE76: to=, relay=gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b]:25, delay=1.9, delays=0.05/0/1.1/0.71, dsn=5.7.1, status=bounced (host gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b] said: 550-5.7.1 [2607:4100:3:25:100::2 12] Our system has detected that this 550-5.7.1 message is likely unsolicited mail. To reduce the amount of spam sent 550-5.7.1 to Gmail, this message has been blocked. Please visit 550-5.7.1 http://support.google.com/mail/bin/answer.py?hl=en&answer=188131 for 550 5.7.1 more information. qc12si8475009pab.211 - gsmtp (in reply to end of DATA command))", + { + postfix_keyvalue_data = "to=, relay=gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b]:25, delay=1.9, delays=0.05/0/1.1/0.71, dsn=5.7.1,", + postfix_queueid = "4E74613FE76", + postfix_smtp_response = "host gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b] said: 550-5.7.1 [2607:4100:3:25:100::2 12] Our system has detected that this 550-5.7.1 message is likely unsolicited mail. To reduce the amount of spam sent 550-5.7.1 to Gmail, this message has been blocked. Please visit 550-5.7.1 http://support.google.com/mail/bin/answer.py?hl=en&answer=188131 for 550 5.7.1 more information. qc12si8475009pab.211 - gsmtp (in reply to end of DATA command)", + postfix_status = "bounced" + } + }, + ["smtp_0019.yaml"] = { + "POSTFIX_SMTP", + "warning: problem talking to service private/scache: Connection timed out", + { + postfix_warning = "problem talking to service private/scache: Connection timed out" + } + }, + ["smtpd_0001.yaml"] = { + "POSTFIX_SMTPD", + "connect from 061238241086.static.ctinets.com[61.238.241.86]", + { + postfix_client_hostname = "061238241086.static.ctinets.com", + postfix_client_ip = "61.238.241.86" + } + }, + ["smtpd_0002.yaml"] = { + "POSTFIX_SMTPD", + "disconnect from 061238241086.static.ctinets.com[61.238.241.86]", + { + postfix_client_hostname = "061238241086.static.ctinets.com", + postfix_client_ip = "61.238.241.86" + } + }, + ["smtpd_0003.yaml"] = { + "POSTFIX_SMTPD", + "4104E3D: client=061238241086.static.ctinets.com[61.238.241.86]", + { + postfix_keyvalue_data = "client=061238241086.static.ctinets.com[61.238.241.86]", + postfix_queueid = "4104E3D" + } + }, + ["smtpd_0004.yaml"] = { + "POSTFIX_SMTPD", + "NOQUEUE: reject: RCPT from 061238241086.static.ctinets.com[61.238.241.86]: 550 5.1.1 : Recipient address rejected: User unknown in virtual mailbox table; from= to= proto=ESMTP helo=", + { + postfix_action = "reject", + postfix_client_hostname = "061238241086.static.ctinets.com", + postfix_client_ip = "61.238.241.86", + postfix_keyvalue_data = "from= to= proto=ESMTP helo=", + postfix_smtp_stage = "RCPT", + postfix_status_code = 550, + postfix_status_code_enhanced = "5.1.1", + postfix_status_data = "toernooicommissie@example.com", + postfix_status_message = "Recipient address rejected: User unknown in virtual mailbox table" + } + }, + ["smtpd_0005.yaml"] = { + "POSTFIX_SMTPD", + "lost connection after STARTTLS from unknown[72.13.58.7]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtp_stage = "STARTTLS", + postfix_smtpd_lostconn_data = "lost connection" + } + }, + ["smtpd_0006.yaml"] = { + "POSTFIX_SMTPD", + "warning: hostname exemple.com does not resolve to address 185.14.29.32", + { + postfix_warning = "hostname exemple.com does not resolve to address 185.14.29.32" + } + }, + ["smtpd_0007.yaml"] = { + "POSTFIX_SMTPD", + "timeout after CONNECT from unknown[177.227.18.3]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "177.227.18.3", + postfix_smtp_stage = "CONNECT", + postfix_smtpd_lostconn_data = "timeout" + } + }, + ["smtpd_0008.yaml"] = { + "POSTFIX_POSTSCREEN", + "NOQUEUE: reject: RCPT from unknown[2001:980:cfb1:1:82f:f74e:a45c:3033]: 504 5.5.2 : Sender address rejected: need fully-qualified address; from= to= proto=SMTP helo=", + { + postfix_action = "reject", + postfix_client_hostname = "unknown", + postfix_client_ip = "2001:980:cfb1:1:82f:f74e:a45c:3033", + postfix_keyvalue_data = "from= to= proto=SMTP helo=", + postfix_smtp_stage = "RCPT", + postfix_status_code = 504, + postfix_status_code_enhanced = "5.5.2", + postfix_status_data = "aap@henk", + postfix_status_message = "Sender address rejected: need fully-qualified address" + } + }, + ["smtpd_0009.yaml"] = { + "POSTFIX_SMTPD", + "NOQUEUE: reject: RCPT from news.zihan-promo.com[192.36.205.58]: 554 5.7.1 Service unavailable; Helo command [news.zihan-promo.com] blocked using dbl.spamhaus.org; http://www.spamhaus.org/query/dbl?domain=zihan-promo.com; from= to= proto=ESMTP helo=", + { + postfix_action = "reject", + postfix_client_hostname = "news.zihan-promo.com", + postfix_client_ip = "192.36.205.58", + postfix_keyvalue_data = "from= to= proto=ESMTP helo=", + postfix_smtp_stage = "RCPT", + postfix_status_code = 554, + postfix_status_code_enhanced = "5.7.1", + postfix_status_data = "news.zihan-promo.com", + postfix_status_message = "blocked using dbl.spamhaus.org; http://www.spamhaus.org/query/dbl?domain=zihan-promo.com" + } + }, + ["smtpd_0010.yaml"] = { + "POSTFIX_SMTPD", + "Anonymous TLS connection established from julie.example.com[10.163.89.202]: TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)", + { + postfix_client_hostname = "julie.example.com", + postfix_client_ip = "10.163.89.202", + postfix_tls_cipher = "AECDH-AES256-SHA", + postfix_tls_cipher_size = "256/256", + postfix_tls_version = "TLSv1.2" + } + }, + ["smtpd_0011.yaml"] = { + "POSTFIX_SMTPD", + "lost connection after DATA (7774180 bytes) from unknown[72.13.58.7]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtp_stage = "DATA", + postfix_smtpd_lostconn_data = "lost connection" + } + }, + ["smtpd_0012.yaml"] = { + "POSTFIX_SMTPD", + "lost connection after UNKNOWN from unknown[2001:456:cfb1:1:f5d7:dead:beef:cafe]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "2001:456:cfb1:1:f5d7:dead:beef:cafe", + postfix_smtp_stage = "UNKNOWN", + postfix_smtpd_lostconn_data = "lost connection" + } + }, + ["smtpd_0013.yaml"] = { + "POSTFIX_SMTPD", + "lost connection after RSET from unknown[72.13.58.7]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtp_stage = "RSET", + postfix_smtpd_lostconn_data = "lost connection" + } + }, + ["smtpd_0014.yaml"] = { + "POSTFIX_SMTPD", + "timeout after END-OF-MESSAGE from unknown[72.13.58.7]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtp_stage = "END-OF-MESSAGE", + postfix_smtpd_lostconn_data = "timeout" + } + }, + ["smtpd_0015.yaml"] = { + "POSTFIX_SMTPD", + "improper command pipelining after EHLO from unknown[2001:968:9999:20:415c:cd2:da8e:d0cf]: MAIL FROM:<>\nRCPT TO:\nDATA\nSubject: pipeline test\n\nThis is a test\n.\nQUIT\n\r\n", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "2001:968:9999:20:415c:cd2:da8e:d0cf", + postfix_smtp_stage = "EHLO" + } + }, + ["smtpd_0016.yaml"] = { + "POSTFIX_SMTPD", + "lost connection after DATA (0 bytes) from unknown[72.13.58.7]", + { + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtp_stage = "DATA", + postfix_smtpd_lostconn_data = "lost connection" + } + }, + ["smtpd_0017.yaml"] = { + "POSTFIX_SMTPD", + "NOQUEUE: reject: VRFY from unknown[2001:968:9999:20:88b:9b7d:2a54:2bd2]: 454 4.7.1 : Relay access denied; to= proto=SMTP helo=", + { + postfix_action = "reject", + postfix_client_hostname = "unknown", + postfix_client_ip = "2001:968:9999:20:88b:9b7d:2a54:2bd2", + postfix_keyvalue_data = "to= proto=SMTP helo=", + postfix_smtp_stage = "VRFY", + postfix_status_code = 454, + postfix_status_code_enhanced = "4.7.1", + postfix_status_data = "aap@henk.com", + postfix_status_message = "Relay access denied" + } + }, + ["smtpd_0018.yaml"] = { + "POSTFIX_SMTPD", + "NOQUEUE: reject: VRFY from unknown[2001:968:9999:20:88b:9b7d:2a54:2bd2]: 550 5.1.1 : Recipient address rejected: User unknown in virtual mailbox table; to= proto=SMTP helo=", + { + postfix_action = "reject", + postfix_client_hostname = "unknown", + postfix_client_ip = "2001:968:9999:20:88b:9b7d:2a54:2bd2", + postfix_keyvalue_data = "to= proto=SMTP helo=", + postfix_smtp_stage = "VRFY", + postfix_status_code = 550, + postfix_status_code_enhanced = "5.1.1", + postfix_status_data = "does-not-exist@example.com", + postfix_status_message = "Recipient address rejected: User unknown in virtual mailbox table" + } + }, + ["smtpd_0019.yaml"] = { + "POSTFIX_SMTPD", + "SSL_accept error from unknown[72.13.58.7]: lost connection", + { + postfix_action = "SSL_accept error", + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtpd_lostconn_data = "lost connection" + } + }, + ["smtpd_0020.yaml"] = { + "POSTFIX_SMTPD", + "proxy-accept: END-OF-MESSAGE: 250 2.0.0 from MTA(smtp:[127.0.0.1]:10013): 250 2.0.0 Ok: queued as DF66E940333; from= to= proto=ESMTP helo=<[127.0.0.1]>", + { + postfix_keyvalue_data = "from= to= proto=ESMTP helo=<[127.0.0.1]>", + -- postfix_proxy_message = "250 2.0.0 from MTA(smtp:[127.0.0.1]:10013): 250 2.0.0 Ok: queued as DF66E940333", -- FIXME + postfix_proxy_result = "accept", + postfix_proxy_smtp_stage = "END-OF-MESSAGE", + postfix_proxy_status_code = 250, + postfix_proxy_status_code_enhanced = "2.0.0" + } + }, + ["smtpd_0021.yaml"] = { + "POSTFIX_SMTPD", + "proxy-reject: END-OF-MESSAGE: ; from= to= proto=ESMTP helo=<[192.168.0.16]>", + { + postfix_keyvalue_data = "from= to= proto=ESMTP helo=<[192.168.0.16]>", + -- postfix_proxy_message = "", -- FIXME + postfix_proxy_result = "reject", + postfix_proxy_smtp_stage = "END-OF-MESSAGE" + } + }, + ["smtpd_0022.yaml"] = { + "POSTFIX_SMTPD", + "proxy-reject: END-OF-MESSAGE: 554 5.7.0 Reject, contact postmaster@example.com, id=31619-02 - spam; from=<> to= proto=SMTP helo=<17.33.12.53>", + { + postfix_keyvalue_data = "from=<> to= proto=SMTP helo=<17.33.12.53>", + -- postfix_proxy_message = "554 5.7.0 Reject, contact postmaster@example.com, id=31619-02 - spam", -- FIXME + postfix_proxy_result = "reject", + postfix_proxy_smtp_stage = "END-OF-MESSAGE", + postfix_proxy_status_code = 554, + postfix_proxy_status_code_enhanced = "5.7.0" + } + }, + ["smtpd_0023.yaml"] = { + "POSTFIX_SMTPD", + "SSL_accept error from unknown[72.13.58.7]: Connection timed out", + { + postfix_action = "SSL_accept error", + postfix_client_hostname = "unknown", + postfix_client_ip = "72.13.58.7", + postfix_smtpd_lostconn_data = "Connection timed out" + } + }, + ["tlsmgr_0001.yaml"] = { + "POSTFIX_TLSMGR", + "warning: request to update table btree:/var/spool/postfix/smtpd_scache in non-postfix directory /var/spool/postfix", + { + postfix_warning = "request to update table btree:/var/spool/postfix/smtpd_scache in non-postfix directory /var/spool/postfix" + } + }, + ["tlsproxy_0001.yaml"] = { + "POSTFIX_TLSPROXY", + "DISCONNECT [82.118.17.142]:51637", + { + postfix_client_ip = "82.118.17.142", + postfix_client_port = 51637 + } + }, + ["tlsproxy_0002.yaml"] = { + "POSTFIX_TLSPROXY", + "CONNECT from [82.118.17.142]:51637", + { + postfix_client_ip = "82.118.17.142", + postfix_client_port = 51637 + } + }, + ["trivial_rewrite_0001.yaml"] = { + "POSTFIX_TRIVIAL_REWRITE", + "warning: virtual_alias_domains lookup failure", + { + postfix_warning = "virtual_alias_domains lookup failure" + } + }, + ["trivial_rewrite_0002.yaml"] = { + "POSTFIX_TRIVIAL_REWRITE", + "warning: dict_ldap_lookup: Search error -1: Can't contact LDAP server ", + { + postfix_warning = "dict_ldap_lookup: Search error -1: Can't contact LDAP server " + } + }, + -- testsuite additions, not in postfix-grok-patterns + ["keyvalue_data_0001.yaml"] = { + "POSTFIX_KEYVALUE_DATA", + "from=<>, to=, relay=dovecot, delay=4.3, delays=4.1/0.01/0/0.15, dsn=2.0.0, status=sent", + { + from = '', + to = 'tom@example.com', + relay = 'dovecot', + delay = '4.3', + delays = '4.1/0.01/0/0.15', + dsn = '2.0.0', + status='sent' + } + } +} + +function process_one(filename, programmname, message, expect, extract_keyvalue_data) + local ret = pf.postfix_match(programmname, message, extract_keyvalue_data) + if not ret then + error(string.format("%s: failed match", filename)) + elseif type(ret) ~= 'table' then + error(string.format("%s: incorrect type %s: %s", filename, type(ret), ret)) + else + for k,v in pairs(expect) do + if not ret[k] then + error(string.format("%s: missing key '%s'", filename, k)) + else + if type(ret[k]) == 'table' then + if ret[k]['value'] ~= v then + error(string.format("%s: data mismatch for '%s': '%s' != '%s'", filename, k, ret[k]['value'], v)) + end + elseif ret[k] ~= v then + error(string.format("%s: data mismatch for '%s': '%s' != '%s'", filename, k, ret[k], v)) + end + end + end + end +end + + +function process() + for filename, test in pairs(tests) do + local pattern = test[1] + local data = test[2] + local results = test[3] + local programmname = string.gsub(string.lower(pattern), '_', '/', 1) + process_one(filename, programmname, data, results) + end + -- test with extract_keyvalue_data + local message = '25F2E5E061: to=, relay=none, delay=0.05, delays=0.05/0/0/0, dsn=2.0.0, status=sent (test.example.com)' + local expect = { + postfix_queueid = "25F2E5E061", + postfix_to = 'george.desantis@test.example.com', + postfix_relay = 'none', + postfix_delay = 0.05, + postfix_delay_before_qmgr = 0.05, + postfix_delay_in_qmgr = 0, + postfix_delay_conn_setup = 0, + postfix_delay_transmission = 0, + postfix_dsn = '2.0.0', + postfix_status = 'sent' + } + process_one("discard_0001.yaml+extract_keyvalue_data", 'postfix/discard', message, expect, true) + return 0 +end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 3dfa5d4..1e0388e 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -820,6 +820,7 @@ static char* test_lpeg() , "lua/lpeg_date_time.lua" , "lua/lpeg_ip_address.lua" , "lua/lpeg_mysql.lua" + , "lua/lpeg_postfix.lua" , "lua/lpeg_syslog.lua" , "lua/lpeg_sfl4j.lua" , NULL From d5e3ee760d553cf91f385cd351a57dac653aa406 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Thu, 6 Aug 2015 14:23:00 +0200 Subject: [PATCH 069/235] Generic syslog message parser --- README.md | 1 + modules/syslog_message.lua | 626 +++++++++++++++++++++++++++ src/test/lua/lpeg_syslog_message.lua | 354 +++++++++++++++ src/test/test_lua_sandbox.c | 1 + 4 files changed, 982 insertions(+) create mode 100644 modules/syslog_message.lua create mode 100644 src/test/lua/lpeg_syslog_message.lua diff --git a/README.md b/README.md index 75f0748..0a8929e 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,7 @@ By default only the base library is loaded additional libraries must be loaded w - mysql - MySQL and MariaDB slow query and short slow query parsers - postfix - Postfix messages - syslog - Rsyslog meta grammar generator (creates a grammar based on the template configuration) + - syslog_message - Syslog messages - _user provided_ (lua, so/dll) *Return* diff --git a/modules/syslog_message.lua b/modules/syslog_message.lua new file mode 100644 index 0000000..8b30038 --- /dev/null +++ b/modules/syslog_message.lua @@ -0,0 +1,626 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- Copyright 2015 Mathieu Parent + +local string = require "string" +local ip = require "ip_address" +local l = require "lpeg" +l.locale(l) +local tonumber = tonumber +local type = type + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +-- Exported variables: +local prog_grammar = {} +local wildcard_grammar = {} + +-- LPEG helpers +local integer = (l.P"-"^-1 * l.digit^1) / tonumber +local float = (l.P"-"^-1 * l.digit^1 * (l.P"." * l.digit^1)^1) / tonumber +local ipv4 = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) +local ipv6 = l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation")) +local ipv46 = ipv4 + ipv6 + +local function capture_until(var, txt) + return l.Cg((l.P(1) - l.P(txt))^0, var) +end +local function capture_followed_by(var, txt) + return capture_until(var, txt) * l.P(txt) +end + +-- programname=CRON +prog_grammar["CRON"] = l.Ct( + l.P"(" + * capture_followed_by("cron_username", ") ") + * capture_followed_by("cron_event", " (") + * capture_followed_by("cron_detail", ")") + * l.P(-1) + ) + +-- programname=crontab +prog_grammar["crontab"] = prog_grammar["CRON"] + +-- programname=dhclient +prog_grammar["dhclient"] = l.Ct( + ( -- "DHCPDISCOVER on %s to %s port %d interval %ld" + l.Cg(l.P"DHCPDISCOVER", "dhcp_type") + * l.P" on " + * capture_followed_by("dhcp_client_interface", " to ") + * l.Cg(ipv4, "dhcp_server_addr") + * l.P" port " + * l.Cg(l.digit^1 / tonumber, "dhcp_server_port") + * l.P" interval " + * l.Cg(l.digit^1 / tonumber, "dhcp_client_interval_seconds") + * l.P(-1) + ) + ( -- "DHCPREQUEST on %s to %s port %d" + -- "DHCPDECLINE on %s to %s port %d" + -- "DHCPRELEASE on %s to %s port %d" + l.Cg(l.P"DHCPREQUEST" + l.P"DHCPDECLINE" + l.P"DHCPRELEASE", "dhcp_type") + * l.P" on " + * capture_followed_by("dhcp_client_interface", " to ") + * l.Cg(ipv4, "dhcp_server_addr") + * l.P" port " + * l.Cg(l.digit^1 / tonumber, "dhcp_server_port") + * l.P(-1) + ) + ( -- "DHCPACK from %s" + l.Cg(l.P"DHCPACK", "dhcp_type") + * l.P" from " + * l.Cg(ipv4, "dhcp_server_addr") + * l.P(-1) + ) + ( -- "bound to %s -- renewal in %ld seconds." + l.P"bound to " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" -- renewal in " + * l.Cg(l.digit^1 / tonumber, "dhcp_client_renewal_seconds") + * l.P" seconds." + * l.P(-1) + )) + +-- programname=dhcpd +local dhcpd_hw_addr = l.xdigit * l.xdigit * (l.S":" * l.xdigit * l.xdigit)^0 +prog_grammar["dhcpd"] = l.Ct( + ( + l.Cg(l.P"BOOTREQUEST", "dhcp_type") + * l.P" from " + * l.Cg(dhcpd_hw_addr, "dhcp_client_hw_addr") + * l.P" via " + * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") + * l.P(-1) + ) + ( + l.Cg(l.P"BOOTREPLY", "dhcp_type") + * l.P" for " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" to " + * capture_followed_by("dhcp_client_addr", " (") + * l.Cg(dhcpd_hw_addr, "dhcp_client_hw_addr") + * l.P") via " + * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") + * l.P(-1) + ) + ( + l.Cg(l.P"DHCPDISCOVER", "dhcp_type") + * l.P" from " + * l.Cg(dhcpd_hw_addr + l.P"", "dhcp_client_hw_addr") + * l.P" " + * (l.P"(" * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hostname") * l.P") ")^-1 + * l.P"via " + * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") + * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 + * l.P(-1) + ) + ( + l.Cg(l.P"DHCPOFFER" + l.P"DHCPACK" + l.P"BOOTREPLY", "dhcp_type") + * l.P" on " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" to " + * l.Cg((l.P(1)-l.P" ")^1, "dhcp_client_hw_addr") + * l.P" " + * (l.P"(" * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hostname") * l.P") ")^-1 + * l.P"via " + * l.Cg((l.P(1)-l.P" ")^1, "dhcp_source") + * ( l.P" [" + * l.Cg(integer, "dhcp_lease_time") + * l.P"]")^-1 + * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 + * l.P(-1) + ) + ( + l.Cg(l.P"DHCPACK", "dhcp_type") + * l.P" to " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" (" + * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hw_addr") + * l.P") via " + * l.Cg((l.P(1)-l.P" ")^1, "dhcp_source") + * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 + * ( l.P" [" + * l.Cg(integer, "dhcp_lease_time") + * l.P"]")^-1 + * l.P(-1) + ) + ( + l.Cg(l.P"DHCPNAK", "dhcp_type") + * l.P" on " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" to " + * l.Cg((l.P(1)-l.P" ")^1, "dhcp_client_hw_addr") + * l.P" via " + * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") + * l.P(-1) + ) + ( + l.Cg(l.P"DHCPREQUEST", "dhcp_type") + * l.P" for " + * l.Cg(ipv4, "dhcp_client_addr") + * (l.P" (" * l.Cg((l.P(1)-l.P")")^1, "dhcp_server_addr") * l.P")")^-1 + * l.P" from " + * l.Cg(dhcpd_hw_addr + l.P"", "dhcp_client_hw_addr") + * l.P" " + * (l.P"(" * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hostname") * l.P") ")^-1 + * l.P"via " + * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") + * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 + * l.P(-1) + ) + ( + l.Cg(l.P"DHCPINFORM", "dhcp_type") + * l.P" from " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" via " + * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") + * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 + * l.P(-1) + ) + ( + l.Cg(l.P"balancing", "dhcp_pool_result") + * l.P" pool" + * capture_followed_by("dhcp_pool_id", " ") + * capture_followed_by("dhcp_network", " total ") + * l.Cg(integer, "dhcp_lease_count") + * l.P" free " + * l.Cg(integer, "dhcp_free_leases") + * l.P" backup " + * l.Cg(integer, "dhcp_backup_leases") + * l.P" lts " + * l.Cg(integer, "dhcp_lts") + * l.P" max-own (+/-)" + * l.Cg(integer, "dhcp_hold") + * (l.P" (requesting peer rebalance!)")^-1 + * l.P(-1) + ) + ( + l.Cg(l.P"balanced" + l.P"IMBALANCED", "dhcp_pool_result") + * l.P" pool" + * capture_followed_by("dhcp_network", " total ") + * l.Cg(integer, "dhcp_lease_count") + * l.P" free " + * l.Cg(integer, "dhcp_free_leases") + * l.P" backup " + * l.Cg(integer, "dhcp_backup_leases") + * l.P" lts " + * l.Cg(integer, "dhcp_lts") + * l.P" max-misbal " + * l.Cg(integer, "dhcp_thresh") + * l.P(-1) + ) + ( + l.P"bind update on " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" from " + * capture_followed_by("dhcp_failover_peer", " ") + * l.Cg(l.P"rejected", "dhcp_bind_update_status") + * l.P": " + * l.Cg(l.P(1)^1, "dhcp_message") + ) + ( + l.P"bind update on " + * l.Cg(ipv4, "dhcp_client_addr") + * l.P" " + * l.Cg(l.P"got ack", "dhcp_bind_update_status") + * l.P" from " + * capture_followed_by("dhcp_failover_peer", ": ") + * l.Cg(l.P(1)^1, "dhcp_message") + )) + +-- programname=login +prog_grammar["login"] = l.Ct( + ( + l.P"FAILED LOGIN (" + * l.Cg(l.digit^1 / tonumber, "failcount") + * l.P")" + * l.P" on '" + * capture_followed_by("tty", "'") + * (l.P" from '" * capture_followed_by("from", "'"))^-1 + * l.P" FOR '" + * capture_followed_by("user", "', ") + * l.Cg(l.P(1)^1, "pam_error") + ) + ( + l.P"ROOT LOGIN " + * l.P" on '" + * capture_followed_by("tty", "'") + * (l.P" from '" * capture_followed_by("from", "'"))^-1 + * l.P(-1) + )) + +-- programname=groupadd +prog_grammar["groupadd"] = l.Ct( + ( + l.P"new group: name=" + * capture_followed_by("group_name", ", GID=") + * l.Cg(l.digit^1 / tonumber, "gid") + * l.P(-1) + )) + +-- programname=groupdel +prog_grammar["groupdel"] = l.Ct( + ( + l.P"group '" + * capture_followed_by("group_name", "' removed") + * (l.P" from " * l.Cg(l.P(1)^1, "group_dbname"))^-1 + )) + +-- programname=named +prog_grammar["named"] = l.Ct( + ( -- "lame server resolving "%s" (in "%s"?): %s" + l.Cg(l.P"lame server", "dns_error") + * l.P" resolving '" + * capture_followed_by("dns_name", "' (in '") + * capture_followed_by("dns_domain", "'?): ") + * l.Cg(ipv46, "dns_addr") + * l.P"#" + * l.Cg(l.digit^1 / tonumber, "dns_port") + * l.P(-1) + ) + ( -- "error (%s%s%s) resolving "%s/%s/%s": %s" before 1d761cb453c76353deb8423c78e98d00c5f86ffa + l.P"error (" + * capture_followed_by("dns_error", ") resolving '") + * capture_followed_by("dns_name", "/") + * capture_followed_by("dns_type", "/") + * capture_followed_by("dns_class", "': ") + * l.Cg(ipv46, "dns_addr") + * l.P"#" + * l.Cg(l.digit^1 / tonumber, "dns_port") + * l.P(-1) + ) + ( -- "%s%s%s resolving "%s/%s/%s": %s" after 1d761cb453c76353deb8423c78e98d00c5f86ffa + capture_followed_by("dns_error", " resolving '") + * capture_followed_by("dns_name", "/") + * capture_followed_by("dns_type", "/") + * capture_followed_by("dns_class", "': ") + * l.Cg(ipv46, "dns_addr") + * l.P"#" + * l.Cg(l.digit^1 / tonumber, "dns_port") + * l.P(-1) + ) + ( -- "DNS format error from %s resolving %s%s%s: %s" + l.Cg(l.P"DNS format error", "dns_error") + * l.P" from " + * l.Cg(ipv46, "dns_addr") + * l.P"#" + * l.Cg(l.digit^1 / tonumber, "dns_port") + * l.P" resolving " + * capture_followed_by("dns_name", "/") + * capture_until("dns_type", l.P" for client " + l.P":") + * (l.P" for client " + * l.Cg(ipv46, "dns_client_addr") + * l.P"#" + * l.Cg(l.digit^1 / tonumber, "dns_client_port") + )^-1 + * l.P": " + * l.Cg(l.P(1)^1, "dns_message") + ) + ( -- "skipping nameserver '%s' because it is a CNAME, while resolving '%s'" + l.Cg(l.P"skipping nameserver", "dns_error") + * l.P" '" + * capture_followed_by("dns_nameserver", "' because it is a CNAME, while resolving '") + * capture_followed_by("dns_name", "'") + * l.P(-1) + ) + ( -- "client %s%s%s%s%s%s%s%s: %s" + l.P"client " + * l.Cg(ipv46, "dns_client_addr") + * l.P"#" + * l.Cg(l.digit^1 / tonumber, "dns_client_port") + * (l.P"/key " * l.Cg((l.P(1)-l.S" :")^1, "dns_client_signer"))^-1 + * (l.P" (" * l.Cg((l.P(1)-l.P")")^1, "dns_name") * l.P")")^-1 + * (l.P": view " * l.Cg((l.P(1)-l.P": ")^1, "dns_view"))^-1 + * l.P": " + * l.Cg(l.P(1)^1, "dns_message") + ) + ( -- "success resolving "%s" (in "%s"?) after %s" + l.P"success resolving '" + * capture_followed_by("dns_name", "/") + * capture_followed_by("dns_type", "' (in '") + * capture_followed_by("dns_domain", "'?) after ") + * l.Cg(l.P(1)^1, "dns_message") + ) + ( -- "sending notifies (serial %u)" + l.P"zone " + * capture_followed_by("dns_domain", "/") + * capture_followed_by("dns_class", "/") + * capture_followed_by("dns_view", ": ") + * l.Cg("sending notifies", "dns_message") + * l.P" (serial " + * l.Cg(l.digit^1 / tonumber, "dns_serial") + * l.P")" + * l.P(-1) + ) + ( -- "clients-per-query decreased to %u" + l.P"clients-per-query decreased to " + * l.Cg(l.digit^1 / tonumber, "dns_clients_per_query") + * l.P(-1) + )) + +-- programname=puppet-agent +-- see http://docs.puppetlabs.com/puppet/latest/reference/lang_reserved.html#classes-and-defined-types +local puppet_namespace_segment = l.upper + * (l.lower + l.digit + l.P"_")^0 +local puppet_type = -- example: Mod::Config + puppet_namespace_segment + * (l.P"::" * puppet_namespace_segment)^0 +local puppet_resource = ( -- example: Mod::Config[foo] + puppet_type + * l.P"[" + * (l.P(1)-l.P"]")^1 + * l.P"]" + ) +local puppet_resource_path = ( -- example: /Stage[main]/Profile_one/Mod::Config[foo] + (l.P"/" * (puppet_resource + puppet_type))^1 + ) +--http://docs.puppetlabs.com/puppet/latest/reference/lang_reserved.html#parameters +local puppet_parameter = ( -- example: /Stage[main]/Mod::Config[foo]/ensure + (l.lower + l.digit + l.P"_")^1 + ) +local puppet_resource_message_cg = (-- "Triggered "#{callback}" from #{events.length} events" + -- "Would have triggered "#{callback}" from #{events.length} events" + l.Cg((l.P"Would have triggered" * l.Cg(l.Cc(true), "puppet_noop")) + l.P"Triggered", "puppet_msg") + * l.P" '" + * capture_followed_by("puppet_callback","' from ") + * l.Cg(integer, "puppet_events_count") + * l.P" events" + * l.P(-1) + ) + ( + l.P"Scheduling " + * capture_followed_by("puppet_callback"," of ") -- most probably "refresh" + * l.Cg(puppet_resource, "puppet_callback_target") + * l.P(-1) + ) + ( + l.P"Unscheduling " + * capture_followed_by("puppet_callback"," on ") -- most probably "refresh" + * l.Cg(puppet_resource, "puppet_callback_target") + * l.P(-1) + ) + ( + l.P"Filebucketed " + * capture_followed_by("puppet_file_path"," to ") + * capture_followed_by("puppet_bucket"," with sum ") + * capture_followed_by("puppet_file_sum",l.P(-1)) + ) +local puppet_parameter_message_cg = ( + l.P"current_value " + * capture_followed_by("puppet_current_value", ", should be ") + * capture_followed_by("puppet_should_value", " (noop)") + * (l.P" (previously recorded value was " *l.Cg(l.P(1)^1, "puppet_historical_value"))^-1 + * l.Cg(l.Cc(true), "puppet_noop") + * l.P(-1) + ) + ( + l.Cg(puppet_parameter, "puppet_ensure_parameter") + * l.P" changed '" + * capture_followed_by("puppet_old_value", "' to '") + * capture_followed_by("puppet_new_value", "'" * l.P(-1)) + ) + ( + l.Cg(l.P"executed successfully", "puppet_change") + * l.P(-1) + ) +prog_grammar["puppet-agent"] = l.Ct( + ( + l.P"(" + * l.Cg(puppet_resource_path, "puppet_resource_path") + * l.P"/" + * l.Cg(puppet_parameter, "puppet_parameter") + * l.P")" + * ( + (l.P" " * puppet_parameter_message_cg) + + l.P(1)^0 -- parameter can send arbitrary message + ) + ) + ( + l.P"(" + * capture_followed_by("puppet_resource_path", ") ") + * puppet_resource_message_cg + ) + (-- msg + (" in %0.2f seconds" % seconds) + l.Cg(l.P"Finished catalog run", "puppet_msg") + * l.P" in " + * l.Cg(float, "puppet_benchmark_seconds") + * l.P" seconds" + * l.P(-1) + ) + ( -- Keep as is + (l.P"Retrieving pluginfacts" * l.P(-1)) + + (l.P"Retrieving plugin" * l.P(-1)) + + (l.P"Loading facts" * l.P(-1)) + + (l.P"Caching catalog for ") + + (l.P"Applying configuration version '") + + (l.P"Computing checksum on file ") + + (l.P"Run of Puppet configuration client already in progress; skipping (") -- /var/lib/puppet/state/agent_catalog_run.lock exists) + )) +-- programname=sshd +prog_grammar["sshd"] = l.Ct( + ( + l.Cg(l.P"Accepted" + l.P"Failed" + l.P"Partial" + l.P"Postponed", "sshd_authmsg") + * l.P" " + * l.Cg((l.P(1)-l.S"/ ")^1, "sshd_method") + * (l.P"/" * l.Cg((l.P(1)-l.S"/ ")^1, "sshd_submethod"))^-1 + * l.P" for " + * l.P"invalid user "^-1 + * capture_followed_by("remote_user", " from ") + * l.Cg(ipv46, "remote_addr") + * l.P" port " + * l.Cg(l.digit^1 / tonumber, "remote_port") + * l.P" " + * (l.P"ssh2" + l.P"ssh1") + * (l.P": " * l.Cg(l.P(1)^1, "sshd_info"))^-1 + ) + ( + l.P"Received disconnect from " + * l.Cg(ipv46, "remote_addr") + * l.P": " + * (l.Cg(l.digit^1 / tonumber, "disconnect_reason") * l.P": ")^-1 + * l.Cg(l.P(1)^1, "disconnect_msg") + ) + ( + l.P"reverse mapping checking getaddrinfo for " + * capture_followed_by("remote_host", "[") + * l.Cg(ipv46, "remote_addr") + * l.P"] failed - POSSIBLE BREAK-IN ATTEMPT!" + * l.P(-1) + ) + ( + l.P"subsystem request for " + * capture_followed_by("sshd_subsystem", " by user " + l.P(-1)) + * l.Cg((l.P(1)-l.S" ")^0, "remote_user") + * l.P(-1) + ) + ( + l.P"Connection closed by " + * l.Cg(ipv46, "remote_addr") + * l.P" [preauth]" + * l.P(-1) + ) + ( + l.P"Invalid user " + * capture_followed_by("remote_user", " from ") + * l.Cg(ipv46, "remote_addr") + * l.P(-1) + ) + ( + l.P"input_userauth_request: invalid user " + * capture_followed_by("remote_user", " [preauth]") + * l.P(-1) + ) + ( + l.P"Exiting on signal " + * l.Cg(l.digit^1, "signal") + * l.P(-1) + ) + ( + l.P"Received signal " + * l.Cg(l.digit^1, "signal") + * l.P"; terminating." + * l.P(-1) + ) + ( + l.P"Server listening on " + * l.Cg((l.P(1)-l.S" ")^1, "listen_address") + * l.P" port " + * l.Cg(l.digit^1, "listen_port") + * l.P"." + * l.P(-1) + ) + ( + l.P"Did not receive identification string from " + * l.Cg(ipv46, "remote_addr") + * l.P(-1) + ) + ( + l.Cg(l.P"error" + l.P"fatal", "sshd_errorlevel") + * l.P": " + * l.Cg(l.P(1)^1, "sshd_error") + )) + +-- programname=su +prog_grammar["su"] = l.Ct( + ( + l.Cg(l.P"Successful" + l.P"FAILED", "su_status") + * l.P" su for " + * capture_followed_by("su_name", " by ") + * l.Cg(l.P(1)^1, "su_oldname") + ) + ( + l.P"pam_authenticate: " + * l.Cg(l.P(1)^1, "pam_error") + ) + ( + l.S"+-" + * l.P" " -- FIXME what to capture? + )) + +-- programname=sudo +local function sudo_field(name) + return (l.P(name) * l.P"=" * capture_followed_by("sudo_" .. string.lower(name), " ; "))^-1 +end +prog_grammar["sudo"] = l.Ct( + capture_followed_by("sudo_message", l.P" : ") + * sudo_field("TTY") + * sudo_field("PWD") + * sudo_field("USER") + * sudo_field("GROUP") + * sudo_field("TSID") + * sudo_field("ENV") + * l.P"COMMAND=" * l.Cg(l.P(1)^1, "sudo_command") + ) + +-- programname=systemd-logind +prog_grammar["systemd-logind"] = l.Ct( + ( + l.P"New session " + * l.Cg(integer, "session_id") + * l.P" of user " + * capture_followed_by("user_id", "." * l.P(-1)) + * l.Cg(l.Cc("SESSION_START"), "sd_message") + ) + ( + l.P"Removed session " + * l.Cg(integer, "session_id") + * l.P"." + * l.Cg(l.Cc("SESSION_STOP"), "sd_message") + * l.P(-1) + )) + +-- programname=useradd +prog_grammar["useradd"] = l.Ct( + ( + l.P"new user: name=" + * capture_followed_by("user_name", ", UID=") + * l.Cg(l.digit^1 / tonumber, "uid") + * l.P", GID=" + * l.Cg(l.digit^1 / tonumber, "gid") + * l.P", home=" + * capture_followed_by("user_home", ", shell=") + * l.Cg(l.P(1)^1, "user_shell") + ) + ( + l.P"add '" + * capture_followed_by("user_name", l.P"' to group '" + l.P"' to shadow group '") + * capture_followed_by("group_name", "'") + * l.P(-1) + )) + +-- PAM +local pam_header = capture_followed_by("pam_module", "(") + * capture_followed_by("pam_service", ":") + * capture_followed_by("pam_type", "): ") +wildcard_grammar["PAM"] = l.Ct( + ( pam_header + * l.Cg(l.P"session opened", "pam_action") + * l.P" for user " + * capture_followed_by("user_name", " by ") + * capture_followed_by("login_name", "(uid=") + * l.Cg(l.digit^1 / tonumber, "uid") + * l.P")" + ) + ( + pam_header + * l.Cg(l.P"session closed", "pam_action") + * l.P" for user " + * l.Cg(l.P(1)^1, "user_name") + ) + ( + pam_header + * l.Cg(l.P"authentication failure", "pam_action") + * l.P"; logname=" + * capture_followed_by("logname", " uid=") + * l.Cg(l.digit^1 / tonumber, "uid") + * l.P" euid=" + * l.Cg(l.digit^1 / tonumber, "euid") + * l.P" tty=" + * capture_followed_by("tty", " ruser=") + * capture_followed_by("ruser", " rhost=") + * l.Cg((l.P(1) - l.P(" user="))^0, "rhost") + * l.P" " -- duplicate space + * (l.P" user=" * l.Cg(l.P(1)^1, "user"))^-1 + ) + ( + pam_header + * l.P"check pass; user " + * (l.P"(" * capture_followed_by("user_name", ") "))^-1 + * l.P"unknown" + )) + +function get_prog_grammar(prog) + return prog_grammar[prog] +end + +-- If prog is a table, return a table of wildcard grammars +-- If prog is a string, return this particular wildcard grammar +-- If prog is nil, return all wildcard grammars +function get_wildcard_grammar(prog) + if type(prog) == "table" then + local ret = {} + for i,v in ipairs(prog) do + table.insert(ret, wildcard_grammar[prog]) + end + return ret + elseif type(prog) == "string" then + return wildcard_grammar[prog] + elseif prog == nil then + return wildcard_grammar + end +end + +return M diff --git a/src/test/lua/lpeg_syslog_message.lua b/src/test/lua/lpeg_syslog_message.lua new file mode 100644 index 0000000..2e57326 --- /dev/null +++ b/src/test/lua/lpeg_syslog_message.lua @@ -0,0 +1,354 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local syslog_message = require("syslog_message") + +local function CRON() + local grammar = syslog_message.get_prog_grammar('CRON') + local log + local fields + + log = '(root) CMD ( cd / && run-parts --report /etc/cron.hourly)' + fields = grammar:match(log) + assert(fields.cron_username == 'root', fields.cron_username) + assert(fields.cron_event == 'CMD', fields.cron_event) + assert(fields.cron_detail == ' cd / && run-parts --report /etc/cron.hourly', fields.cron_detail) + +end + +local function crontab() + local grammar = syslog_message.get_prog_grammar('crontab') + local log + local fields + + log = '(root) LIST (root)' + fields = grammar:match(log) + assert(fields.cron_username == 'root', fields.cron_username) + assert(fields.cron_event == 'LIST', fields.cron_event) + assert(fields.cron_detail == 'root', fields.cron_detail) +end + +local function dhclient() + local grammar = syslog_message.get_prog_grammar('dhclient') + local log + local fields + + log = 'DHCPDISCOVER on eth0 to 10.20.30.42 port 67 interval 18000' + fields = grammar:match(log) + assert(fields.dhcp_type == 'DHCPDISCOVER', fields.dhcp_type) + assert(fields.dhcp_client_interface == 'eth0', fields.dhcdhcp_client_interfacep_source) + assert(fields.dhcp_server_addr.value == '10.20.30.42', fields.dhcp_server_addr) + assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) + assert(fields.dhcp_server_port == 67, fields.dhcp_server_port) + assert(fields.dhcp_client_interval_seconds == 18000, fields.dhcp_client_interval_seconds) + + + log = 'DHCPREQUEST on eth0 to 10.20.30.42 port 67' + fields = grammar:match(log) + assert(fields.dhcp_type == 'DHCPREQUEST', fields.dhcp_type) + assert(fields.dhcp_client_interface == 'eth0', fields.dhcdhcp_client_interfacep_source) + assert(fields.dhcp_server_addr.value == '10.20.30.42', fields.dhcp_server_addr) + assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) + assert(fields.dhcp_server_port == 67, fields.dhcp_server_port) + + log = 'DHCPACK from 10.20.30.42' + fields = grammar:match(log) + assert(fields.dhcp_type == 'DHCPACK', fields.dhcp_type) + assert(fields.dhcp_server_addr.value == '10.20.30.42', fields.dhcp_server_addr) + assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) + + log = 'bound to 10.20.30.40 -- renewal in 20346 seconds.' + fields = grammar:match(log) + assert(fields.dhcp_client_addr.value == '10.20.30.40', fields.dhcp_client_addr) + assert(fields.dhcp_client_addr.representation == 'ipv4', fields.dhcp_client_addr) + assert(fields.dhcp_client_renewal_seconds == 20346, fields.dhcp_client_renewal_seconds) +end + +local function dhcpd() + local grammar = syslog_message.get_prog_grammar('dhcpd') + local log + local fields + + log = 'DHCPINFORM from 192.168.0.45 via 192.168.0.1' + fields = grammar:match(log) + assert(fields.dhcp_type == 'DHCPINFORM', fields.dhcp_type) + assert(fields.dhcp_client_addr.value == '192.168.0.45', fields.dhcp_client_addr) + assert(fields.dhcp_client_addr.representation == 'ipv4', fields.dhcp_client_addr) + assert(fields.dhcp_source == '192.168.0.1', fields.dhcp_source) + + log = 'DHCPDISCOVER from aa:bb:cc:dd:ee:ff via 10.2.3.4: unknown network segment' + fields = grammar:match(log) + assert(fields.dhcp_type == 'DHCPDISCOVER', fields.dhcp_type) + assert(fields.dhcp_client_hw_addr == 'aa:bb:cc:dd:ee:ff', fields.dhcp_client_hw_addr) + assert(fields.dhcp_source == '10.2.3.4', fields.dhcp_source) + assert(fields.dhcp_message == 'unknown network segment', fields.dhcp_message) + + log = 'DHCPACK to 192.168.2.3 (aa:bb:cc:dd:ee:ff) via vlan42' + fields = grammar:match(log) + assert(fields.dhcp_type == 'DHCPACK', fields.dhcp_type) + assert(fields.dhcp_client_hw_addr == 'aa:bb:cc:dd:ee:ff', fields.dhcp_client_hw_addr) + assert(fields.dhcp_source == 'vlan42', fields.dhcp_source) +end + +local function login() + local grammar = syslog_message.get_prog_grammar('login') + local log + local fields + + log = "FAILED LOGIN (1) on '/dev/tty1' FOR 'root', Authentication failure" + fields = grammar:match(log) + assert(fields.failcount == 1, fields.failcount) + assert(fields.tty == '/dev/tty1', fields.tty) + assert(fields.user == 'root', fields.user) + assert(fields.pam_error == 'Authentication failure', fields.pam_error) +end + +local function named() + local grammar = syslog_message.get_prog_grammar('named') + local log + local fields + + log = "lame server resolving '40.30.20.10.in-addr.arpa' (in '30.20.10.in-addr.arpa'?): 10.11.12.13#53" + fields = grammar:match(log) + assert(fields.dns_error == 'lame server', fields.dns_error) + assert(fields.dns_name == '40.30.20.10.in-addr.arpa', fields.dns_name) + assert(fields.dns_domain == '30.20.10.in-addr.arpa', fields.dns_domain) + assert(fields.dns_addr.value == '10.11.12.13', fields.dns_addr.value) + assert(fields.dns_addr.representation == 'ipv4', fields.dns_addr.representation) + assert(fields.dns_port == 53, fields.dns_port) + + log = "host unreachable resolving 'ipv6.example.org/AAAA/IN': 2001:503:231d::2:30#53" + fields = grammar:match(log) + assert(fields.dns_error == 'host unreachable', fields.dns_error) + assert(fields.dns_name == 'ipv6.example.org', fields.dns_name) + assert(fields.dns_type == 'AAAA', fields.dns_type) + assert(fields.dns_class == 'IN', fields.dns_class) + assert(fields.dns_addr.value == '2001:503:231d::2:30', fields.dns_addr.value) + assert(fields.dns_addr.representation == 'ipv6', fields.dns_addr.representation) + assert(fields.dns_port == 53, fields.dns_port) + + log = "DNS format error from 134.170.107.24#53 resolving cid-ff58b408a75804a8.users.storage.live.com/AAAA for client 10.2.3.4#60466: Name storage.live.com (SOA) not subdomain of zone users.storage.live.com -- invalid response" + fields = grammar:match(log) + assert(fields.dns_error == 'DNS format error', fields.dns_error) + assert(fields.dns_addr.value == '134.170.107.24', fields.dns_addr.value) + assert(fields.dns_addr.representation == 'ipv4', fields.dns_addr.representation) + assert(fields.dns_port == 53, fields.dns_port) + assert(fields.dns_name == 'cid-ff58b408a75804a8.users.storage.live.com', fields.dns_name) + assert(fields.dns_type == 'AAAA', fields.dns_type) + assert(fields.dns_client_addr.value == '10.2.3.4', fields.dns_client_addr.value) + assert(fields.dns_client_addr.representation == 'ipv4', fields.dns_client_addr.representation) + assert(fields.dns_client_port == 60466, fields.dns_client_port) + assert(fields.dns_message == "Name storage.live.com (SOA) not subdomain of zone users.storage.live.com -- invalid response", fields.dns_message) + + log = 'DNS format error from 184.105.66.196#53 resolving ns-os1.qq.com/AAAA: Name qq.com (SOA) not subdomain of zone ns-os1.qq.com -- invalid response' + fields = grammar:match(log) + assert(fields.dns_error == 'DNS format error', fields.dns_error) + assert(fields.dns_addr.value == '184.105.66.196', fields.dns_addr.value) + assert(fields.dns_addr.representation == 'ipv4', fields.dns_addr.representation) + assert(fields.dns_port == 53, fields.dns_port) + assert(fields.dns_name == 'ns-os1.qq.com', fields.dns_name) + assert(fields.dns_type == 'AAAA', fields.dns_type) + assert(fields.dns_message == "Name qq.com (SOA) not subdomain of zone ns-os1.qq.com -- invalid response", fields.dns_message) + + log = "client 10.8.6.1#17069/key trusty (pc.example.org): view internal: transfer of 'pc.example.org/IN': IXFR started: TSIG trusty (serial 12 -> 14)" + fields = grammar:match(log) + assert(fields.dns_client_addr.value == '10.8.6.1', fields.dns_client_addr.value) + assert(fields.dns_client_addr.representation == 'ipv4', fields.dns_client_addr.representation) + assert(fields.dns_client_port == 17069, fields.dns_client_port) + assert(fields.dns_client_signer == 'trusty', fields.dns_client_signer) + assert(fields.dns_name == 'pc.example.org', fields.dns_name) + --assert(fields.dns_view == 'internal', fields.dns_view) + assert(fields.dns_message == "transfer of 'pc.example.org/IN': IXFR started: TSIG trusty (serial 12 -> 14)", fields.dns_message) + + log = "success resolving 'ns1.example.com/AAAA' (in 'example.com'?) after disabling EDNS" + fields = grammar:match(log) + assert(fields.dns_name == 'ns1.example.com', fields.dns_name) + assert(fields.dns_type == 'AAAA', fields.dns_type) + assert(fields.dns_domain == 'example.com', fields.dns_domain) + assert(fields.dns_message == "disabling EDNS", fields.dns_message) + + log = "zone 12.11.10.in-addr.arpa/IN/internal: sending notifies (serial 42)" + fields = grammar:match(log) + assert(fields.dns_domain == '12.11.10.in-addr.arpa', fields.dns_domain) + assert(fields.dns_class == 'IN', fields.dns_class) + assert(fields.dns_view == 'internal', fields.dns_view) + assert(fields.dns_message == "sending notifies", fields.dns_message) + assert(fields.dns_serial == 42, fields.dns_serial) + + log = "clients-per-query decreased to 22" + fields = grammar:match(log) + assert(fields.dns_clients_per_query == 22, fields.dns_clients_per_query) +end + +local function puppet_agent() + local grammar = syslog_message.get_prog_grammar('puppet-agent') + local log + local fields + + log = '(/Stage[main]/Nantes::Profile::Heka_base/Exec[setcap CAP_NET_BIND_SERVICE=+eip /usr/bin/hekad]/returns) executed successfully' + fields = grammar:match(log) + assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/Exec[setcap CAP_NET_BIND_SERVICE=+eip /usr/bin/hekad]', fields.puppet_resource_path) + assert(fields.puppet_parameter == 'returns', fields.puppet_parameter) + assert(fields.puppet_change == 'executed successfully', fields.puppet_change) + + log = "(/Stage[main]/Logstash::Service/Logstash::Service::Init[logstash]/Service[logstash]/ensure) ensure changed 'stopped' to 'running'" + fields = grammar:match(log) + assert(fields.puppet_resource_path == '/Stage[main]/Logstash::Service/Logstash::Service::Init[logstash]/Service[logstash]', fields.puppet_resource_path) + assert(fields.puppet_parameter == 'ensure', fields.puppet_parameter) + assert(fields.puppet_old_value == 'stopped', fields.puppet_old_value) + assert(fields.puppet_new_value == 'running', fields.puppet_new_value) + + log = '(/Stage[main]/Elasticsearch::Config/File[/var/log/elasticsearch/test.log]/owner) current_value root, should be elasticsearch (noop)' + fields = grammar:match(log) + assert(fields.puppet_resource_path == '/Stage[main]/Elasticsearch::Config/File[/var/log/elasticsearch/test.log]', fields.puppet_resource_path) + assert(fields.puppet_parameter == 'owner', fields.puppet_parameter) + assert(fields.puppet_current_value == 'root', fields.puppet_current_value) + assert(fields.puppet_should_value == 'elasticsearch', fields.puppet_should_value) + assert(fields.puppet_noop == true, fields.puppet_noop) + + log = '(/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_local_syslog.toml]/content)' + fields = grammar:match(log) + assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_local_syslog.toml]', fields.puppet_resource_path) + assert(fields.puppet_parameter == 'content', fields.puppet_parameter) + + log = '(/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]) Scheduling refresh of Service[heka]' + fields = grammar:match(log) + assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]', fields.puppet_resource_path) + assert(fields.puppet_callback == 'refresh', fields.puppet_callback) + assert(fields.puppet_callback_target == 'Service[heka]', fields.puppet_callback_target) + + log = "(Class[Logstash::Service]) Would have triggered 'refresh' from 1 events" + fields = grammar:match(log) + assert(fields.puppet_resource_path == 'Class[Logstash::Service]', fields.puppet_resource_path) + assert(fields.puppet_msg == 'Would have triggered', fields.puppet_msg) + assert(fields.puppet_callback == 'refresh', fields.puppet_callback) + assert(fields.puppet_events_count == 1, fields.puppet_events_count) + + log = '(/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]) Filebucketed /etc/heka/conf.d/10_net_syslog.toml to main with sum 70358a826b06f61f36bdc6aecaa3db14' + fields = grammar:match(log) + assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]', fields.puppet_resource_path) + assert(fields.puppet_file_path == '/etc/heka/conf.d/10_net_syslog.toml', fields.puppet_file_path) + assert(fields.puppet_bucket == 'main', fields.puppet_bucket) + assert(fields.puppet_file_sum == '70358a826b06f61f36bdc6aecaa3db14', fields.puppet_file_sum) + + log = 'Finished catalog run in 7.11 seconds' + fields = grammar:match(log) + assert(fields.puppet_msg == 'Finished catalog run', fields.puppet_msg) + assert(fields.puppet_benchmark_seconds == 7.11, fields.puppet_benchmark_seconds) +end + +local function sshd() + local grammar = syslog_message.get_prog_grammar('sshd') + local log + local fields + + log = 'Accepted publickey for sathieu from 10.11.12.13 port 4242 ssh2' + fields = grammar:match(log) + assert(fields.sshd_authmsg == 'Accepted', fields.sshd_authmsg) + assert(fields.sshd_method == 'publickey', fields.sshd_method) + assert(fields.remote_user == 'sathieu', fields.remote_user) + assert(fields.remote_addr.value == '10.11.12.13', fields.remote_addr) + assert(fields.remote_port == 4242, fields.remote_port) + + log = "Failed password for invalid user administrator from 10.20.30.40 port 4242 ssh2" + fields = grammar:match(log) + assert(fields.sshd_authmsg == 'Failed', fields.sshd_authmsg) + assert(fields.sshd_method == 'password', fields.sshd_method) + assert(fields.remote_user == 'administrator', fields.remote_user) + assert(fields.remote_addr.value == '10.20.30.40', fields.remote_addr) + assert(fields.remote_port == 4242, fields.remote_port) + + log = "Received disconnect from 10.2.3.4: 11: The user disconnected the application [preauth]" + fields = grammar:match(log) + assert(fields.remote_addr.value == '10.2.3.4', fields.remote_addr) + assert(fields.disconnect_reason == 11, fields.disconnect_reason) + assert(fields.disconnect_msg == 'The user disconnected the application [preauth]', fields.disconnect_msg) +end + +local function sudo() + local grammar = syslog_message.get_prog_grammar('sudo') + local log + local fields + + log = "usrnagios : TTY=unknown ; PWD=/home/usrnagios ; USER=root ; COMMAND=/usr/bin/ctdb -Y status" + fields = grammar:match(log) + assert(fields.sudo_message == 'usrnagios', fields.sudo_message) + assert(fields.sudo_tty == 'unknown', fields.sudo_tty) + assert(fields.sudo_pwd == '/home/usrnagios', fields.sudo_pwd) + assert(fields.sudo_user == 'root', fields.sudo_user) + assert(fields.sudo_command == '/usr/bin/ctdb -Y status', fields.sudo_command) +end + +local function systemd_logind() + local grammar = syslog_message.get_prog_grammar('systemd-logind') + local log + local fields + + log = "New session 42 of user sathieu." + fields = grammar:match(log) + assert(fields.sd_message == 'SESSION_START', fields.sd_message) + assert(fields.session_id == 42, fields.session_id) + assert(fields.user_id == 'sathieu', fields.user_id) + + log = "Removed session 42." + fields = grammar:match(log) + assert(fields.sd_message == 'SESSION_STOP', fields.sd_message) + assert(fields.session_id == 42, fields.session_id) +end + +local function pam() + local grammar = syslog_message.get_wildcard_grammar('PAM') + local log + local fields + + log = "pam_unix(login:session): session opened for user sathieu by LOGIN(uid=1000)" + fields = grammar:match(log) + assert(fields.pam_module == 'pam_unix', fields.pam_module) + assert(fields.pam_service == 'login', fields.pam_service) + assert(fields.pam_type == 'session', fields.pam_type) + assert(fields.pam_action == 'session opened', fields.pam_action) + assert(fields.user_name == 'sathieu', fields.user_name) + assert(fields.uid == 1000, fields.uid) + + log = "pam_unix(login:auth): authentication failure; logname=LOGIN uid=0 euid=0 tty=/dev/tty1 ruser= rhost= user=root" + fields = grammar:match(log) + assert(fields.pam_module == 'pam_unix', fields.pam_module) + assert(fields.pam_service == 'login', fields.pam_service) + assert(fields.pam_type == 'auth', fields.pam_type) + assert(fields.pam_action == 'authentication failure', fields.pam_action) + assert(fields.logname == 'LOGIN', fields.logname) + assert(fields.uid == 0, fields.uid) + assert(fields.euid == 0, fields.euid) + assert(fields.tty == '/dev/tty1', fields.tty) + assert(fields.ruser == '', fields.ruser) + assert(fields.rhost == '', fields.rhost) + assert(fields.user == 'root', fields.user) + + log = "pam_unix(login:session): session closed for user sathieu" + fields = grammar:match(log) + assert(fields.pam_module == 'pam_unix', fields.pam_module) + assert(fields.pam_service == 'login', fields.pam_service) + assert(fields.pam_type == 'session', fields.pam_type) + assert(fields.pam_action == 'session closed', fields.pam_action) + assert(fields.user_name == 'sathieu', fields.user_name) +end + +function process() + CRON() + crontab() + dhclient() + dhcpd() + login() + named() + puppet_agent() + sshd() + sudo() + systemd_logind() + + -- wildcard grammars + pam() + + return 0 +end + diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 1e0388e..c273ca2 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -822,6 +822,7 @@ static char* test_lpeg() , "lua/lpeg_mysql.lua" , "lua/lpeg_postfix.lua" , "lua/lpeg_syslog.lua" + , "lua/lpeg_syslog_message.lua" , "lua/lpeg_sfl4j.lua" , NULL }; From bf6e1d0c13e7c3e0a7b526e75a6f9fa7cf80b529 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 15 Sep 2015 19:34:10 -0700 Subject: [PATCH 070/235] Add a method to access the sandbox Lua file name --- CMakeLists.txt | 2 +- include/luasandbox.h | 9 +++++++++ src/luasandbox.c | 6 ++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b392d81..2f20eeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR) project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 8) +set(CPACK_PACKAGE_VERSION_MINOR 9) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") diff --git a/include/luasandbox.h b/include/luasandbox.h index 59c7e2d..db32602 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -176,6 +176,15 @@ LSB_EXPORT void lsb_set_error(lua_sandbox* lsb, const char* err); */ LSB_EXPORT lua_State* lsb_get_lua(lua_sandbox* lsb); +/** + * Returns the filename of the Lua source. + * + * @param lsb Pointer to the sandbox. + * + * @return const char* filename. + */ +LSB_EXPORT const char* lsb_get_lua_file(lua_sandbox* lsb); + /** * Access the parent pointer stored in the sandbox. * diff --git a/src/luasandbox.c b/src/luasandbox.c index 4bd2015..f7458d5 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -480,6 +480,12 @@ lua_State* lsb_get_lua(lua_sandbox* lsb) } +const char* lsb_get_lua_file(lua_sandbox* lsb) +{ + return lsb->lua_file; +} + + void* lsb_get_parent(lua_sandbox* lsb) { return lsb->parent; From 2e77f0be736d9abe7e3653c7c311e05ce74edbf9 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 24 Sep 2015 07:54:12 -0700 Subject: [PATCH 071/235] Fix the error message typo --- src/lsb_output.c | 2 +- src/test/test_lua_sandbox.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lsb_output.c b/src/lsb_output.c index f0b1bcd..52a018b 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -198,7 +198,7 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) } break; default: - luaL_argerror(lsb->lua, i, "unsuported type"); + luaL_argerror(lsb->lua, i, "unsupported type"); break; } } diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index c273ca2..6660777 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -496,7 +496,7 @@ static char* test_output_errors() { const char* tests[] = { - "process() lua/output_errors.lua:10: bad argument #1 to 'output' (unsuported type)" + "process() lua/output_errors.lua:10: bad argument #1 to 'output' (unsupported type)" , "process() lua/output_errors.lua:16: output_limit exceeded" , "process() lua/output_errors.lua:19: write_message() could not encode protobuf - array has mixed types" , "process() lua/output_errors.lua:22: write_message() could not encode protobuf - unsupported type: nil" From ecfd265d6f17fb9cc09de3a3c47c8a85520ab794 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 25 Sep 2015 08:32:13 -0700 Subject: [PATCH 072/235] Add a Windows 10 ua_os_matcher --- CMakeLists.txt | 2 +- modules/common_log_format.lua | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f20eeb..b3a02aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 9) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index 5d31c4a..367159a 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -330,6 +330,7 @@ local ua_os_matchers = { , {"Windows NT 6.0" ,"Windows Vista"} , {"Windows NT 5.1" ,"Windows XP" } , {"Windows NT 5.0" ,"Windows 2000" } + , {"Windows NT 10.0" ,"Windows 10" } } local ua_browser_matchers = { From d2fe6d9aa1cf8291113ef65202a6fc46a0f0e9ad Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 25 Sep 2015 08:42:32 -0700 Subject: [PATCH 073/235] Put in decending order --- modules/common_log_format.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index 367159a..fc8de00 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -324,13 +324,13 @@ local ua_os_matchers = { , {"Macintosh" ,"Macintosh" } , {"Mozilla/5.0 (Mobile;" ,"FirefoxOS" } --http://en.wikipedia.org/wiki/Microsoft_Windows#Timeline_of_releases + , {"Windows NT 10.0" ,"Windows 10" } , {"Windows NT 6.3" ,"Windows 8.1" } , {"Windows NT 6.2" ,"Windows 8" } , {"Windows NT 6.1" ,"Windows 7" } , {"Windows NT 6.0" ,"Windows Vista"} , {"Windows NT 5.1" ,"Windows XP" } , {"Windows NT 5.0" ,"Windows 2000" } - , {"Windows NT 10.0" ,"Windows 10" } } local ua_browser_matchers = { From d9ab8594065c286ee14f7e0a8647cd88af038174 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Wed, 30 Sep 2015 07:01:20 -0700 Subject: [PATCH 074/235] Fix size_t usage on Windows - mingw 64 (32 bit will not build on a 64 bit system since the -m32 flag is not passed) - msvc 32/64 --- cmake/mozsvc.cmake | 2 ++ src/lsb_deserialize_protobuf.c | 4 ++-- src/lsb_serialize.c | 4 +++- src/test/test_lua_sandbox.c | 44 +++++++++++++++++++--------------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 2818048..af83690 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -27,6 +27,8 @@ else() set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra") if (NOT WIN32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__USE_MINGW_ANSI_STDIO") endif() set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") set(CMAKE_C_FLAGS_DEBUG "-g") diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c index f1741ed..58d7d26 100644 --- a/src/lsb_deserialize_protobuf.c +++ b/src/lsb_deserialize_protobuf.c @@ -157,7 +157,7 @@ process_fields(lua_State* lua, case 0: p = read_varint(p, p + len, &val); if (!p) break; - lua_pushinteger(lua, val); + lua_pushnumber(lua, (lua_Number)val); lua_rawseti(lua, 5, ++value_count); break; case 2: @@ -169,7 +169,7 @@ process_fields(lua_State* lua, do { p = read_varint(p, p + len, &val); if (!p) break; - lua_pushinteger(lua, val); + lua_pushnumber(lua, (lua_Number)val); lua_rawseti(lua, 5, ++value_count); } while (p < e); diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index ecab416..f1057d7 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -487,7 +487,9 @@ static int file_exists(const char* fn) int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) { - if (!file_exists(data_file)) return 0; + if (!file_exists(data_file)) { + return 0; + } // Clear the sandbox limits during restoration. #ifdef LUA_JIT diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 6660777..d8c068b 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -21,6 +21,12 @@ #define snprintf _snprintf #endif +#ifdef _MSC_VER +#define PRIuSIZE "Iu" +#else +#define PRIuSIZE "zu" +#endif + #define mu_assert(cond, ...) \ do { \ if (!(cond)) { \ @@ -290,21 +296,21 @@ static char* test_destroy_error() static char* test_usage_error() { size_t u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "NULL sandbox memory usage received: %zu", u); + mu_assert(u == 0, "NULL sandbox memory usage received: %" PRIuSIZE, u); lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); mu_assert(sb, "lsb_create() received: NULL"); u = lsb_usage(NULL, LSB_UT_MAX + 1, LSB_US_CURRENT); - mu_assert(u == 0, "Invalid usage type received: %zu", u); + mu_assert(u == 0, "Invalid usage type received: %" PRIuSIZE, u); u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_MAX + 1); - mu_assert(u == 0, "Invalid usage stat received: %zu", u); + mu_assert(u == 0, "Invalid usage stat received: %" PRIuSIZE, u); mu_assert(sb, "lsb_create() received: NULL"); lsb_terminate(sb, "forced termination"); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "Terminated memory usage received: %zu", u); + mu_assert(u == 0, "Terminated memory usage received: %" PRIuSIZE, u); e = lsb_destroy(sb, NULL); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -336,38 +342,38 @@ static char* test_simple() lsb_get_error(sb)); size_t u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u > 0, "Current memory usage received: %zu", u); - printf("cur_mem %zu\n", u); + mu_assert(u > 0, "Current memory usage received: %" PRIuSIZE, u); + printf("cur_mem %" PRIuSIZE "\n", u); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_MAXIMUM); - mu_assert(u > 0, "Maximum memory usage received: %zu", u); - printf("max_mem %zu\n", u); + mu_assert(u > 0, "Maximum memory usage received: %" PRIuSIZE, u); + printf("max_mem %" PRIuSIZE "\n", u); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_LIMIT); - mu_assert(u == 65765, "Memory limit received: %zu", u); + mu_assert(u == 65765, "Memory limit received: %" PRIuSIZE, u); u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_CURRENT); - mu_assert(u == 7, "Current instructions received: %zu", u); + mu_assert(u == 7, "Current instructions received: %" PRIuSIZE, u); u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_MAXIMUM); - mu_assert(u == 7, "Maximum instructions received: %zu", u); - printf("max_ins %zu\n", u); + mu_assert(u == 7, "Maximum instructions received: %" PRIuSIZE, u); + printf("max_ins %" PRIuSIZE "\n", u); u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_LIMIT); - mu_assert(u == 1000, "Instruction limit received: %zu", u); + mu_assert(u == 1000, "Instruction limit received: %" PRIuSIZE, u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_CURRENT); - mu_assert(u == 0, "Current output received: %zu", u); + mu_assert(u == 0, "Current output received: %" PRIuSIZE, u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_MAXIMUM); - mu_assert(u == 0, "Maximum output received: %zu", u); - printf("max_out %zu\n", u); + mu_assert(u == 0, "Maximum output received: %" PRIuSIZE, u); + printf("max_out %" PRIuSIZE "\n", u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); - mu_assert(u == 1024, "Output limit received: %zu", u); + mu_assert(u == 1024, "Output limit received: %" PRIuSIZE, u); u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); - mu_assert(u == 1024, "Output limit received: %zu", u); + mu_assert(u == 1024, "Output limit received: %" PRIuSIZE, u); lsb_state s = lsb_get_state(sb); mu_assert(s == LSB_RUNNING, "lsb_get_state() received: %d", s); @@ -464,7 +470,7 @@ static char* test_output() if (outputs[x][0] == 0x10) { size_t header = 18; if (x == 19) { - mu_assert(written_data_len == 12346, "test: %d received: %zu", x, written_data_len); + mu_assert(written_data_len == 12346, "test: %d received: %" PRIuSIZE, x, written_data_len); written_data_len = header + 28; // just compare the protobuf before the hyperloglog } if (memcmp(outputs[x], written_data + header, written_data_len - header) From 9bbdd7206d75f02e8142035e1b3430d2fce9b89e Mon Sep 17 00:00:00 2001 From: Vladislav Vlastovskiy Date: Wed, 30 Sep 2015 18:54:52 +0300 Subject: [PATCH 075/235] Add asterisk before number of connections --- modules/common_log_format.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index fc8de00..7aaac4f 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -426,7 +426,7 @@ end nginx_error_grammar = l.Ct(l.Cg(dt.build_strftime_grammar("%Y/%m/%d %T") / dt.time_to_ns, "time") * l.space * "[" * nginx_error_levels * "]" * l.space * l.Cg(l.digit^1 / tonumber, "Pid") * "#" - * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": " * (l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") + * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": *" * (l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") * l.Cg(l.P(1)^0, "Payload")) return M From cfe9e456bc778bea9adfae6482d32c6ebc2fbde5 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 30 Sep 2015 10:40:59 -0700 Subject: [PATCH 076/235] Issue #96 fix OSX test failure when loading the shared lib --- src/test/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index b167b79..836d2ae 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -20,4 +20,7 @@ endif() add_test(NAME test_move_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -add_test(NAME test_sandbox WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMAND test_lua_sandbox) +add_test(NAME test_sandbox COMMAND test_lua_sandbox) +if(APPLE) + set_tests_properties(test_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=../lib) +endif() From 71a3a59dd0537f811d9a2bbee713d8b65aa45c1e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 30 Sep 2015 11:06:25 -0700 Subject: [PATCH 077/235] Issue #99 Pushed the merge button a little too fast - Fix for the fix --- modules/common_log_format.lua | 2 +- src/test/lua/lpeg_clf.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index 7aaac4f..41f87a9 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -426,7 +426,7 @@ end nginx_error_grammar = l.Ct(l.Cg(dt.build_strftime_grammar("%Y/%m/%d %T") / dt.time_to_ns, "time") * l.space * "[" * nginx_error_levels * "]" * l.space * l.Cg(l.digit^1 / tonumber, "Pid") * "#" - * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": *" * (l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") + * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": " * (l.P"*" * l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") * l.Cg(l.P(1)^0, "Payload")) return M diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index 33e1309..1a55296 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -384,7 +384,7 @@ end local function nginx_error_connection() -- optional connection - local log = '2014/03/01 11:29:39 [notice] 16842#0: 8878 using inherited sockets from "6;"' + local log = '2014/03/01 11:29:39 [notice] 16842#0: *8878 using inherited sockets from "6;"' local fields = clf.nginx_error_grammar:match(log) local result = { Pid = 16842, From a00b2c76a9701950769e376b05ffbedef2c0f786 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sat, 3 Oct 2015 14:38:56 -0700 Subject: [PATCH 078/235] Issue #102 serialization and output support for nan, inf, -inf --- CMakeLists.txt | 4 +-- cmake/externals.cmake | 2 +- include/luasandbox_output.h | 9 +++++++ src/lsb_output.c | 5 ++-- src/lsb_serialize.c | 37 +++++++++++++++++++++----- src/luasandbox.c | 1 + src/test/lua/circular_buffer_delta.lua | 10 ++++++- src/test/lua/serialize.lua | 3 +++ src/test/output/serialize.lua51.data | 13 +++++---- src/test/test_lua_sandbox.c | 2 ++ 10 files changed, 69 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b3a02aa..2e321ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 2.8 FATAL_ERROR) project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 9) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_MINOR 10) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index a88110e..a7fca85 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -123,7 +123,7 @@ externalproject_add( externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG e87bcc0a3ae29c5a7acaabbb7b1f2b56700769c6 + GIT_TAG 444a68f839026a976a15fa63e339c4b1a0f8bad0 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index 1bfee87..c7cf753 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -93,4 +93,13 @@ LSB_EXPORT void lsb_output(lua_sandbox* lsb, int start, int end, int append); */ LSB_EXPORT int lsb_output_protobuf(lua_sandbox* lsb, int index, int append); +/** + * More efficient output of a double to a string + * + * @param output Pointer the output collector. + * @param d Double value to convert to a string. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT int lsb_output_double(lsb_output_data* output, double d); #endif diff --git a/src/lsb_output.c b/src/lsb_output.c index 52a018b..08c2836 100644 --- a/src/lsb_output.c +++ b/src/lsb_output.c @@ -160,7 +160,7 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) for (int i = start; result == 0 && i <= end; ++i) { switch (lua_type(lsb->lua, i)) { case LUA_TNUMBER: - if (lsb_serialize_double(&lsb->output, lua_tonumber(lsb->lua, i))) { + if (lsb_output_double(&lsb->output, lua_tonumber(lsb->lua, i))) { result = 1; } break; @@ -192,9 +192,10 @@ void lsb_output(lua_sandbox* lsb, int start, int end, int append) luaL_argerror(lsb->lua, i, "unknown userdata type"); return; // never reaches here but the compiler doesn't know it } + lua_pushvalue(lsb->lua, i); lua_pushlightuserdata(lsb->lua, &lsb->output); result = fp(lsb->lua); - lua_pop(lsb->lua, 1); // remove output + lua_pop(lsb->lua, 2); // remove the copy of the value and the output } break; default: diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index f1057d7..23d34e3 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -37,7 +37,6 @@ typedef struct const void* globals; } serialization_data; -static const char* not_a_number = "nan"; static const char* preservation_version = "_PRESERVATION_VERSION"; static const char* serialize_function = "lsb_serialize"; @@ -559,12 +558,8 @@ int lsb_serialize_binary(const void* src, size_t len, lsb_output_data* output) return 0; } - -int lsb_serialize_double(lsb_output_data* output, double d) +static int fast_double(lsb_output_data* output, double d) { - if (isnan(d)) { - return lsb_appends(output, not_a_number, 3); - } if (d > INT_MAX) { return lsb_appendf(output, "%0.17g", d); } @@ -636,3 +631,33 @@ int lsb_serialize_double(lsb_output_data* output, double d) return 0; } + + +int lsb_serialize_double(lsb_output_data* output, double d) +{ + if (isnan(d)) { + return lsb_appends(output, "0/0", 3); + } + if (d == INFINITY) { + return lsb_appends(output, "1/0", 3); + } + if (d == -INFINITY) { + return lsb_appends(output, "-1/0", 4); + } + return fast_double(output, d); +} + + +int lsb_output_double(lsb_output_data* output, double d) +{ + if (isnan(d)) { + return lsb_appends(output, "nan", 3); + } + if (d == INFINITY) { + return lsb_appends(output, "inf", 3); + } + if (d == -INFINITY) { + return lsb_appends(output, "-inf", 4); + } + return fast_double(output, d); +} diff --git a/src/luasandbox.c b/src/luasandbox.c index f7458d5..4981553 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -312,6 +312,7 @@ lua_sandbox* lsb_create_custom(void* parent, lsb->lua_file = malloc(len + 1); if (!lsb->output.data || !lsb->lua_file) { + free(lsb->output.data); free(lsb->lua_file); lua_close(lsb->lua); lsb->lua = NULL; diff --git a/src/test/lua/circular_buffer_delta.lua b/src/test/lua/circular_buffer_delta.lua index fb6a2ad..512dc6e 100644 --- a/src/test/lua/circular_buffer_delta.lua +++ b/src/test/lua/circular_buffer_delta.lua @@ -73,6 +73,14 @@ function report(tc) cb:add(0, MIN_COL, 0) write_output(cb:format("cbufd")) elseif tc == 11 then - cb:fromstring(0, 0, 1, 2, 3, 4, 0, 3, 4) + cb:fromstring("1 1 1 2 3 4 0 3 4") + write_output(cb:format("cbufd")) + elseif tc == 12 then + data:set(0, ADD_COL, 1/0) + data:set(0, ADD_COL, 1/0) + data:set(0, SET_COL, -1/0) + data:add(0, SET_COL, 1) + data:set(0, GET_COL, data:get(0, ADD_COL)) + write_output(data:format("cbufd")) end end diff --git a/src/test/lua/serialize.lua b/src/test/lua/serialize.lua index 96703d2..a2b5e14 100644 --- a/src/test/lua/serialize.lua +++ b/src/test/lua/serialize.lua @@ -18,6 +18,9 @@ uuids = { {uuid="BD48B609-8922-4E59-A358-C242075CE088", type="test"}, {uuid="BD48B609-8922-4E59-A358-C242075CE089", type="test1"} } +nan = 0/0 +inf = 1/0 +ninf = -1/0 large_key = { aaaaaaaaaaaaaaaaaaa = {["BD48B609-8922-4E59-A358-C242075CE081"] = 1, diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index 66a7439..bd5e432 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -1,4 +1,5 @@ if _PRESERVATION_VERSION and _PRESERVATION_VERSION ~= 0 then return end +_G["ninf"] = -1/0 if _G["dataRef"] == nil then _G["dataRef"] = circular_buffer.new(3, 3, 1) end _G["dataRef"]:set_header(1, "Column_1", "count", "sum") _G["dataRef"]:set_header(2, "Column_2", "count", "sum") @@ -15,6 +16,9 @@ _G["kvp"]["r"][4] = 92.002 _G["kvp"]["r"][5] = 91.10001 _G["kvp"]["r"]["key"] = "val" _G["kvp"]["b"] = "bar" +if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end +_G["delta"]:set_header(1, "Column_1", "count", "sum") +_G["delta"]:fromstring("1 1 2 nan 0 2") _G["uuids"] = {} _G["uuids"][1] = {} _G["uuids"][1]["type"] = "test" @@ -22,16 +26,14 @@ _G["uuids"][1]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE088" _G["uuids"][2] = {} _G["uuids"][2]["type"] = "test1" _G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" +_G["inf"] = 1/0 +_G["rates"] = _G["kvp"]["r"] +_G["data"] = _G["dataRef"] _G["cycleb"] = {} _G["cycleb"]["a"] = {} _G["cycleb"]["a"]["b"] = _G["cycleb"] _G["cycleb"]["a"]["type"] = "cycle a" _G["cycleb"]["type"] = "cycle b" -_G["rates"] = _G["kvp"]["r"] -if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end -_G["delta"]:set_header(1, "Column_1", "count", "sum") -_G["delta"]:fromstring("1 1 2 nan 0 2") -_G["data"] = _G["dataRef"] _G["cyclea"] = _G["cycleb"]["a"] _G["large_key"] = {} _G["large_key"]["aaaaaaaaaaaaaaaaaaa"] = {} @@ -69,4 +71,5 @@ _G["nested"]["cb"]:set_header(5, "Column_5", "count", "sum") _G["nested"]["cb"]:set_header(6, "Column_6", "count", "sum") _G["nested"]["cb"]:fromstring("1 1 nan nan nan nan nan nan nan nan nan nan nan nan") _G["count"] = 0 +_G["nan"] = 0/0 _G["_VERSION"] = "Lua 5.1" diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index d8c068b..676f91c 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -677,6 +677,8 @@ static char* test_cbuf_delta() , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t1\tnan\n" , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t-2\t0\n" , "" + , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t3\t4\n" + , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n0\tinf\t-inf\tinf\n" , NULL }; From b2b888ccac63b90e28c72bb694c9597c07701bd8 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 4 Oct 2015 09:04:09 -0700 Subject: [PATCH 079/235] Add the symtseries SAX library to sandbox --- cmake/externals.cmake | 8 ++++ src/test/lua/sax.lua | 53 +++++++++++++++++++++++ src/test/lua/sax_benchmark.lua | 24 +++++++++++ src/test/test_lua_sandbox.c | 78 ++++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+) create mode 100644 src/test/lua/sax.lua create mode 100644 src/test/lua/sax_benchmark.lua diff --git a/cmake/externals.cmake b/cmake/externals.cmake index a7fca85..dbaa9b5 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -143,3 +143,11 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) + +externalproject_add( + lua_sax + GIT_REPOSITORY https://github.com/trink/symtseries.git + GIT_TAG f8231b556bc53dbf7a71ea7b577e388e0e21a728 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} +) diff --git a/src/test/lua/sax.lua b/src/test/lua/sax.lua new file mode 100644 index 0000000..99985b4 --- /dev/null +++ b/src/test/lua/sax.lua @@ -0,0 +1,53 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "sax" + +local SAX_N = 9 +local SAX_W = 3 +local SAX_C = 5 + +local data = {16,18,12,24,11,15,16,12,17} +local nan_data = {0/0,0/0,0/0,0/0,0/0,0/0,0/0,0/0,0/0} +local partial_inf_data = {16,0/0,12,1/0,11,15,-1/0,12,17} + +unused = sax.window.new(SAX_N, SAX_W, SAX_C) +win = sax.window.new(SAX_N, SAX_W, SAX_C) +nan_win = sax.window.new(SAX_N, SAX_W, SAX_C) +partial_inf_win = sax.window.new(SAX_N, SAX_W, SAX_C) +mm = sax.window.new(100, 5, 8) +mm_tail = sax.window.new(100, 5, 8) + +word = sax.word.new(data, SAX_W, SAX_C) +nan_word = sax.word.new("###", SAX_C) +w1 = word + +assert(tostring(win) == "###") + +function process() + for i,v in ipairs(data) do + win:add(v) + end + + for i,v in ipairs(nan_data) do + nan_win:add(v) + end + + for i,v in ipairs(partial_inf_data) do + partial_inf_win:add(v) + end + + for i=1, 1000000 do + mm:add(i-1) + end + + for i=999001, 1000000 do + mm_tail:add(i-1) + end + return 0 +end + +function report(tc) + write_output(win, " ", word, " ", mm, " ", mm_tail) +end diff --git a/src/test/lua/sax_benchmark.lua b/src/test/lua/sax_benchmark.lua new file mode 100644 index 0000000..31b2c5a --- /dev/null +++ b/src/test/lua/sax_benchmark.lua @@ -0,0 +1,24 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "sax" + +local data = {16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, + 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, + 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, + 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, + 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13} + +win = sax.window.new(100, 5, 8) +win:add(data) + +function process(ts) + win:add(ts) + return 0 +end + +function report(tc) + write_output(tostring(win:get_word())) +end + diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 676f91c..309934f 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -1299,6 +1299,51 @@ static char* test_cuckoo_filter() } +static char* test_sax() +{ + const char* output_file = "sax.preserve"; + const char* word = "CDC CDC ABEGH ABEGH"; + + lua_sandbox* sb = lsb_create(NULL, "lua/sax.lua", "modules", 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + lsb_add_function(sb, &write_output, "write_output"); + + result = report(sb, 0); + mu_assert(result == 0, "report() received: %d %s", result, lsb_get_error(sb)); + mu_assert(strcmp("### CDC ##### #####", written_data) == 0, "received: %s", written_data); + + result = process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + result = report(sb, 0); + mu_assert(result == 0, "report() received: %d", result); + mu_assert(strcmp(word, written_data) == 0, "received: %s", written_data); + e = lsb_destroy(sb, output_file); + mu_assert(!e, "lsb_destroy() received: %s", e); + + // re-load to test the preserved data + sb = lsb_create(NULL, "lua/sax.lua", "modules", 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + result = lsb_init(sb, output_file); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + lsb_add_function(sb, &write_output, "write_output"); + + report(sb, 0); + mu_assert(strcmp(word, written_data) == 0, "received: %s", written_data); + + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -1626,6 +1671,37 @@ static char* benchmark_cuckoo_filter_add() } +static char* benchmark_sax_add() +{ + int iter = 1000000; + + lua_sandbox* sb = lsb_create(NULL, "lua/sax_benchmark.lua", "modules", + 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + lsb_add_function(sb, &write_output, "write_output"); + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed + } + t = clock() - t; + report(sb, 0); + mu_assert(strcmp("ABEGH", written_data) == 0, "received: %s", written_data); + mu_assert(lsb_get_state(sb) == LSB_RUNNING, + "benchmark_sax_add() failed %s", lsb_get_error(sb)); + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + printf("benchmark_sax_add() %g seconds\n", ((float)t) + / CLOCKS_PER_SEC / iter); + + return NULL; +} + + + static char* all_tests() { mu_run_test(test_create_error); @@ -1657,6 +1733,7 @@ static char* all_tests() mu_run_test(test_sandbox_config); mu_run_test(test_decode_message); mu_run_test(test_cuckoo_filter); + mu_run_test(test_sax); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); @@ -1670,6 +1747,7 @@ static char* all_tests() mu_run_test(benchmark_hyperloglog_add); mu_run_test(benchmark_decode_message); mu_run_test(benchmark_cuckoo_filter_add); + mu_run_test(benchmark_sax_add); return NULL; } From cd2f624449f31bc44c9eb137662018d626650dfc Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 4 Oct 2015 09:12:56 -0700 Subject: [PATCH 080/235] Add SAX to the readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0a8929e..0b01fa8 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,8 @@ By default only the base library is loaded additional libraries must be loaded w - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. - Disabled functions (default): getenv, execute, exit, remove, rename, setlocale, tmpname. + - [sax](https://github.com/trink/symtseries/blob/master/README.md) Symbolic time series data analysis based on + [Symbolic Aggregate approXimation](http://www.cs.ucr.edu/~eamonn/SAX.pdf). - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs - [table](http://www.lua.org/manual/5.1/manual.html#5.5) From 6a7294aec037eff898efc475f9ed9df47e4eb34e Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Mon, 5 Oct 2015 08:26:39 -0700 Subject: [PATCH 081/235] Pull in the Windows build fixes --- cmake/externals.cmake | 6 +++--- src/lsb_serialize.c | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index dbaa9b5..9076107 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -93,7 +93,7 @@ add_dependencies(lua_cjson ${LUA_PROJECT}) externalproject_add( lua_struct GIT_REPOSITORY https://github.com/trink/struct.git - GIT_TAG 5cf31819bee0d829d058cb5219e95ef0b1dd43a8 + GIT_TAG b7e9b87d1ee36a5e22c6749be0959b45858beaad UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} @@ -123,7 +123,7 @@ externalproject_add( externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG 444a68f839026a976a15fa63e339c4b1a0f8bad0 + GIT_TAG bb6dd9f88f148813315b5a660b7e2ba47f958b31 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG f8231b556bc53dbf7a71ea7b577e388e0e21a728 + GIT_TAG ee04aa0f3f7a6527e5345e8d1f7c767c11915da8 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/src/lsb_serialize.c b/src/lsb_serialize.c index 23d34e3..3bdef0e 100644 --- a/src/lsb_serialize.c +++ b/src/lsb_serialize.c @@ -16,6 +16,10 @@ #include "luasandbox/lualib.h" #include "lsb_private.h" +#ifdef _MSC_VER +#pragma warning( disable : 4056 ) +#endif + typedef struct { const void* ptr; From 89c540babd3070b5abab3177607664b8c4989c77 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 5 Oct 2015 09:35:38 -0700 Subject: [PATCH 082/235] Pull in symtseries with cmake 2.8.7 compatible scripts --- cmake/externals.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 9076107..59accad 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG ee04aa0f3f7a6527e5345e8d1f7c767c11915da8 + GIT_TAG 3de6f7c8e9865ba1f98fe97ca9784f39b36657ba CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From 93be8e95dad9067007209f5edd475c54a7649d85 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 5 Oct 2015 10:40:09 -0700 Subject: [PATCH 083/235] Pull in SAX memory leak fixes --- cmake/externals.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 59accad..5114e6d 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG 3de6f7c8e9865ba1f98fe97ca9784f39b36657ba + GIT_TAG 3e71ac7c33836fb5ae17ef96aaebcd2a54a7679c CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From f1cfc92bdfa235c74b576c72fd52e11bc997c4f8 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Wed, 7 Oct 2015 12:46:46 +0200 Subject: [PATCH 084/235] Fix Timestamp field for Nginx error logs --- modules/common_log_format.lua | 2 +- src/test/lua/lpeg_clf.lua | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index 5d31c4a..223ccd3 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -422,7 +422,7 @@ function build_apache_grammar(log_format) return l.Ct(grammar) end -nginx_error_grammar = l.Ct(l.Cg(dt.build_strftime_grammar("%Y/%m/%d %T") / dt.time_to_ns, "time") +nginx_error_grammar = l.Ct(l.Cg(dt.build_strftime_grammar("%Y/%m/%d %T") / dt.time_to_ns, "Timestamp") * l.space * "[" * nginx_error_levels * "]" * l.space * l.Cg(l.digit^1 / tonumber, "Pid") * "#" * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": " * (l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index 33e1309..42f397c 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -376,7 +376,7 @@ local function nginx_error() Pid = 16842, Payload = 'using inherited sockets from "6;"', Severity = 5, - time = 1393673379000000000 + Timestamp = 1393673379000000000 } verify_result(fields, result) assert(fields.Fields.tid == 0, "expected a thread id of 0") @@ -390,7 +390,7 @@ local function nginx_error_connection() Pid = 16842, Payload = 'using inherited sockets from "6;"', Severity = 5, - time = 1393673379000000000 + Timestamp = 1393673379000000000 } verify_result(fields, result) assert(fields.Fields.tid == 0, "invalid tid") From 130e9fc514c55691942c17e05d2ea274d8707b79 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 8 Oct 2015 07:42:21 -0700 Subject: [PATCH 085/235] Pull in the SAX changes https://github.com/Quadrocube/symtseries/issues/44 --- CMakeLists.txt | 2 +- cmake/externals.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e321ff..20f997e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 10) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 5114e6d..37da65b 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG 3e71ac7c33836fb5ae17ef96aaebcd2a54a7679c + GIT_TAG 88ed8f81323f9a4c931bd690f44d1ab5c54b0340 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From ca4f91af2b6b1aeda18695c7b4867751de71648a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 14 Oct 2015 06:59:09 -0700 Subject: [PATCH 086/235] Fix the gcc 4.4 warning on the lua_sax build --- cmake/externals.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 37da65b..bbfedfa 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG 88ed8f81323f9a4c931bd690f44d1ab5c54b0340 + GIT_TAG bda5c929ca7e5a44677279c1b2d5c1ada6d385ab CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From d7ebd6b64bee14234870820e6b67313e3c660367 Mon Sep 17 00:00:00 2001 From: Rob Miller Date: Wed, 14 Oct 2015 12:55:39 -0700 Subject: [PATCH 087/235] Add `behead_array` utility function. --- .gitignore | 2 ++ modules/util.lua | 23 +++++++++++++++++++- src/test/lua/behead_array_test.lua | 35 ++++++++++++++++++++++++++++++ src/test/test_lua_sandbox.c | 21 ++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 src/test/lua/behead_array_test.lua diff --git a/.gitignore b/.gitignore index cfe34ee..2fdd6b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *.preserve release/ src/test/modules/ +Testing +*~ diff --git a/modules/util.lua b/modules/util.lua index bed263c..52ac840 100644 --- a/modules/util.lua +++ b/modules/util.lua @@ -10,7 +10,7 @@ local string = require "string" local M = {} setfenv(1, M) -- Remove external access to contain everything in the module --- flattens a Lua table so it can be encoded as a protobuf fields object +-- Flattens a Lua table so it can be encoded as a protobuf fields object. function table_to_fields(t, fields, parent) for k,v in pairs(t) do if parent then @@ -28,4 +28,25 @@ function table_to_fields(t, fields, parent) end end +-- Effectively removes all array values up to the provided index from an array +-- by copying end values to the array head and setting now unused entries at +-- the end of the array to `nil`. +function behead_array(idx, array) + if idx <= 1 then return end + local array_len = #array + local start_nil_idx = 1 -- If idx > #array we zero it out completely. + if idx <= array_len then + -- Copy values to lower indexes. + local difference = idx - 1 + for i = idx, array_len do + array[i-difference] = array[i] + end + start_nil_idx = array_len - difference + 1 + end + -- Empty out the end of the array. + for i = start_nil_idx, array_len do + array[i] = nil + end +end + return M diff --git a/src/test/lua/behead_array_test.lua b/src/test/lua/behead_array_test.lua new file mode 100644 index 0000000..f5b6f40 --- /dev/null +++ b/src/test/lua/behead_array_test.lua @@ -0,0 +1,35 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local string = require("string") +local util = require("util") + +local function alpha() + return {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", + "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", + "w", "x", "y", "z"} +end + +function process () + local a = alpha() + util.behead_array(10, a) + assert(a[1] == "j", string.format("a[1] should be 'j', is '%s'", tostring(a[1]))) + assert(#a == 17, string.format("#a should be 17, is %d", #a)) + + a = alpha() + util.behead_array(16, a) + assert(a[1] == "p", string.format("a[1] should be 'p', is '%s'", tostring(a[1]))) + assert(#a == 11, string.format("#a should be 11, is %d", #a)) + + a = alpha() + util.behead_array(0, a) + assert(a[1] == "a", string.format("a[1] should be 'a', is '%s'", tostring(a[1]))) + assert(#a == 26, string.format("#a should be 26, is %d", #a)) + + a = alpha() + util.behead_array(45, a) + assert(a[1] == nil, string.format("a[1] should be 'nil', is '%s'", tostring(a[1]))) + assert(#a == 0, string.format("#a should be 0, is %d", #a)) + return 0 +end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 309934f..65fd6d6 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -876,6 +876,26 @@ static char* test_util() } +static char* test_behead_array() +{ + lua_sandbox* sb = lsb_create(NULL, "lua/behead_array_test.lua", "modules", 0, 0, 0); + mu_assert(sb, "lsb_create() received: NULL"); + + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + + result = process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + + e = lsb_destroy(sb, NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + static char* test_serialize() { const char* output_file = "serialize.preserve"; @@ -1722,6 +1742,7 @@ static char* all_tests() mu_run_test(test_errors); mu_run_test(test_lpeg); mu_run_test(test_util); + mu_run_test(test_behead_array); mu_run_test(test_serialize); mu_run_test(test_restore); mu_run_test(test_serialize_failure); From d6f2ef75529fba546d73c2a8dab60c3b4a8513a4 Mon Sep 17 00:00:00 2001 From: Rob Miller Date: Wed, 14 Oct 2015 14:41:13 -0700 Subject: [PATCH 088/235] Merge behead_array tests into existing util_test suite. --- src/test/lua/behead_array_test.lua | 35 ----------------------------- src/test/lua/util_test.lua | 36 +++++++++++++++++++++++++++++- src/test/test_lua_sandbox.c | 21 ----------------- 3 files changed, 35 insertions(+), 57 deletions(-) delete mode 100644 src/test/lua/behead_array_test.lua diff --git a/src/test/lua/behead_array_test.lua b/src/test/lua/behead_array_test.lua deleted file mode 100644 index f5b6f40..0000000 --- a/src/test/lua/behead_array_test.lua +++ /dev/null @@ -1,35 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local string = require("string") -local util = require("util") - -local function alpha() - return {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", - "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", - "w", "x", "y", "z"} -end - -function process () - local a = alpha() - util.behead_array(10, a) - assert(a[1] == "j", string.format("a[1] should be 'j', is '%s'", tostring(a[1]))) - assert(#a == 17, string.format("#a should be 17, is %d", #a)) - - a = alpha() - util.behead_array(16, a) - assert(a[1] == "p", string.format("a[1] should be 'p', is '%s'", tostring(a[1]))) - assert(#a == 11, string.format("#a should be 11, is %d", #a)) - - a = alpha() - util.behead_array(0, a) - assert(a[1] == "a", string.format("a[1] should be 'a', is '%s'", tostring(a[1]))) - assert(#a == 26, string.format("#a should be 26, is %d", #a)) - - a = alpha() - util.behead_array(45, a) - assert(a[1] == nil, string.format("a[1] should be 'nil', is '%s'", tostring(a[1]))) - assert(#a == 0, string.format("#a should be 0, is %d", #a)) - return 0 -end diff --git a/src/test/lua/util_test.lua b/src/test/lua/util_test.lua index da2af6c..440cf8c 100644 --- a/src/test/lua/util_test.lua +++ b/src/test/lua/util_test.lua @@ -2,16 +2,50 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +local string = require("string") local util = require("util") local t = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} local f = {} -function process () +local function table_to_fields() util.table_to_fields(t, f, nil) assert(f.toplevel == 0, f.toplevel) assert(f["struct.item0"] == 0, f["struct.item0"]) assert(f["struct.item1"] == 1, f["struct.item1"]) assert(f["struct.item2.nested"] == "n1", f["struct.item2.nested"]) +end + +local function alpha() + return {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", + "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", + "w", "x", "y", "z"} +end + +local function behead_array() + local a = alpha() + util.behead_array(10, a) + assert(a[1] == "j", string.format("a[1] should be 'j', is '%s'", tostring(a[1]))) + assert(#a == 17, string.format("#a should be 17, is %d", #a)) + + a = alpha() + util.behead_array(16, a) + assert(a[1] == "p", string.format("a[1] should be 'p', is '%s'", tostring(a[1]))) + assert(#a == 11, string.format("#a should be 11, is %d", #a)) + + a = alpha() + util.behead_array(0, a) + assert(a[1] == "a", string.format("a[1] should be 'a', is '%s'", tostring(a[1]))) + assert(#a == 26, string.format("#a should be 26, is %d", #a)) + + a = alpha() + util.behead_array(45, a) + assert(a[1] == nil, string.format("a[1] should be 'nil', is '%s'", tostring(a[1]))) + assert(#a == 0, string.format("#a should be 0, is %d", #a)) +end + +function process () + table_to_fields() + behead_array() return 0 end diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c index 65fd6d6..309934f 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_lua_sandbox.c @@ -876,26 +876,6 @@ static char* test_util() } -static char* test_behead_array() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/behead_array_test.lua", "modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_serialize() { const char* output_file = "serialize.preserve"; @@ -1742,7 +1722,6 @@ static char* all_tests() mu_run_test(test_errors); mu_run_test(test_lpeg); mu_run_test(test_util); - mu_run_test(test_behead_array); mu_run_test(test_serialize); mu_run_test(test_restore); mu_run_test(test_serialize_failure); From 4bf74cc480c2306fcffe1e423186e115cc87222d Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 20 Oct 2015 06:52:32 -0700 Subject: [PATCH 089/235] Pull in the latest lua_sax - mindist support for NaN - cmake C only restriction --- CMakeLists.txt | 2 +- cmake/externals.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 20f997e..4d4de4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 10) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index bbfedfa..f437703 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG bda5c929ca7e5a44677279c1b2d5c1ada6d385ab + GIT_TAG e50724abf8b22198ab275b9d9ccabbef89250bc1 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From 409a80f861a9a9af51fdfbb64a49f0978df079b2 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Fri, 18 Dec 2015 11:22:24 +0100 Subject: [PATCH 090/235] Postfix syslog_name can be different than postfix See http://www.postfix.org/postconf.5.html#syslog_name --- modules/postfix.lua | 3 +-- src/test/lua/lpeg_postfix.lua | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/postfix.lua b/modules/postfix.lua index 07d12dd..cf8960a 100644 --- a/modules/postfix.lua +++ b/modules/postfix.lua @@ -411,8 +411,7 @@ postfix_patterns['lmtp'] = postfix_patterns['smtp'] -local postfix_programname = l.P'postfix' - * (l.P(1) - l.P'/')^0 +local postfix_programname = (l.P(1) - l.P'/')^1 * '/' * l.Cg(l.P(1)^1) diff --git a/src/test/lua/lpeg_postfix.lua b/src/test/lua/lpeg_postfix.lua index 08ced5a..64b526b 100644 --- a/src/test/lua/lpeg_postfix.lua +++ b/src/test/lua/lpeg_postfix.lua @@ -1095,6 +1095,6 @@ function process() postfix_dsn = '2.0.0', postfix_status = 'sent' } - process_one("discard_0001.yaml+extract_keyvalue_data", 'postfix/discard', message, expect, true) + process_one("discard_0001.yaml+extract_keyvalue_data", 'mta-in/discard', message, expect, true) return 0 end From 90528889ced5aeac0613799ab27ac480c3c82890 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 7 Jan 2016 07:35:26 -0800 Subject: [PATCH 091/235] Fix timestamp tests that have a variable year --- src/test/lua/lpeg_date_time.lua | 2 +- src/test/lua/lpeg_syslog.lua | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua index eb1b0b7..ee26f07 100644 --- a/src/test/lua/lpeg_date_time.lua +++ b/src/test/lua/lpeg_date_time.lua @@ -59,7 +59,7 @@ end local function rfc3164() local tests = {"Feb 10 16:46:36"} - local results = {1423586796000000000} + local results = {os.time{year = os.date("%Y"), month = 2, day = 10, hour = 16, min = 46, sec = 36} * 1e9} test_valid(dt.rfc3164_timestamp, tests, results) end diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua index 799e1dc..93d34d3 100644 --- a/src/test/lua/lpeg_syslog.lua +++ b/src/test/lua/lpeg_syslog.lua @@ -9,7 +9,8 @@ local function traditional_file_format() local log = 'Feb 9 14:17:01 trink-x230 CRON[20758]: (root) CMD ( cd / && run-parts --report /etc/cron.hourly)\n' local fields = grammar:match(log) assert(fields.msg == "(root) CMD ( cd / && run-parts --report /etc/cron.hourly)", fields.msg) - assert(fields.timestamp == 1423491421000000000, fields.timestamp) + local ts = os.time{year = os.date("%Y"), month = 2, day = 9, hour = 14, min = 17, sec = 01} * 1e9 + assert(fields.timestamp == ts, fields.timestamp) assert(fields.syslogtag.pid == 20758, fields.syslogtag.pid) assert(fields.syslogtag.programname == "CRON", fields.syslogtag.programname) assert(fields.hostname == "trink-x230", fields.hostname) @@ -42,7 +43,8 @@ local function traditional_forward_format() local log = '<6>Feb 10 08:38:47 trink-x230 kernel: imklog 5.8.6, log source = /proc/kmsg started.' local fields = grammar:match(log) assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) - assert(fields.timestamp == 1423557527000000000, fields.timestamp) + local ts = os.time{year = os.date("%Y"), month = 2, day = 10, hour = 8, min = 38, sec = 47} * 1e9 + assert(fields.timestamp == ts, fields.timestamp) assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) assert(fields.pri.severity == 6, fields.pri.severity) assert(fields.pri.facility == 0, fields.pri.facility) @@ -67,7 +69,8 @@ local function kitchen_sink() assert(fields["syslogfacility-text"] == 0, fields["syslogfacility-text"]) assert(fields.syslogpriority == 6, fields.syslogpriority) assert(fields["syslogpriority-text"] == 6, fields["syslogpriority-text"]) - assert(fields.timegenerated == 1423560053000000000, fields.timegenerated) + local ts = os.time{year = os.date("%Y"), month = 2, day = 10, hour = 9, min = 20, sec = 53} * 1e9 + assert(fields.timegenerated == ts, fields.timegenerated) -- time reported is mapped to timestamp assert(fields.timestamp == 1392052853559934000, fields.timestamp) assert(fields["protocol-version"] == "0", fields["protocol-version"]) From 76ec3b9f954bf495503184d8c177c372249b9e3d Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 20 Jan 2016 14:07:19 -0800 Subject: [PATCH 092/235] Pull in most of the Hindsight sandbox and utilites Pulling the Hindsight functionality into libraries to make it reusable and embeddable in other tools. This commit contains the majority of the refactoring but there is still some remaining work (missing functionality and Mac and Windows platform support most are called out in the code as 'todo'). --- CMakeLists.txt | 2 + README.md | 122 +- cmake/externals.cmake | 12 +- cmake/mozsvc.cmake | 2 +- docs/heka_sandbox/analysis_plugins.md | 168 +++ docs/heka_sandbox/index.md | 18 + docs/heka_sandbox/input_plugins.md | 160 +++ docs/heka_sandbox/message.md | 32 + docs/heka_sandbox/message_matcher.md | 76 ++ docs/heka_sandbox/output_plugins.md | 428 +++++++ docs/heka_sandbox/stream_reader.md | 63 + docs/index.md | 16 + docs/sandbox.md | 131 +++ include/heka_sandbox/message_matcher.h | 79 ++ include/heka_sandbox/sandbox.h | 245 ++++ include/luasandbox.h | 170 ++- include/luasandbox_output.h | 84 +- include/luasandbox_serialize.h | 30 +- include/util/heka_message.h | 210 ++++ include/util/input_buffer.h | 67 ++ include/util/output_buffer.h | 117 ++ include/util/protobuf.h | 144 +++ include/util/running_stats.h | 54 + include/util/string.h | 37 + include/util/string_matcher.h | 37 + include/util/util.h | 47 + src/CMakeLists.txt | 10 +- src/heka_sandbox/CMakeLists.txt | 17 + src/heka_sandbox/message.c | 1010 +++++++++++++++++ src/heka_sandbox/message_impl.h | 64 ++ src/heka_sandbox/message_matcher.c | 588 ++++++++++ src/heka_sandbox/sandbox.c | 735 ++++++++++++ src/heka_sandbox/sandbox_impl.h | 44 + src/heka_sandbox/stream_reader.c | 242 ++++ src/heka_sandbox/stream_reader_impl.h | 29 + src/heka_sandbox/test/CMakeLists.txt | 16 + src/heka_sandbox/test/lua/aim.lua | 40 + src/heka_sandbox/test/lua/analysis.lua | 19 + .../test/lua/decode_message.lua | 4 + .../test/lua/decode_message_benchmark.lua | 6 +- src/heka_sandbox/test/lua/encode_message.lua | 43 + src/heka_sandbox/test/lua/iim.lua | 68 ++ src/heka_sandbox/test/lua/input.lua | 18 + src/heka_sandbox/test/lua/output.lua | 17 + src/heka_sandbox/test/lua/pm_no_return.lua | 7 + src/heka_sandbox/test/lua/read_message.lua | 54 + src/heka_sandbox/test/test.h.in | 28 + src/heka_sandbox/test/test_message_matcher.c | 300 +++++ src/heka_sandbox/test/test_sandbox.c | 503 ++++++++ src/lsb_deserialize_protobuf.c | 387 ------- src/lsb_output.c | 235 ---- src/lsb_private.h | 49 - src/lsb_serialize_protobuf.c | 640 ----------- src/lsb_serialize_protobuf.h | 24 - src/luasandbox.c | 451 +++++--- src/luasandbox_impl.h | 31 + src/luasandbox_output.c | 119 ++ ...lsb_serialize.c => luasandbox_serialize.c} | 235 ++-- src/test/CMakeLists.txt | 12 +- src/test/lua/output.lua | 62 - src/test/lua/output_errors.lua | 55 +- src/test/lua/read_config.lua | 17 + src/test/lua/sandbox_config.lua | 1 + src/test/mu_test.h | 66 ++ src/test/output/serialize.lua51.data | 12 +- .../{test_lua_sandbox.c => test_luasandbox.c} | 670 ++++------- src/util/CMakeLists.txt | 22 + src/util/heka_message.c | 538 +++++++++ src/util/input_buffer.c | 72 ++ src/util/output_buffer.c | 223 ++++ src/util/protobuf.c | 130 +++ src/util/running_stats.c | 42 + src/util/string.c | 15 + src/util/string_matcher.c | 275 +++++ src/util/test/CMakeLists.txt | 31 + src/util/test/test_heka_message.c | 334 ++++++ src/util/test/test_input_buffer.c | 148 +++ src/util/test/test_output_buffer.c | 180 +++ src/util/test/test_protobuf.c | 230 ++++ src/util/test/test_running_stats.c | 84 ++ src/util/test/test_string_matcher.c | 142 +++ src/util/test/test_util.c | 71 ++ src/util/util.c | 34 + 83 files changed, 9563 insertions(+), 2487 deletions(-) create mode 100644 docs/heka_sandbox/analysis_plugins.md create mode 100644 docs/heka_sandbox/index.md create mode 100644 docs/heka_sandbox/input_plugins.md create mode 100644 docs/heka_sandbox/message.md create mode 100644 docs/heka_sandbox/message_matcher.md create mode 100644 docs/heka_sandbox/output_plugins.md create mode 100644 docs/heka_sandbox/stream_reader.md create mode 100644 docs/index.md create mode 100644 docs/sandbox.md create mode 100644 include/heka_sandbox/message_matcher.h create mode 100644 include/heka_sandbox/sandbox.h create mode 100644 include/util/heka_message.h create mode 100644 include/util/input_buffer.h create mode 100644 include/util/output_buffer.h create mode 100644 include/util/protobuf.h create mode 100644 include/util/running_stats.h create mode 100644 include/util/string.h create mode 100644 include/util/string_matcher.h create mode 100644 include/util/util.h create mode 100644 src/heka_sandbox/CMakeLists.txt create mode 100644 src/heka_sandbox/message.c create mode 100644 src/heka_sandbox/message_impl.h create mode 100644 src/heka_sandbox/message_matcher.c create mode 100644 src/heka_sandbox/sandbox.c create mode 100644 src/heka_sandbox/sandbox_impl.h create mode 100644 src/heka_sandbox/stream_reader.c create mode 100644 src/heka_sandbox/stream_reader_impl.h create mode 100644 src/heka_sandbox/test/CMakeLists.txt create mode 100644 src/heka_sandbox/test/lua/aim.lua create mode 100644 src/heka_sandbox/test/lua/analysis.lua rename src/{ => heka_sandbox}/test/lua/decode_message.lua (97%) rename src/{ => heka_sandbox}/test/lua/decode_message_benchmark.lua (75%) create mode 100644 src/heka_sandbox/test/lua/encode_message.lua create mode 100644 src/heka_sandbox/test/lua/iim.lua create mode 100644 src/heka_sandbox/test/lua/input.lua create mode 100644 src/heka_sandbox/test/lua/output.lua create mode 100644 src/heka_sandbox/test/lua/pm_no_return.lua create mode 100644 src/heka_sandbox/test/lua/read_message.lua create mode 100644 src/heka_sandbox/test/test.h.in create mode 100644 src/heka_sandbox/test/test_message_matcher.c create mode 100644 src/heka_sandbox/test/test_sandbox.c delete mode 100644 src/lsb_deserialize_protobuf.c delete mode 100644 src/lsb_output.c delete mode 100644 src/lsb_private.h delete mode 100644 src/lsb_serialize_protobuf.c delete mode 100644 src/lsb_serialize_protobuf.h create mode 100644 src/luasandbox_impl.h create mode 100644 src/luasandbox_output.c rename src/{lsb_serialize.c => luasandbox_serialize.c} (69%) create mode 100644 src/test/lua/read_config.lua create mode 100644 src/test/mu_test.h rename src/test/{test_lua_sandbox.c => test_luasandbox.c} (65%) create mode 100644 src/util/CMakeLists.txt create mode 100644 src/util/heka_message.c create mode 100644 src/util/input_buffer.c create mode 100644 src/util/output_buffer.c create mode 100644 src/util/protobuf.c create mode 100644 src/util/running_stats.c create mode 100644 src/util/string.c create mode 100644 src/util/string_matcher.c create mode 100644 src/util/test/CMakeLists.txt create mode 100644 src/util/test/test_heka_message.c create mode 100644 src/util/test/test_input_buffer.c create mode 100644 src/util/test/test_output_buffer.c create mode 100644 src/util/test/test_protobuf.c create mode 100644 src/util/test/test_running_stats.c create mode 100644 src/util/test/test_string_matcher.c create mode 100644 src/util/test/test_util.c create mode 100644 src/util/util.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 4d4de4b..84167ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,6 +22,7 @@ if(LUA_JIT) add_definitions(-DLUA_JIT) endif() +find_library(LIBM_LIBRARY m) find_package(Git REQUIRED) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") @@ -31,6 +32,7 @@ include(mozsvc) include(externals) include_directories("${EP_BASE}/include") +include_directories("${CMAKE_SOURCE_DIR}/include") install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION share/doc/${PROJECT_NAME} COMPONENT core) diff --git a/README.md b/README.md index 0b01fa8..a102272 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -Lua Sandbox Library {#mainpage} +Lua Sandbox Library ------------------- ## Overview @@ -11,27 +11,27 @@ exposed to and facilitates new uses of the data (i.e. debugging, monitoring, dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, etc.) -The Lua sandbox is library allowing customized control over the Lua execution +The Lua sandbox is a library allowing customized control over the Lua execution environment including functionality like global data preservation/restoration on -shutdown/startup, output collection in textual, binary or [Heka protobuf format] -(https://hekad.readthedocs.org/en/latest/sandbox/index.html#sample-lua-message-structure) -and an array of parsers for various data types (Nginx, Apache, Syslog, MySQL and -many RFC grammars) +shutdown/startup, output collection in textual or binary formats and an array of +parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) ### Features -- small - memory requirements are as little as 6 KiB for a basic sandbox +- small - memory requirements are as little as 8 KiB for a basic sandbox - fast - microsecond execution times - stateful - ability to resume where it left off after a restart/reboot - isolated - failures are contained and malfunctioning sandboxes are terminated. Containment is defined in terms of restriction to the operating system, file system, libraries, memory use, Lua instruction use, and output size. +[Full Documentation](docs/index.md) + ## Installation ### Prerequisites -* C compiler (GCC 4.7+, Visual Studio 2013 (LuaJIT), MinGW (Lua 5.1)) -* CMake (2.8.7+) - http://cmake.org/cmake/resources/software.html +* C compiler (GCC 4.7+, Visual Studio 2013) +* CMake (3.0+) - http://cmake.org/cmake/resources/software.html * Git http://git-scm.com/download #### Optional (used for documentation) @@ -53,108 +53,4 @@ many RFC grammars) cmake -DCMAKE_BUILD_TYPE=release -G "NMake Makefiles" .. nmake - # Windows MinGW - cmake -DCMAKE_BUILD_TYPE=release -G "MinGW Makefiles" .. - mingw32-make - ctest - -## Sandbox API - -### Lua functions exposed to C by the core sandbox - -There are no functions exposed by default, see lsb_pcall_setup() and -lsb_pcall_teardown() when defining an API. - -### Functions exposed to Lua by the core sandbox -**require(libraryName)** - -By default only the base library is loaded additional libraries must be loaded with require(). - -*Arguments* - -- libraryName (string) - - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) - - The require() function has been modified to not expose any of the package table to the sandbox. - - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, newproxy, print. - - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) Test whether an element is a member of a set - - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) In memory time series data base and analysis - - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) JSON parser with the following modifications: - - Loads the cjson module in a global cjson table - - The encode buffer is limited to the sandbox output_limit. - - The decode buffer will be roughly limited to one half of the sandbox memory_limit. - - The NULL value is not decoded to cjson.null it is simply discarded. - If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - - The new() function has been disabled so only a single cjson parser can be created. - - The encode_keep_buffer() function has been disabled (the buffer is always reused). - - [cuckoo_filter](https://github.com/mozilla-services/lua_cuckoo_filter/blob/master/README.md) Bloom filter alternative supporting deletions - - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) Efficiently count the number of elements in a set - - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library - - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG - - [math](http://www.lua.org/manual/5.1/manual.html#5.6) - - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - - The local timezone is set to UTC in all sandboxes. - - Disabled functions (default): getenv, execute, exit, remove, rename, setlocale, tmpname. - - [sax](https://github.com/trink/symtseries/blob/master/README.md) Symbolic time series data analysis based on - [Symbolic Aggregate approXimation](http://www.cs.ucr.edu/~eamonn/SAX.pdf). - - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs - - [table](http://www.lua.org/manual/5.1/manual.html#5.5) - - Grammar Libraries - - cbufd - Parses the circular buffer library delta output (use for data aggregation) - - common_log_format - Nginx and Apache meta grammar generators (creates a grammar based on the log_format configuration) - - data_time - RFC3339, RFC3164, strftime, common log format, MySQL and Postgres timestamps - - ip_address - IPv4 and IPv6 address - - mysql - MySQL and MariaDB slow query and short slow query parsers - - postfix - Postfix messages - - syslog - Rsyslog meta grammar generator (creates a grammar based on the template configuration) - - syslog_message - Syslog messages - - _user provided_ (lua, so/dll) - -*Return* -- a table - For non user provided libraries the table is also globally registered - with the library name. User provided libraries may implement there own semantics - i.e. the grammar libraries return a table but do not globally register the table name - (see the individual module documentation for the correct usage). - -____ -**output(arg0, arg1, ...argN)** - lsb_appends data to the output buffer, which cannot exceed the output_limit - configuration parameter. See lsb_get_output() to connect the output to the - host application. - -*Arguments* -- arg (number, string, bool, nil, circular_buffer) Lua variable or literal to be appended the output buffer - -*Return* -- none - -**Note:** To extend the function set exposed to Lua see lsb_add_function() - -## How to interact with the sandbox (creating an API) - -The best place to start is with some examples: - -### Unit Test API - -[Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_lua_sandbox.c) - -#### Lua Functions Exposed to C - -- int **process** (double) - - exposes a process function that takes a test case number as its argument and returns and integer result code. -- void **report** (double) - - exposes a report function that takes a test case number as its argument and returns nothing. - -#### C Functions Exposed to Lua - -- void **write_output** (optionalArg1, optionalArg2, optionalArgN) - - captures whatever is in the output buffer for use by the host application, appending any optional arguments - (optional arguments have the same restriction as output). -- void **write_message** (table) - - serializes the Lua table into a Heka protobuf message and captures the output. - - the table must use the following [schema](https://hekad.readthedocs.org/en/latest/sandbox/index.html#sample-lua-message-structure). - -### Heka Sandbox API - -[Heka Sandbox](https://hekad.readthedocs.org/en/latest/sandbox/index.html#lua-sandbox) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index f437703..e6c9fad 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -65,7 +65,7 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG 57d3e82ef270b20dc976f3a6001439589c807793 + GIT_TAG cd455beaacf009838c2a493a6be98438a584b8b7 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -115,7 +115,7 @@ add_dependencies(lua_socket ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG 4151be752c3dd2f74d1c3487d8352ca54055eb81 + GIT_TAG c22db3f486a5debc269c7f20ff161f30852a4deb CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -123,7 +123,7 @@ externalproject_add( externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG bb6dd9f88f148813315b5a660b7e2ba47f958b31 + GIT_TAG b70d98f33f1b7abefab5da1747f4922c6969b634 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -131,7 +131,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 92f8a59e9e6f4ee381482f449b6cee78bb55460a + GIT_TAG 8d86410b27563f678ba0b88661b7975a7506be96 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -139,7 +139,7 @@ externalproject_add( externalproject_add( lua_cuckoo_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG 8343207aa7656d0324d23f79a5732fcb0bdb5917 + GIT_TAG eab57dde3d7f8e35e6c42ce5fb4bf606c4064e1a CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG e50724abf8b22198ab275b9d9ccabbef89250bc1 + GIT_TAG 028e84132f48431bb01a945392b3a212e1a53a47 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index af83690..42137fb 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -26,7 +26,7 @@ else() # Compiler options: http://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html#Invoking-GCC set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra") if (NOT WIN32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fvisibility=hidden") else() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__USE_MINGW_ANSI_STDIO") endif() diff --git a/docs/heka_sandbox/analysis_plugins.md b/docs/heka_sandbox/analysis_plugins.md new file mode 100644 index 0000000..ca9a808 --- /dev/null +++ b/docs/heka_sandbox/analysis_plugins.md @@ -0,0 +1,168 @@ +## Analysis Plugins + +### Required Lua Functions (called by the host) + +#### process_message + +Called when the host has a message available for analysis. Usually used in +combination with a [message matcher](message_matcher.md) expression. + +Recommenation: specify this as a `message_matcher` configuration option. + +*Arguments* +* none + +*Return* +* status_code (number) + - success (less than or equal to zero) + - fatal error (greater than zero) +* status_message (optional: string) logged when the status code is less than zero + +#### timer_event + +Called when the host timer expires or on shutdown. + +Recommendation: specify this as a `ticker_interval` configuration option. + +*Arguments* +* ns (number) - nanosecond timestamp of the function call (it is actually `time_t * 1e9` +to keep the timestamp units consistent so it will only have a one second resolution) +* shutdown (bool) - true if timer_event is being called due to a shutdown + +*Return* +* none + +### Available C Functions (called from the plugin) + +#### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +#### read_message + +Provides access to the Heka message data. Note that both fieldIndex and arrayIndex are zero-based +(i.e. the first element is 0) as opposed to Lua's standard indexing, which is one-based. + +*Arguments* +* variableName (string) + * framed (returns the Heka message protobuf string including the framing header) + * raw (returns the Heka message protobuf string) + * Uuid + * Type + * Logger + * Payload + * EnvVersion + * Hostname + * Timestamp + * Severity + * Pid + * Fields[*name*] +* fieldIndex (unsigned) - only used in combination with the Fields variableName + use to retrieve a specific instance of a repeated field *name*; zero indexed +* arrayIndex (unsigned) - only used in combination with the Fields variableName + use to retrieve a specific element out of a field containing an array; zero indexed + +*Return* +* value (number, string, bool, nil depending on the type of variable requested) + +#### decode_message + +Converts a Heka protobuf encoded message string into a Lua table. + +*Arguments* +* heka_pb (string) - Heka protobuf binary string + +*Return* +* msg ( [Heka message table (array fields)](message.md#array-based-message-fields)) + +#### inject_message + +Creates a new Heka protocol buffer message using the contents of the specified Lua table +(overwriting whatever is in the payload buffer). `Logger` and `Hostname` are set by +the infrastructure using the corresponding configuration setting and cannot be overridden +by the plugin. + +*Arguments* +* msg ([Heka message table](message.md)) + +*Return* +* none (throws an error if the table does not match the Heka message schema) + +#### add_to_payload + +Appends the arguments to the payload buffer for incremental construction of the final payload output +(`inject_payload` finalizes the buffer and sends the message to the infrastructure). This function +is a rename of the generic sandbox output function to improve the readability of the plugin code. + +*Arguments* +* arg (number, string, bool, nil, supported userdata) + +*Return* +* none (throws an error if arg is an unsupported type) + +#### inject_payload + +This is a wrapper function for `inject_message` that is included for backwards compatibility. The function +creates a new Heka message using the contents of the payload buffer (pre-populated with `add_to_payload`) and combined +with any additional payload_args passed here. The payload buffer is cleared after the injection. The `payload_type` +and `payload_name` arguments are two pieces of optional metadata stored is message fields. The resulting message is +structured like this: +```lua +msg = { +Timestamp = +Uuid = "" +Hostname = "" +Logger = "" +Type = "inject_payload" +Payload = "" +Fields = { + payload_type = "txt", + payload_name = "", +} +``` + +*Arguments* + +* payload_type (optional: string, default "txt") - describes the content type of the injected payload data +* payload_name (optional: string, default "") - names the content to aid in downstream filtering +* arg3 (optional) -ame type restrictions as `add_to_payload` +* ... +* argN + +*Return* +* none (throws an error if arg is an unsupported type) + +#### Modes of Operation + +##### Lock Step +* Receives one call to `process_message`, operates on the message, and returns success (0) or failure (-1) + +##### Example simple counter plugin +```lua +-- cfg +message_matcher = "TRUE" +ticker_interval = 5 +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local cnt = 0 + +function process_message() + cnt = cnt + 1 + return 0 +end + +function timer_event() + inject_payload("txt", "count", cnt) +end +``` diff --git a/docs/heka_sandbox/index.md b/docs/heka_sandbox/index.md new file mode 100644 index 0000000..96a0ba3 --- /dev/null +++ b/docs/heka_sandbox/index.md @@ -0,0 +1,18 @@ +# Heka Sandbox + +## Overview + +This is the 1.0 release of the Heka sandbox API. It is a refined implementation +of its predecessor which was developed in [Heka](https://github.com/mozilla-services/heka). +The goal is to decople it from Go and make it easily embeddable in any language. +Although this could be ported back to the Heka Go implementation a new design +of that infrastructure has been created to replace it called +[Hindsight](https://github.com/trink/hindsight). + +## Table of Contents + +* [Message Matcher](message_matcher.md) +* [Input Plugins](input_plugins.md) + * [Heka Stream Reader](heka_stream_reader.md) +* [Analysis Plugins](analysis_plugins.md) +* [Output Plugins](output_plugins.md) diff --git a/docs/heka_sandbox/input_plugins.md b/docs/heka_sandbox/input_plugins.md new file mode 100644 index 0000000..7424deb --- /dev/null +++ b/docs/heka_sandbox/input_plugins.md @@ -0,0 +1,160 @@ +## Input Plugins + +### Recommendations +Since he sandbox does not run in isolation there are some expectations of how +the host infrastructure behaves. The current recommendation are based on the +Hindsight reference implementation. + +### Required Lua Functions (called by the host) + +#### process_message + +Entry point for message creation. + +*Arguments* +* offset (nil number, string) - value of the last offset value passed into + `inject_message` + +*Return* +* status_code (number) + - success (less than or equal to zero) + - fatal error (greater than zero) +* status_message (optional: string) logged when the status code is less than zero + +### Available C Functions (called from the plugin) + +#### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +#### decode_message + +Converts a Heka protobuf encoded message string into a Lua table. + +*Arguments* +* heka_pb (string) - Heka protobuf binary string + +*Return* +* msg ([Heka message table (array fields)](message.md#array-based-message-fields)) + +#### inject_message + +Sends a Heka protocol buffer message into the host. + +*Arguments* +* msg ([Heka message table](message.md), [Heka stream reader](stream_reader.md) or Heka protobuf string) +* offset (optional: string, number) - checkpoint offset to be returned in the `process_message` call + +*Return* +* none - throws an error on invalid input + +### Modes of Operation + +#### Run Once +* Set the `ticker_interval` to zero and return from `process_message` when you + are done. +* The `instruction_limit` configuration can be set if desired. + +##### Example startup ping +```lua +-- cfg +-- send a simple 'hello' messages every time the host is started +ticker_interval = 0 +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- the checkpoint is optional and not being used by this plugin +function process_message() + inject_message({Type = "hello"}) + return 0 +end + +``` + +#### Polling + +* Set the `ticker_interval` greater than zero and non fatally (<=0) return from + `process_message`, when the ticker interval expires `process_message` will be + called again. +* The `instruction_limit` configuration can be set if desired. + +##### Example startup ping +```lua +-- cfg +ticker_interval = 60 +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "io" + +local msg = { +Type = "uptime", +Payload = "", +} + +function process_message() + local fh = io.popen("uptime") + if fh then + msg.Payload = fh:read("*a") + fh:close() + else + return -1, "popen failed" + end + if msg.Payload then inject_message(msg) end + return 0 +end + +``` + +#### Continuous + +* Don't return from `process_message`. +* The `instruction_limit` configuration **MUST** be set to zero. + +##### Example of a Heka protobuf stdin reader + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Reads a Heka protobuf stream from the stdin file handle + +-- .cfg +instruction_limit = 0 + +--]] + +local stdin = require "io".stdin +require "heka_stream_reader" + +local hsr = heka_stream_reader.new(read_config("Logger")) + +function process_message() + local found, consumed, read + repeat + repeat + found, consumed, read = hsr:find_message(stdin) + if found then + inject_message(hsr) + end + until not found + until read == 0 + return 0 +end +``` diff --git a/docs/heka_sandbox/message.md b/docs/heka_sandbox/message.md new file mode 100644 index 0000000..bb6d4f7 --- /dev/null +++ b/docs/heka_sandbox/message.md @@ -0,0 +1,32 @@ +## Heka Message Table + +### Hash Based Message Fields + +```lua +{ +Uuid = "data", -- ignored if not 16 byte raw binary UUID +Logger = "nginx", -- ignored by analysis plugins (the plugin name is used instead) +Hostname = "example.com", -- ignored by analysis plugins (the Hindsight hostname name is used instead) +Timestamp = 1e9, +Type = "TEST", +Payload = "Test Payload", +EnvVersion = "0.8", +Pid = 1234, -- ignored by analysis plugins (the Hindsight PID is used instead) +Severity = 6, +Fields = { + http_status = 200, -- encoded as a double + request_size = {value=1413, value_type=2, representation="B"} -- encoded as an integer + } +} +``` + +### Array Based Message Fields +```lua +{ +-- Message headers are the same as above +Fields = { + {name="http_status" , value=200}, -- encoded as a double + {name="request_size", value=1413, value_type=2, representation="B"} -- encoded as an integer + } +} +``` diff --git a/docs/heka_sandbox/message_matcher.md b/docs/heka_sandbox/message_matcher.md new file mode 100644 index 0000000..1bcfe60 --- /dev/null +++ b/docs/heka_sandbox/message_matcher.md @@ -0,0 +1,76 @@ +### Message Matcher Syntax + +The message matcher allows plugins to select which messages they want to consume +(see [Heka Message Structure](message.md)) + +#### Examples + +* Type == "test" && Severity == 6 +* (Severity == 7 || Payload == "Test Payload") && Type == "test" +* Fields[foo] != "bar" +* Fields[foo][1][0] == "alternate" +* Fields[MyBool] == TRUE +* TRUE +* Fields[created] =~ "^2015" +* Fields[widget] != NIL + +#### Relational Operators + +* == equals +* != not equals +* > greater than +* >= greater than equals +* < less than +* <= less than equals +* =~ Lua pattern match +* !~ Lua negated pattern match + +#### Logical Operators + +* Parentheses are used for grouping expressions +* && and (higher precedence) +* || or + +#### Boolean + +* TRUE +* FALSE + +#### Constants + +* NIL used to test the existence (!=) or non-existence (==) of a field variable + +#### Message Variables + +All message variables must be on the left hand side of the relational comparison + +##### String + +* Uuid - 16 byte raw binary type 4 UUID (useful for partitioning data) +* Type +* Logger +* Payload +* EnvVersion +* Hostname + +##### Numeric + +* Timestamp +* Severity +* Pid + +##### Fields + +* Fields[_field_name_] - shorthand for Field[_field_name_][0][0] +* Fields[_field_name_][_field_index_] - shorthand for Field[_field_name_][_field_index_][0] +* Fields[_field_name_][_field_index_][_array_index_] +* If a field type is mis-match for the relational comparison, false will be returned e.g., Fields[foo] == 6 where "foo" is a string + +#### Quoted String + +* single or double quoted strings are allowed + +#### Lua Pattern Matching Expression + +* see [Lua Patterns](http://www.lua.org/manual/5.1/manual.html#5.4.1) +* capture groups are ignored diff --git a/docs/heka_sandbox/output_plugins.md b/docs/heka_sandbox/output_plugins.md new file mode 100644 index 0000000..3ae5d75 --- /dev/null +++ b/docs/heka_sandbox/output_plugins.md @@ -0,0 +1,428 @@ +## Output Plugins + +### Required Lua Functions (called by the host) + +#### process_message + +Called when the host has a message available for analysis. Usually used in +combination with a [message matcher](message_matcher.md) expression. + +Recommenation: specify this as a `message_matcher` configuration option. + +*Arguments* +* sequence_id (optional: lightuserdata) - pass in when `async_buffer_size` is configured + +*Return* +* status_code (number) - see the [Modes of Operation](#modes-of-operation) for a detail explanation +of the return codes + - fatal error (greater than zero) + - success (0) + - non fatal failure (-1) + - skip (-2) + - retry (-3) + - batching (-4) + - async output (-5) +* status_message (optional: string) logged when the status code is -1 + +#### timer_event + +Called when the host timer expires or on shutdown. + +Recommendation: specify this as a `ticker_interval` configuration option. + +*Arguments* +* ns (number) - nanosecond timestamp of the function call (it is actually `time_t * 1e9` +to keep the timestamp units consistent so it will only have a one second resolution) +* shutdown (bool) - true if timer_event is being called due to a shutdown + +*Return* +* none + +### Available C Functions (called from the plugin) + +#### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +#### read_message + +Provides access to the Heka message data. See [read_message](analysis_plugins.md#read_message) for details. + +#### decode_message + +Converts a Heka protobuf encoded message string into a Lua table. + +*Arguments* +* heka_pb (string) - Heka protobuf binary string + +*Return* +* msg ([Heka message table (array fields)](heka_message_table.md#array-based-message-fields)) or an error is thrown + +#### encode_message + +Returns a Heka protocol buffer message using the contents of the specified Lua table. +Note: this operation uses the internal output buffer so it is goverened by the +`output_limit` configuration setting. If the `Logger` and `Hostname` are not set they will +be added by infrastructure using the corresponding configuration setting. + +*Arguments* +* msg ([Heka message table](heka_message_table.md) +* framed (bool default: false) A value of true includes the framing header + +*Return* +* heka_pb (string) - Heka protobuf binary string, framed as specified or an error is thrown + +#### update_checkpoint + +##### Batch Mode +Advances the output checkpoint when in batching mode. The standard use case is +to call it from `timer_event` after successfully flushing a batch on +timeout/shutdown. + +*Arguments* +* none + +##### Asynchronous Mode +Advances the output checkpoint and optionally reports the number of failures that occured. + +*Arguments* +* sequence_id (lightuserdata) - sequence_id for the message that was just successfully delivered/acknowledged +* failures (optional: integer) - number of failures that occured in the asynchronus processing (added to the failure count) + +*Return* +* none (throws an error on invalid arg types) + +#### Modes of Operation + +##### Lock Step + +* `process_message` operates on the message and returns one of the following values: + * success (0) - the message was successfully processed and the output checkpoint is advanced + * failure (-1) - the message was not successfully processed + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * the checkpoint is advanced + * skip (-2) - the message was intentionally not processed and the checkpoint is advanced + * retry (-3) - the message was not successfully processed and the host will call `process_message` + again, with the same message, after a one second delay + +##### Example Payload Output + +```lua +-- cfg +message_matcher = "Type == 'inject_payload'" +ticker_interval = 0 + +--location where the payload is written +output_dir = "/tmp" +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "io" +require "string" + +local output_dir = read_config("output_dir") or "/tmp" + +function process_message() + local pt = read_message("Fields[payload_type]") + if type(pt) ~= "string" then return -1, "invalid payload_type" end + + local pn = read_message("Fields[payload_name]") or "" + if type(pn) ~= "string" then return -1, "invalid payload_name" end + + local logger = read_message("Logger") or "" + + pn = string.gsub(pn, "%W", "_") + pt = string.gsub(pt, "%W", "_") + logger = string.gsub(logger, "%W", "_") + + local fn = string.format("%s/%s.%s.%s", output_dir, logger, pn, pt) + local fh, err = io.open(fn, "w") + if err then return -1, err end + + local payload = read_message("Payload") or "" + fh:write(payload) + fh:close() + return 0 +end + +function timer_event(ns) + -- no op +end +``` + +##### Batching + +* `process_message` batches the message/transformation in memory or on disk and returns one of the following values: + * batching (-4) - the message was successfully added to the batch + * failure (-1) - the message cannot be batch + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * skip (-2) - the message was intentionally not added to the batch + * retry (-3) - the message was not successfully added to the batch and the host will call `process_message` + again, with the same message, after a one second delay + * success (0) - the batch has been successfully committed and the output checkpoint is advanced to the most recent message + +##### Example Postgres Output + +```lua +-- cfg +message_matcher = "Type == 'logfile'" +memory_limit = 0 +ticker_interval = 60 + +buffer_max = 1000 +db_config = { + host = "example.com", + port = 5432, + name = "dev", + user = "test", + _password = "testpw", +} +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "os" +require "string" +require "table" + +local driver = require "luasql.postgres" + +local ticker_interval = read_config("ticker_interval") +local db_config = read_config("db_config") or error("db_config must be set") +local buffer_max = read_config("buffer_max") or 1000 +assert(buffer_max > 0, "buffer_max must be greater than zero") + +local env = assert(driver.postgres()) +local con, err = env:connect(db_config.name, db_config.user, db_config._password, db_config.host, db_config.port) +assert(con, err) + +local buffer = {} +local buffer_len = 0 +local table_name = "test_table" +local MAX_LENGTH = 65535 +local columns = { +-- column name field name field type field length + {"msg_Timestamp" ,"Timestamp" ,"TIMESTAMP" ,nil}, + {"payload" ,"Payload" ,"VARCHAR" ,MAX_LENGTH}, + {"sourceName" ,"Fields[sourceName]" ,"VARCHAR" ,30}, + {"sourceVersion" ,"Fields[sourceVersion]" ,"VARCHAR" ,12}, + {"submissionDate" ,"Fields[submissionDate]" ,"DATE" ,nil}, + {"sampleId" ,"Fields[sampleId]" ,"SMALLINT" ,nil}, +} + +local function make_create_table() + local pieces = {"CREATE TABLE IF NOT EXISTS ", table_name, " ("} + for i, c in ipairs(columns) do + table.insert(pieces, c[1]) + table.insert(pieces, " ") + table.insert(pieces, c[3]) + if c[4] ~= nil then + table.insert(pieces, "(") + table.insert(pieces, c[4]) + table.insert(pieces, ")") + end + if c[4] == MAX_LENGTH then + table.insert(pieces, " ENCODE LZO") + end + if i < #columns then + table.insert(pieces, ", ") + end + end + table.insert(pieces, ")") + return table.concat(pieces) +end +assert(con:execute(make_create_table())) + +local function bulk_load() + local cnt, err = con:execute(table.concat(buffer)) + if not err then + buffer = {} + buffer_len = 0 + else + return err + end +end + +local function esc_str(v) + if v == nil then return "NULL" end + + if type(v) ~= "string" then v = tostring(v) end + if string.len(v) > MAX_LENGTH then + v = "TRUNCATED:" .. string.sub(v, 1, MAX_LENGTH - 10) + end + + local escd = con:escape(v) + if not escd then return "NULL" end + + return string.format("'%s'", escd) +end + +local function esc_num(v) + if v == nil then return "NULL" end + if type(v) ~= "number" then return esc_str(v) end + return tostring(v) +end + +local function esc_ts(v) + if v == nil then return "NULL" end + if type(v) ~= "number" then return esc_str(v) end + local seconds = v / 1e9 + return table.concat({"(TIMESTAMP 'epoch' + ", seconds, " * INTERVAL '1 seconds')"}) +end + +local function make_insert(sep) + local pieces = {sep, "("} + for i=1,#columns do + if i > 1 then + table.insert(pieces, ",") + end + local col = columns[i] + if col[3] == "TIMESTAMP" then + table.insert(pieces, esc_ts(read_message(col[2]))) + elseif col[3] == "SMALLINT" then + table.insert(pieces, esc_num(read_message(col[2]))) + else + table.insert(pieces, esc_str(read_message(col[2]))) + end + end + table.insert(pieces, ")") + return table.concat(pieces) +end + +local last_load = 0 +function process_message() + local sep = "," + if buffer_len == 0 then + buffer_len = buffer_len + 1 + buffer[buffer_len] = string.format("INSERT INTO %s VALUES", table_name) + sep = " " + end + buffer_len = buffer_len + 1 + buffer[buffer_len] = make_insert(sep) + + if buffer_len - 1 >= buffer_max then + local err = bulk_load() + if err then + buffer[buffer_len] = nil + buffer_len = buffer_len - 1 + return -3, err + else + last_load = os.time() + return 0 + end + end + return -4 +end + +function timer_event(ns, shutdown) + if buffer_len > 1 and (shutdown or last_load + ticker_interval <= ns / 1e9)then + local err = bulk_load() + if not err then + batch_checkpoint_update() + end + end +end +``` + +##### Asynchronous + +* `async_buffer_size` **RECOMMENDED** that this configuration variable be set and consumed by the host +* `process_message` is called with a sequence_id parameter and asynchronously sends the message/transformation +to the destination and returns one of the following values: + * asynchronous (-5) - the message was successfully queued + * failure (-1) - the message cannot be queue + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * skip (-2) - the message was intentionally not queued + * retry (-3) - the message was not successfully queued and the host will call `process_message` + again, with the same message, after a one second delay +* When an asynchronously sent message is acknowledged [update_checkpoint](#update_checkpoint) +**MUST** be called to advance the checkpoint to that specific message + +##### Example Kafka Output + +```lua +-- cfg +message_matcher = "TRUE" +output_limit = 8 * 1024 * 1024 +brokers = "localhost:9092" +ticker_interval = 60 +async_buffer_size = 20000 + +topic_constant = "test" +producer_conf = { + ["queue.buffering.max.messages"] = async_buffer_size, + ["batch.num.messages"] = 200, + ["message.max.bytes"] = output_limit, + ["queue.buffering.max.ms"] = 10, + ["topic.metadata.refresh.interval.ms"] = -1, +} +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "kafka.producer" +require "kafka.topic" + +local brokers = read_config("brokers") or error("brokers must be set") +local topic_constant = read_config("topic_constant") +local topic_variable = read_config("topic_variable") or "Logger" +local producer_conf = read_config("producer_conf") + +local producer = kafka.producer.new(brokers, producer_conf) +local topics = {} + +function process_message(sequence_id) + local var = topic_constant + if not var then + var = read_message(topic_variable) or "unknown" + end + + local topic = topics[var] + if not topic then + topic = kafka.topic.new(producer, var) + topics[var] = topic + end + + producer:poll() -- calls async_checkpoint_update + local ret = topic:send(0, read_message("raw"), sequence_id) + + if ret ~= 0 then + if ret == 105 then + return -3 -- queue full retry + elseif ret == 90 then + return -1 -- message too large + elseif ret == 2 then + error("unknown topic: " .. var) + elseif ret == 3 then + error("unknown partition") + end + end + return -5 -- asynchronous checkpoint management +end + +function timer_event(ns) + producer:poll() -- calls update_checkpoint +end +``` diff --git a/docs/heka_sandbox/stream_reader.md b/docs/heka_sandbox/stream_reader.md new file mode 100644 index 0000000..cf5e583 --- /dev/null +++ b/docs/heka_sandbox/stream_reader.md @@ -0,0 +1,63 @@ +## Heka Stream Reader Module + +Enables parsing of a framed Heka protobuf stream in a Lua sandbox. See: +[Example of a Heka protobuf reader](input_plugins.md#example-of-a-heka-protobuf-stdin-reader) + +### API + +#### new + +Creates a Heka stream reader. + +```lua +local hsr = heka_stream_reader.new("stdin") + +``` + +*Arguments* +* name (string) - name of the stream reader (used in the log) + +*Return* +* hsr (userdata) - Heka stream reader or an error is thrown + +### API Methods + +#### find_message + +Locates a Heka message within the stream. + +```lua +local found, consumed, need = hsr:find_message(buf) + +``` + +*Arguments* +* buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data or a userdate file object +* decode (bool default: true) - true if the framed message should be protobuf decoded + +*Return* +* found (bool) - true if a message was found +* consumed (number) - number of bytes consumed so the offset can be tracked for checkpointing purposes +* need/read (number) - number of bytes needed to complete the message or fill the underlying buffer + or in the case of a file object the number of bytes added to the buffer + +#### decode_message + +Converts a Heka protobuf encoded message string into a stream reader representation. Note: this operation +clears the internal stream reader buffer. + +*Arguments* +* heka_pb (string) - Heka protobuf binary string + +*Return* +* none - throws an error on failure + +#### read_message + +Provides access to the Heka message data within the reader object. + +```lua +local ts = hsr:read_message("Timestamp") + +``` +See [read_message](analysis_plugins.md#read_message) for details. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..b7adeba --- /dev/null +++ b/docs/index.md @@ -0,0 +1,16 @@ +# Library Components + +## Overview +These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/trink/hindsight). +The goal was to decouple the Heka/Hindsight functionality from any particular +infrastructure and make it embeddable into any tool or language. + +## Table of Contents + +### Sandbox User APIs +* [Sandbox](sandbox.md) +* [Heka Sandbox](heka_sandox/index.md) + +### Sandbox Developer APIs +* todo - Links to the Doxygen output + diff --git a/docs/sandbox.md b/docs/sandbox.md new file mode 100644 index 0000000..6e09b3f --- /dev/null +++ b/docs/sandbox.md @@ -0,0 +1,131 @@ +Sandbox API +----------- + +### Configuration + +* **output_limit** - the largest output string an input or analysis plugin can inject into the host (bytes, default 64KiB) +* **memory_limit** - the maximum amount of memory a plugin can use before being terminated (bytes, default 8MiB) +* **instruction_limit** - the maximum number of Lua instructions a plugin can execute in a single API function call (count, default 1MM) +* **path** - The path used by require to search for a Lua loader. See [package loaders](http://www.lua.org/manual/5.1/manual.html#pdf-package.loaders) + for the path syntax. By default no paths are set in the sandbox and everything has been moved to a sandbox configuration table. +* **cpath** - The path used by require to search for a C loader (same notes as above) +* **disabled_modules** - Hash specifying which modules should be completely inaccessible. The existence of the key in the table will + disable the module. + ```lua + disabled_modules = {io = 1} + ``` +* **remove_entries** - Hash specifying which functions within a module should be inaccessible. + ```lua + remove_entries = { + os = {"getenv", "execute"}, + string = {"dump"} + } + ``` +* *user defined* any other variable (string, bool, number, table) is passed through as-is and available via [read_config](#read_config) + +### Lua functions exposed to C by the core sandbox + +There are no functions exposed by default, see lsb_pcall_setup() and +lsb_pcall_teardown() when defining an API. + +### Functions exposed to Lua by the core sandbox + +#### require + +By default only the base library is loaded additional libraries must be loaded with require(). + +*Arguments* + +- libraryName (string) + - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - The require() function has been modified to not expose any of the package table to the sandbox. + - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, newproxy, print. + - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) Test whether an element is a member of a set + - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) In memory time series data base and analysis + - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) JSON parser with the following modifications: + - Loads the cjson module in a global cjson table + - The encode buffer is limited to the sandbox output_limit. + - The decode buffer will be roughly limited to one half of the sandbox memory_limit. + - The NULL value is not decoded to cjson.null it is simply discarded. + If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. + - The new() function has been disabled so only a single cjson parser can be created. + - The encode_keep_buffer() function has been disabled (the buffer is always reused). + - [cuckoo_filter](https://github.com/mozilla-services/lua_cuckoo_filter/blob/master/README.md) Bloom filter alternative supporting deletions + - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) Efficiently count the number of elements in a set + - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library + - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG + - [math](http://www.lua.org/manual/5.1/manual.html#5.6) + - [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - The local timezone is set to UTC in all sandboxes. + - Disabled functions (default): getenv, execute, exit, remove, rename, setlocale, tmpname. + - [sax](https://github.com/trink/symtseries/blob/master/README.md) Symbolic time series data analysis based on + [Symbolic Aggregate approXimation](http://www.cs.ucr.edu/~eamonn/SAX.pdf). + - [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs + - [table](http://www.lua.org/manual/5.1/manual.html#5.5) + - Grammar Libraries + - cbufd - Parses the circular buffer library delta output (use for data aggregation) + - common_log_format - Nginx and Apache meta grammar generators (creates a grammar based on the log_format configuration) + - data_time - RFC3339, RFC3164, strftime, common log format, MySQL and Postgres timestamps + - ip_address - IPv4 and IPv6 address + - mysql - MySQL and MariaDB slow query and short slow query parsers + - postfix - Postfix messages + - syslog - Rsyslog meta grammar generator (creates a grammar based on the template configuration) + - syslog_message - Syslog messages + - _user provided_ (lua, so/dll) + +*Return* +- a table - For non user provided libraries the table is also globally registered + with the library name. User provided libraries may implement there own semantics + i.e. the grammar libraries return a table but do not globally register the table name + (see the individual module documentation for the correct usage). + +#### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +#### output +Appends data to the output buffer, which cannot exceed the output_limit +configuration parameter. See lsb_get_output() to connect the output to the +host application. + +*Arguments* +- arg (number, string, bool, nil, userdata implementing output support) - Lua + variable or literal to be appended the output buffer + +*Return* +- none + +**Note:** To extend the function set exposed to Lua see lsb_add_function() + +## How to interact with the sandbox (creating an API) + +The best place to start is with some examples: + +### Unit Test API + +[Unit Test Source Code](../src/test/test_luasandbox.c) + +#### Lua Functions Exposed to C + +- int **process** (double) + - exposes a process function that takes a test case number as its argument and returns and integer result code. +- void **report** (double) + - exposes a report function that takes a test case number as its argument and returns nothing. + +#### C Functions Exposed to Lua + +- void **write_output** (optionalArg1, optionalArg2, optionalArgN) + - captures whatever is in the output buffer for use by the host application, appending any optional arguments + (optional arguments have the same restriction as output). + +### Heka Sandbox + +[Heka Sandbox](heka_sandbox/index.html) + diff --git a/include/heka_sandbox/message_matcher.h b/include/heka_sandbox/message_matcher.h new file mode 100644 index 0000000..942e1ee --- /dev/null +++ b/include/heka_sandbox/message_matcher.h @@ -0,0 +1,79 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher @file */ + +#ifndef luasandbox_heka_sandbox_message_matcher_h_ +#define luasandbox_heka_sandbox_message_matcher_h_ + +#include + +#include "util/heka_message.h" + +typedef struct lsb_message_matcher lsb_message_matcher; +typedef struct lsb_message_match_builder lsb_message_match_builder; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initializes a Lua state containing the message matcher PEG. + * + * @param lua_path Path to the Lua based modules (date_time) + * @param lua_cpath Path to the C based modules (lpeg) + * @return NULL if the message match builder cannot be created + * + * @todo Remove the dependency on the sandbox and create a standalone + * match parser in the luasandboxutil lib + * + */ +LSB_EXPORT lsb_message_match_builder* +lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath); + +/** + * Frees all memory associated with the message matcher builder + * + * @param mmb Message matcher builder + */ +LSB_EXPORT void +lsb_destroy_message_match_builder(lsb_message_match_builder *mmb); + +/** + * Parsers a message matcher expression and returns the matcher + * + * @param mmb Message matcher builder + * @param exp Expression to parse into a matcher + * + * @return lsb_message_matcher* + */ +LSB_EXPORT lsb_message_matcher* +lsb_create_message_matcher(const lsb_message_match_builder *mmb, + const char *exp); + +/** + * Frees all memory associated with a message matcher instance + * + * @param mm Message matcher + */ +LSB_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); + +/** + * Evaluates the message matcher against the provided message + * + * @param mm Message matcher + * @param m Heka message + * + * @return bool True if the message is a match + */ +LSB_EXPORT bool lsb_eval_message_matcher(lsb_message_matcher *mm, + lsb_heka_message *m); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/heka_sandbox/sandbox.h b/include/heka_sandbox/sandbox.h new file mode 100644 index 0000000..2ac6e9b --- /dev/null +++ b/include/heka_sandbox/sandbox.h @@ -0,0 +1,245 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Heka sandbox implementation @file */ + +#ifndef luasandbox_heka_sandbox_sandbox_h_ +#define luasandbox_heka_sandbox_sandbox_h_ + +#include +#include + +#include "luasandbox.h" +#include "luasandbox/lua.h" +#include "luasandbox/lauxlib.h" +#include "util/heka_message.h" + +#define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" + +typedef struct lsb_heka_sandbox lsb_heka_sandbox; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * inject_message callback function provided by the host. Only one (or neither) + * of the checkpoint values will be set in a call. Numeric checkpoints can have + * a measurable performance benefit but are not always applicable so both option + * are provided to support various types of input plugins. + * + * @param parent Opaque pointer the host object owning this sandbox + * @param pb Pointer to a Heka protobuf encoded message being injected. + * @param pb_en Length of s + * @param cp_numeric Numeric based checkpoint (can be NAN) + * @param cp_string String based checkpoint (can be NULL) + * + * @return 0 on success + */ +typedef int (*lsb_heka_inject_message_input)(void *parent, + const char *pb, + size_t pb_len, + double cp_numeric, + const char *cp_string); + +/** + * inject_message callback function provided by the host. + * + * @param parent Opaque pointer the host object owning this sandbox. + * @param pb Pointer to a Heka protobuf encoded message being injected. + * @param len Length of pb_len + * + * @return 0 on success + */ +typedef int (*lsb_heka_inject_message_analysis)(void *parent, + const char *pb, + size_t pb_len); + +/** + * update_checkpoint callback function provided by the host. + * + * @param parent Opaque pointer the host object owning this sandbox. + * @param sequence_id Opaque pointer to the host message sequencing (passed into + * process_message). + * + * @return 0 on success + */ +typedef int (*lsb_heka_update_checkpoint)(void *parent, void *sequence_id); + +/** + * Create a sandbox supporting the Heka Input Plugin API + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified path to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Logger call back (NULL to disable logging) + * @param im inject_message call back + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_EXPORT +lsb_heka_sandbox* lsb_heka_create_input(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger logger, + lsb_heka_inject_message_input im); + +/** + * Host access to the input sandbox process_message API. If a numeric + * checkpoint is set the string checkpoint is ignored. + * + * @param hsb Heka input sandbox + * @param msg Heka message to process + * @param cp_numeric NAN if no numeric checkpoint + * @param cp_string NULL if no string checkpoint + * @param profile Take a timing sample on this execution + * + * @return int + * >0 fatal error + * 0 success + * <0 non-fatal error (status message is logged) + * + */ +LSB_EXPORT +int lsb_heka_process_message_input(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + double cp_numeric, + const char *cp_string, + bool profile); + +/** + * Create a sandbox supporting the Heka Analysis Plugin API + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified filename to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Logger call back (NULL to disable logging) + * @param im inject_message call back + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_EXPORT +lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger logger, + lsb_heka_inject_message_analysis im); + +/** + * Host access to the analysis sandbox process_message API + * + * @param hsb Heka analysis sandbox + * @param msg Heka message to process + * @param profile Take a timing sample on this execution + * + * @return int + * >0 fatal error + * 0 success + * <0 non-fatal error (status message is logged) + * + */ +LSB_EXPORT +int lsb_heka_process_message_analysis(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + bool profile); + +/** + * Create a sandbox supporting the Heka Output Plugin API + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified path to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Logger call back (NULL to disable logging) + * @param upc checkpoint_updated call back when using batch or async output + * + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_EXPORT +lsb_heka_sandbox* lsb_heka_create_output(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger logger, + lsb_heka_update_checkpoint ucp); + +/** + * Host access to the output sandobx process_message API + * + * @param hsb Heka output sandbox + * @param msg Heka message to process + * @param sequence_id Opaque pointer to the message sequence id (only used for + * async output plugin otherwise it should be NULL) + * @param profile Take a timing sample on this execution + * + * @return int + * >0 fatal error + * 0 success + * -1 non-fatal_error (status message is logged) + * -2 skip + * -3 retry + * -4 batching + * -5 async output + * + */ +LSB_EXPORT +int lsb_heka_process_message_output(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + void *sequence_id, + bool profile); + +/** + * Frees all memory associated with the sandbox; hsb cannont be used after this + * point and the host should set it to NULL. + * + * @param hsb Heka sandbox to free + * + * @return + * + */ +LSB_EXPORT void +lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); + +/** + * Host access to the timer_event API + * + * @param hsb Heka sandbox + * @param t Clock time of the timer_event execution + * @param shutdown Flag indicating the Host is shutting down allowing the + * sandbox to do any desired finialization) + * + * @return int 0 on success + */ +LSB_EXPORT +int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown); + + +/** + * Return the last error in human readable form. + * + * @param hsb Heka sandbox + * + * @return const char* error message + */ +LSB_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); + +// todo need access to the sandbox statistics. For simplicity it will most +// likely return a Heka protobuf string with all the info + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox.h b/include/luasandbox.h index db32602..60fff83 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -6,96 +6,103 @@ /** Generic Lua sandbox for dynamic data analysis @file */ -#ifndef lsb_h_ -#define lsb_h_ +#ifndef luasandbox_h_ +#define luasandbox_h_ #include "luasandbox/lua.h" #ifdef _WIN32 #ifdef luasandbox_EXPORTS -#define LSB_EXPORT __declspec(dllexport) +#define LSB_EXPORT __declspec(dllexport) #else -#define LSB_EXPORT __declspec(dllimport) +#define LSB_EXPORT __declspec(dllimport) #endif #else +#if __GNUC__ >= 4 +#define LSB_EXPORT __attribute__ ((visibility ("default"))) +#else #define LSB_EXPORT #endif +#endif + +#if defined(_MSC_VER) +#define PRIuSIZE "Iu" +#else +#define PRIuSIZE "zu" +#endif #define LSB_ERROR_SIZE 256 +#define LSB_CONFIG_TABLE "lsb_config" +#define LSB_MEMORY_LIMIT "memory_limit" +#define LSB_INSTRUCTION_LIMIT "instruction_limit" +#define LSB_OUTPUT_LIMIT "output_limit" +#define LSB_LUA_PATH "path" +#define LSB_LUA_CPATH "cpath" + typedef enum { - LSB_UNKNOWN = 0, - LSB_RUNNING = 1, - LSB_TERMINATED = 2 + LSB_UNKNOWN = 0, + LSB_RUNNING = 1, + LSB_TERMINATED = 2 } lsb_state; typedef enum { - LSB_US_LIMIT = 0, - LSB_US_CURRENT = 1, - LSB_US_MAXIMUM = 2, + LSB_US_LIMIT = 0, + LSB_US_CURRENT = 1, + LSB_US_MAXIMUM = 2, LSB_US_MAX } lsb_usage_stat; typedef enum { - LSB_UT_MEMORY = 0, - LSB_UT_INSTRUCTION = 1, - LSB_UT_OUTPUT = 2, + LSB_UT_MEMORY = 0, + LSB_UT_INSTRUCTION = 1, + LSB_UT_OUTPUT = 2, LSB_UT_MAX } lsb_usage_type; -typedef struct lua_sandbox lua_sandbox; -typedef struct lsb_output_data lsb_output_data; +typedef void (*lsb_logger)(const char* component, + int level, + const char* fmt, + ...); -/** - * Allocates and initializes the structure around the Lua sandbox. - * - * @param parent Pointer to associate the owner to this sandbox. - * @param lua_file Filename of the Lua script to run in this sandbox. - * @param require_path Location of the common sandbox modules - * @param memory_limit Sets the sandbox memory limit (bytes). - * @param instruction_limit Sets the sandbox Lua instruction limit (count). - * This limit is per call to process_message or timer_event - * @param output_limit Sets the single message payload limit (bytes). This - * limit applies to the in memory output buffer. The buffer is reset back - * to zero when inject_message is called. - * @return lua_sandbox Sandbox pointer or NULL on failure. - */ -LSB_EXPORT lua_sandbox* lsb_create(void* parent, - const char* lua_file, - const char* require_path, - unsigned memory_limit, - unsigned instruction_limit, - unsigned output_limit); +typedef struct lsb_lua_sandbox lsb_lua_sandbox; +typedef struct lsb_output_buffer lsb_output_buffer; + +#ifdef __cplusplus +extern "C" { +#endif /** * Allocates and initializes the structure around the Lua sandbox allowing * full specification of the sandbox configuration using a Lua configuration * string. - * - * { - * memory_limit = 1024*1024*1, - * instruction_limit = 10000, - * output_limit = 64*1024, - * path = '/modules/?.lua', - * cpath = '/modules/?.so', + * memory_limit = 1024*1024*1 + * instruction_limit = 10000 + * output_limit = 64*1024 + * path = '/modules/?.lua' + * cpath = '/modules/?.so' * remove_entries = { - * [''] = {'collectgarbage','coroutine','dofile','load','loadfile'" - * ",'loadstring','newproxy','print'}, - * os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'} - * }, - * disable_modules = {io = 1} + * [''] = + * {'collectgarbage','coroutine','dofile','load','loadfile','loadstring', + * 'newproxy','print'}, + * os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'} * } + * disable_modules = {io = 1} + * * * @param parent Pointer to associate the owner to this sandbox. * @param lua_file Filename of the Lua script to run in this sandbox. - * @param config Lua structure defining the full sandbox restrictions. - * @return lua_sandbox Sandbox pointer or NULL on failure. + * @param cfg Lua structure defining the full sandbox restrictions (may contain + * optional host configuration options, everything is available to + * the sandbox through the read_config API. + * @param logger Optional logger function pointer for error reporting + * @return lsb_lua_sandbox Sandbox pointer or NULL on failure. */ -LSB_EXPORT lua_sandbox* lsb_create_custom(void* parent, - const char* lua_file, - const char* config); +LSB_EXPORT lsb_lua_sandbox* +lsb_create(void *parent, const char *lua_file, const char *cfg, + lsb_logger logger); /** * Initializes the Lua sandbox and loads/runs the Lua script that was specified @@ -114,19 +121,17 @@ LSB_EXPORT lua_sandbox* lsb_create_custom(void* parent, * * @return int Zero on success, non-zero on failure. */ -LSB_EXPORT int lsb_init(lua_sandbox* lsb, const char* state_file); +LSB_EXPORT int lsb_init(lsb_lua_sandbox *lsb, const char *state_file); /** * Frees the memory associated with the sandbox. * * @param lsb Sandbox pointer to discard. - * @param state_file Filename where the sandbox global data is saved. Use a - * NULL or empty string for no preservation. * * @return NULL on success, pointer to an error message on failure that MUST BE * FREED by the caller. */ -LSB_EXPORT char* lsb_destroy(lua_sandbox* lsb, const char* state_file); +LSB_EXPORT char* lsb_destroy(lsb_lua_sandbox *lsb); /** * Retrieve the sandbox usage statistics. @@ -137,8 +142,9 @@ LSB_EXPORT char* lsb_destroy(lua_sandbox* lsb, const char* state_file); * * @return size_t Count or number of bytes depending on the statistic. */ -LSB_EXPORT size_t -lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, lsb_usage_stat ustat); +LSB_EXPORT size_t lsb_usage(lsb_lua_sandbox *lsb, + lsb_usage_type utype, + lsb_usage_stat ustat); /** * Retrieve the current sandbox status. * @@ -146,7 +152,7 @@ lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, lsb_usage_stat ustat); * * @return lsb_state code */ -LSB_EXPORT lsb_state lsb_get_state(lua_sandbox* lsb); +LSB_EXPORT lsb_state lsb_get_state(lsb_lua_sandbox *lsb); /** * Return the last error in human readable form. @@ -155,7 +161,7 @@ LSB_EXPORT lsb_state lsb_get_state(lua_sandbox* lsb); * * @return const char* error message */ -LSB_EXPORT const char* lsb_get_error(lua_sandbox* lsb); +LSB_EXPORT const char* lsb_get_error(lsb_lua_sandbox *lsb); /** * Sets the last error string. @@ -165,7 +171,7 @@ LSB_EXPORT const char* lsb_get_error(lua_sandbox* lsb); * * @return const char* error message */ -LSB_EXPORT void lsb_set_error(lua_sandbox* lsb, const char* err); +LSB_EXPORT void lsb_set_error(lsb_lua_sandbox *lsb, const char *err); /** * Access the Lua pointer. @@ -174,7 +180,7 @@ LSB_EXPORT void lsb_set_error(lua_sandbox* lsb, const char* err); * * @return lua_State* The lua_State pointer. */ -LSB_EXPORT lua_State* lsb_get_lua(lua_sandbox* lsb); +LSB_EXPORT lua_State* lsb_get_lua(lsb_lua_sandbox *lsb); /** * Returns the filename of the Lua source. @@ -183,7 +189,7 @@ LSB_EXPORT lua_State* lsb_get_lua(lua_sandbox* lsb); * * @return const char* filename. */ -LSB_EXPORT const char* lsb_get_lua_file(lua_sandbox* lsb); +LSB_EXPORT const char* lsb_get_lua_file(lsb_lua_sandbox *lsb); /** * Access the parent pointer stored in the sandbox. @@ -192,7 +198,7 @@ LSB_EXPORT const char* lsb_get_lua_file(lua_sandbox* lsb); * * @return void* The parent pointer passed to init. */ -LSB_EXPORT void* lsb_get_parent(lua_sandbox* lsb); +LSB_EXPORT void* lsb_get_parent(lsb_lua_sandbox *lsb); /** * Create a CFunction for use by the Sandbox. The Lua sandbox pointer is pushed @@ -202,20 +208,9 @@ LSB_EXPORT void* lsb_get_parent(lua_sandbox* lsb); * @param func Lua CFunction pointer. * @param func_name Function name exposed to the Lua sandbox. */ -LSB_EXPORT void lsb_add_function(lua_sandbox* lsb, lua_CFunction func, - const char* func_name); - -/** - * Retrieve the data in the output buffer and reset the buffer. The returned - * output string will remain valid until additional sandbox output is performed. - * The output should be copied if the application needs to hold onto it. - * - * @param lsb Pointer to the sandbox. - * @param len If len is not NULL, it will be set to the length of the string. - * - * @return const char* Pointer to the output buffer. - */ -LSB_EXPORT const char* lsb_get_output(lua_sandbox* lsb, size_t* len); +LSB_EXPORT void lsb_add_function(lsb_lua_sandbox *lsb, + lua_CFunction func, + const char *func_name); /** * Helper function to load the Lua function and set the instruction limits @@ -225,14 +220,14 @@ LSB_EXPORT const char* lsb_get_output(lua_sandbox* lsb, size_t* len); * * @return int 0 on success */ -LSB_EXPORT int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name); +LSB_EXPORT int lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name); /** * Helper function to update the statistics after the call * * @param lsb Pointer to the sandbox. */ -LSB_EXPORT void lsb_pcall_teardown(lua_sandbox* lsb); +LSB_EXPORT void lsb_pcall_teardown(lsb_lua_sandbox *lsb); /** * Shutdown the sandbox due to a fatal error. @@ -240,17 +235,10 @@ LSB_EXPORT void lsb_pcall_teardown(lua_sandbox* lsb); * @param lsb Pointer to the sandbox. * @param err Reason for termination */ -LSB_EXPORT void lsb_terminate(lua_sandbox* lsb, const char* err); +LSB_EXPORT void lsb_terminate(lsb_lua_sandbox *lsb, const char *err); -/** - * Deserialize a Heka message protobuf string into a Lua table structure. For - * ease of reuse it is include here (since many of our sandboxes process Heka - * messages). If needed it must be manually added to the sandbox. - * - * @param lua Pointer the Lua state. - * - * @return One on success (table), two on failure (nil, error_message). - */ -LSB_EXPORT int lsb_decode_protobuf(lua_State* lua); +#ifdef __cplusplus +} +#endif #endif diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index c7cf753..9ce473c 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -6,20 +6,25 @@ /** @brief Lua sandbox output buffer functions @file */ -#ifndef lsb_output_h_ -#define lsb_output_h_ +#ifndef luasandbox_output_h_ +#define luasandbox_output_h_ #include #include "luasandbox.h" #include "luasandbox/lua.h" +#include "util/output_buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif /** * Add a output function to the environment table. The environment table must be * on the top of the stack. This function will receive the userdata and - * lsb_output_data struct as pointers on the Lua stack. + * lsb_output_buffer struct as pointers on the Lua stack. * - * lsb_output_data* output = (output_data*)lua_touserdata(lua, -1); + * lsb_output_buffer* output = (output_data*)lua_touserdata(lua, -1); * ud_object* ud = (ud_object*)lua_touserdata(lua, -2); * * @param lua Pointer the Lua state. @@ -27,49 +32,17 @@ * * @return int Zero on success, non-zero on failure. */ -LSB_EXPORT void lsb_add_output_function(lua_State* lua, lua_CFunction fp); - -/** - * Append a character to the output stream. - * - * @param output Pointer the output collector. - * @param ch Character to append to the output. - * - * @return int Zero on success, non-zero if out of memory. - */ -LSB_EXPORT int lsb_appendc(lsb_output_data* output, char ch); - -/** - * Append formatted string to the output stream. - * - * @param output Pointer the output collector. - * @param fmt Printf format specifier. - * - * @return int Zero on success, non-zero if out of memory. - */ -LSB_EXPORT int lsb_appendf(lsb_output_data* output, const char* fmt, ...); - -/** - * Append a fixed string to the output stream. - * - * @param output Pointer the output collector. - * @param str String to append to the output. - * @param len Length of the string to append - * - * @return int Zero on success, non-zero if out of memory. - */ -LSB_EXPORT -int lsb_appends(lsb_output_data* output, const char* str, size_t len); +LSB_EXPORT void lsb_add_output_function(lua_State *lua, lua_CFunction fp); /** - * Resize the output buffer when more space is needed. + * Utility function to retrieve a user data output function * - * @param output Output buffer to resize. - * @param needed Number of additional bytes needed. + * @param lua + * @param index * - * @return int Zero on success, non-zero on failure. + * @return lua_CFunction */ -LSB_EXPORT int lsb_realloc_output(lsb_output_data* output, size_t needed); +LSB_EXPORT lua_CFunction lsb_get_output_function(lua_State *lua, int index); /** * Write an array of variables on the Lua stack to the output buffer. @@ -80,26 +53,23 @@ LSB_EXPORT int lsb_realloc_output(lsb_output_data* output, size_t needed); * @param append 0 to overwrite the output buffer, 1 to append the output to it * */ -LSB_EXPORT void lsb_output(lua_sandbox* lsb, int start, int end, int append); +LSB_EXPORT void +lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append); /** - * Write a Lua table (in a Heka protobuf structure) to the output buffer. + * Retrieve the data in the output buffer and reset the buffer. The returned + * output string will remain valid until additional sandbox output is performed. + * The output should be copied if the application needs to hold onto it. * * @param lsb Pointer to the sandbox. - * @param index Lua stack index of the table. - * @param append 0 to overwrite the output buffer, 1 to append the output to it + * @param len If len is not NULL, it will be set to the length of the string. * - * @return int 0 on success + * @return const char* Pointer to the output buffer. */ -LSB_EXPORT int lsb_output_protobuf(lua_sandbox* lsb, int index, int append); +LSB_EXPORT const char* lsb_get_output(lsb_lua_sandbox *lsb, size_t *len); + +#ifdef __cplusplus +} +#endif -/** - * More efficient output of a double to a string - * - * @param output Pointer the output collector. - * @param d Double value to convert to a string. - * - * @return int Zero on success, non-zero on failure. - */ -LSB_EXPORT int lsb_output_double(lsb_output_data* output, double d); #endif diff --git a/include/luasandbox_serialize.h b/include/luasandbox_serialize.h index f696c9b..e402335 100644 --- a/include/luasandbox_serialize.h +++ b/include/luasandbox_serialize.h @@ -6,41 +6,43 @@ /** Lua sandbox serialization @file */ -#ifndef lsb_serialize_h_ -#define lsb_serialize_h_ +#ifndef luasandbox_serialize_h_ +#define luasandbox_serialize_h_ #include #include "luasandbox/lua.h" #include "luasandbox_output.h" +#ifdef __cplusplus +extern "C" { +#endif + /** * Serialize all user global data to disk. * * @param lsb Pointer to the sandbox. - * @param data_file Filename where the data will be written (create/overwrite) * * @return int Zero on success, non-zero on failure. */ -int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file); +int lsb_preserve_global_data(lsb_lua_sandbox *lsb); /** * Restores previously serialized data from disk. * * @param lsb Pointer to the sandbox. - * @param data_file Filename from where the data will be read. * * @return int Zero on success, non-zero on failure. */ -int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file); +int lsb_restore_global_data(lsb_lua_sandbox *lsb); /** * Add a serialization function to the environment table. The environment table * must be on the top of the stack. This function will receive the userdata, - * fully qualified variable name, and lsb_output_data struct as pointers on the + * fully qualified variable name, and lsb_output_buffer struct as pointers on the * Lua stack. * - * lsb_output_data* output = (output_data*)lua_touserdata(lua, -1); + * lsb_output_buffer* output = (output_data*)lua_touserdata(lua, -1); * const char *key = (const char*)lua_touserdata(lua, -2); * ud_object* ud = (ud_object*)lua_touserdata(lua, -3); * @@ -55,23 +57,27 @@ lsb_add_serialize_function(lua_State *lua, lua_CFunction fp); /** * Serializes a binary data to a Lua string. * + * @param output Pointer the output buffer. * @param src Pointer to the binary data. * @param len Length in bytes of the data to output. - * @param output Pointer the output collector. * * @return int Zero on success, non-zero on failure. */ LSB_EXPORT int -lsb_serialize_binary(const void *src, size_t len, lsb_output_data* output); +lsb_serialize_binary(lsb_output_buffer *output, const void *src, size_t len); /** * More efficient serialization of a double to a string * - * @param output Pointer the output collector. + * @param output Pointer the output buffer. * @param d Double value to convert to a string. * * @return int Zero on success, non-zero on failure. */ -LSB_EXPORT int lsb_serialize_double(lsb_output_data* output, double d); +LSB_EXPORT int lsb_serialize_double(lsb_output_buffer *output, double d); + +#ifdef __cplusplus +} +#endif #endif diff --git a/include/util/heka_message.h b/include/util/heka_message.h new file mode 100644 index 0000000..7fbe098 --- /dev/null +++ b/include/util/heka_message.h @@ -0,0 +1,210 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Heka message representation @file */ + +#ifndef luasandbox_util_heka_message_h_ +#define luasandbox_util_heka_message_h_ + +#include +#include + +#include "luasandbox.h" +#include "util/input_buffer.h" +#include "util/output_buffer.h" +#include "util/string.h" +#include "util/util.h" + +#define LSB_UUID_SIZE 16 +#define LSB_UUID_STR_SIZE 36 +#define LSB_MAX_HDR_SIZE (255 + 3) + +#define LSB_UUID "Uuid" +#define LSB_TIMESTAMP "Timestamp" +#define LSB_TYPE "Type" +#define LSB_LOGGER "Logger" +#define LSB_SEVERITY "Severity" +#define LSB_PAYLOAD "Payload" +#define LSB_ENV_VERSION "EnvVersion" +#define LSB_PID "Pid" +#define LSB_HOSTNAME "Hostname" +#define LSB_FIELDS "Fields" + +typedef enum { + LSB_PB_UUID = 1, + LSB_PB_TIMESTAMP = 2, + LSB_PB_TYPE = 3, + LSB_PB_LOGGER = 4, + LSB_PB_SEVERITY = 5, + LSB_PB_PAYLOAD = 6, + LSB_PB_ENV_VERSION = 7, + LSB_PB_PID = 8, + LSB_PB_HOSTNAME = 9, + LSB_PB_FIELDS = 10 +} lsb_pb_message; + +typedef enum { + LSB_PB_NAME = 1, + LSB_PB_VALUE_TYPE = 2, + LSB_PB_REPRESENTATION = 3, + LSB_PB_VALUE_STRING = 4, + LSB_PB_VALUE_BYTES = 5, + LSB_PB_VALUE_INTEGER = 6, + LSB_PB_VALUE_DOUBLE = 7, + LSB_PB_VALUE_BOOL = 8 +} lsb_pb_field; + +typedef enum { + LSB_PB_STRING = 0, + LSB_PB_BYTES = 1, + LSB_PB_INTEGER = 2, + LSB_PB_DOUBLE = 3, + LSB_PB_BOOL = 4 +} lsb_pb_value_types; + +typedef struct lsb_heka_field +{ + lsb_const_string name; + lsb_const_string representation; + lsb_const_string value; + lsb_pb_value_types value_type; +} lsb_heka_field; + +typedef struct lsb_heka_message +{ + lsb_const_string raw; + lsb_const_string uuid; + lsb_const_string type; + lsb_const_string logger; + lsb_const_string payload; + lsb_const_string env_version; + lsb_const_string hostname; + + lsb_heka_field *fields; + + long long timestamp; + int severity; + int pid; + int fields_len; + int fields_size; +} lsb_heka_message; + +typedef enum { + LSB_READ_NIL, + LSB_READ_NUMERIC, + LSB_READ_STRING, + LSB_READ_BOOL +} lsb_read_type; + +typedef struct { + union + { + lsb_const_string s; + double d; + } u; + lsb_read_type type; +} lsb_read_value; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Zero the structure and allocate memory for at least 'size' fields + * + * @param m Heka message structure + * @param num_fields Preallocated number of fields (must be >0) + * + * @return int 0 on success + * + */ +LSB_EXPORT int lsb_init_heka_message(lsb_heka_message *m, int num_fields); + +/** + * Frees the memory allocated for the message fields + * + * @param m Heka message structure + * + */ +LSB_EXPORT void lsb_free_heka_message(lsb_heka_message *m); + +/** + * Resets the message headers and fields zeroing the allocated memory + * + * @param m Heka message structure + * + */ +LSB_EXPORT void lsb_clear_heka_message(lsb_heka_message *m); + +/** + * Locates a framed Heka message in an input buffer + * + * @param m Heka message structure + * @param ib Input buffer + * @param decode True if the framed message should be protobuf decoded + * @param discarded_bytes Used to track stream corruption + * @param logger Logger call back (can be set to NULL to disable logging) + * + * @return bool True on success + */ +LSB_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, + lsb_input_buffer *ib, + bool decode, + size_t *discarded_bytes, + lsb_logger logger); + +/** + * Decodes an array of bytes into a Heka message. The message structure is + * cleared before decoding. + * + * @param m Heka message structure + * @param buf Protobuf array + * @param len Length of the protobuf array + * @param logger Logger call back (can be set to NULL to disable logging) + * + * @return bool True on success + * + */ +LSB_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, + const char *buf, + size_t len, + lsb_logger logger); + +/** + * Reads a dynamic field from the Heka message + * + * @param m Heka meassage structure + * @param name Field name + * @param fi Field index + * @param ai Array index into the field + * @param val Value structure to be populated by the read + * + * @return bool True on success + */ +LSB_EXPORT bool lsb_read_heka_field(lsb_heka_message *m, + lsb_const_string *name, + int fi, + int ai, + lsb_read_value *val); + +/** + * Writes a binary UUID to the output buffer + * + * @param ob Pointer to the output data buffer. + * @param uuid Uuid string to output (can be NULL, binary, human readable UUID) + * @param len Length of UUID (16 or 36 anything else will cause a new UUID to be + * created). + * + * @return int 0 on success + */ +LSB_EXPORT int +lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/input_buffer.h b/include/util/input_buffer.h new file mode 100644 index 0000000..1153fc7 --- /dev/null +++ b/include/util/input_buffer.h @@ -0,0 +1,67 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream input buffer @file */ + +#ifndef luasandbox_util_input_buffer_h_ +#define luasandbox_util_input_buffer_h_ + +#include +#include + +#include "luasandbox.h" + +typedef struct lsb_input_buffer +{ + char *buf; + size_t size; + size_t maxsize; + size_t readpos; + size_t scanpos; + size_t msglen; +} lsb_input_buffer; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initialize the provided input buffer + * + * @param b Input buffer + * @param max_message_size The maximum message size the buffer will handle + * before erroring (the internal buffer will contain extra space + * for the header) + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_init_input_buffer(lsb_input_buffer *b, + size_t max_message_size); + +/** + * Frees the memory internally allocated by the buffer and resets the state + * + * @param b Input buffer + */ +LSB_EXPORT void lsb_free_input_buffer(lsb_input_buffer *b); + +/** + * Expands the input buffer (if necessary) to accomadate the requested number of + * bytes. The expansion happens in power of two increments up to the maxsize. + * The buffer never shrinks in size. + * + * @param b Input buffer + * @param len The length of the data being added to the buffer + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/output_buffer.h b/include/util/output_buffer.h new file mode 100644 index 0000000..83cceb5 --- /dev/null +++ b/include/util/output_buffer.h @@ -0,0 +1,117 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream output buffer @file */ + +#ifndef luasandbox_util_output_buffer_h_ +#define luasandbox_util_output_buffer_h_ + +#include + +#include "luasandbox.h" + +#define LSB_OUTPUT_SIZE 1024 + +struct lsb_output_buffer { + size_t maxsize; + size_t size; + size_t pos; + char *buf; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initialize the provided input buffer + * + * @param b Output buffer + * @param max_message_size The maximum message size the buffer will handle + * before erroring + * + * @return int 0 on success + */ +LSB_EXPORT +int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); + +/** + * Frees the memory internally allocated by the buffer and resets the state + * + * @param b Output buffer + */ +LSB_EXPORT void lsb_free_output_buffer(lsb_output_buffer *b); + +/** + * Resize the output buffer when more space is needed. + * + * @param b Output buffer to resize. + * @param needed Number of additional bytes needed. + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed); + +/** + * Append a character to the output buffer. + * + * @param b Pointer the b buffer. + * @param ch Character to append to the b. + * + * @return int 0 on success, false if out of memory. + */ +LSB_EXPORT int lsb_outputc(lsb_output_buffer *b, char ch); + +/** + * Append a formatted string to the output buffer. + * + * @param b Pointer the b buffer. + * @param fmt Printf format specifier. + * + * @return int 0 on success, false if out of memory. + */ +LSB_EXPORT int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); + +/** + * Append a fixed string to the output buffer. + * + * @param b Pointer the b buffer. + * @param str String to append to the b. + * @param len Length of the string to append + * + * @return int 0 on success, false if out of memory. + */ +LSB_EXPORT int lsb_outputs(lsb_output_buffer *b, + const char *str, + size_t len); + +/** + * More efficient output of a double to a string. NaN/Inf check and then calls + * lsb_outputfd. + * + * @param output Pointer the output buffer. + * @param d Double value to convert to a string. + * + * @return int 0 on success, false if out of memory. + */ +LSB_EXPORT int lsb_outputd(lsb_output_buffer *output, double d); + + +/** + * More efficient output of a double to a string; no NaN or Inf outputs. + * + * @param output Pointer the output buffer. + * @param d Double value to convert to a string. + * + * @return int 0 on success, false if out of memory. + */ +LSB_EXPORT int lsb_outputfd(lsb_output_buffer *ob, double d); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/protobuf.h b/include/util/protobuf.h new file mode 100644 index 0000000..48f36f2 --- /dev/null +++ b/include/util/protobuf.h @@ -0,0 +1,144 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Generic protobuf utility functions @file */ + +#ifndef luasandbox_util_protobuf_h_ +#define luasandbox_util_protobuf_h_ + +#include "luasandbox.h" + +#include +#include + +#include "util/output_buffer.h" + +#define LSB_MAX_VARINT_BYTES 10 + +typedef enum { + LSB_PB_WT_VARINT = 0, + LSB_PB_WT_FIXED64 = 1, + LSB_PB_WT_LENGTH = 2, + LSB_PB_WT_SGROUP = 3, + LSB_PB_WT_EGROUP = 4, + LSB_PB_WT_FIXED32 = 5 +} lsb_pb_wire_types; + + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Extract the tag and wiretype from a protobuf key + * + * @param p Key + * @param tag Tag Id + * @param wiretype Wiretype Id + * + * @return LSB_EXPORT const char* + */ +LSB_EXPORT +const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype); + +/** + * Writes a field key (tag id/wire type) to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param wiretype Field wire type. + * + * @return int 0 on success + */ +LSB_EXPORT int +lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, + unsigned char wiretype); + +/** + * Reads the varint into the provided variable + * + * @param p Start of buffer + * @param e End of buffer + * @param vi Varint value + * + * @return const char* Position in the buffer after the varint + */ +LSB_EXPORT +const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi); + + +/** + * Outputs the varint to an existing buffer + * + * @param buf Pointer to buffer with at least LSB_MAX_VARINT_BYTES available, + * @param i Number to be encoded. + * + * @return int Number of bytes written + */ +LSB_EXPORT int lsb_pb_output_varint(char* buf, unsigned long long i); + +/** + * Writes a varint encoded number to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param i Number to be encoded. + * + * @return int 0 on success + */ +LSB_EXPORT int +lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i); + +/** + * Writes a bool to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param i Number to be encoded. + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_pb_write_bool(lsb_output_buffer *ob, int i); + +/** + * Writes a double to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param i Double to be encoded. + * + * @return int 0 on success + */ +LSB_EXPORT int lsb_pb_write_double(lsb_output_buffer *ob, double i); + +/** + * Writes a string to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param s String to output. + * @param len Length of s. + * + * @return int 0 on success + */ +LSB_EXPORT int +lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char* s, size_t len); + +/** + * Updates the field length in the output buffer once the size is known, this + * allows for single pass encoding. + * + * @param ob Pointer to the output data buffer. + * @param len_pos Position in the output buffer where the length should be + * written. + * + * @return int 0 on success + */ +LSB_EXPORT int +lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/running_stats.h b/include/util/running_stats.h new file mode 100644 index 0000000..f1bf231 --- /dev/null +++ b/include/util/running_stats.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Calculates the running count, sum, mean, variance, and standard deviation + * @file */ + +#ifndef lsb_util_running_stats_h_ +#define lsb_util_running_stats_h_ + +#include "luasandbox.h" + +typedef struct lsb_running_stats +{ + double count; + double mean; + double sum; +} lsb_running_stats; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Zeros out the stats counters + * + * @param s Stat structure to zero out + */ +LSB_EXPORT void lsb_init_running_stats(lsb_running_stats *s); + +/** + * Value to add to the running stats + * + * @param s Stat structure + * @param d Value to add + */ +LSB_EXPORT void lsb_update_running_stats(lsb_running_stats *s, double d); + +/** + * Return the standard deviation of the stats + * + * @param s Stat structure + * + * @return double Standard deviation of the stats up to this point + */ +LSB_EXPORT double lsb_sd_running_stats(lsb_running_stats *s); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/string.h b/include/util/string.h new file mode 100644 index 0000000..9c2803f --- /dev/null +++ b/include/util/string.h @@ -0,0 +1,37 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Heka message string @file */ + +#ifndef luasandbox_util_string_h_ +#define luasandbox_util_string_h_ + +#include "luasandbox.h" + +typedef struct lsb_const_string +{ + const char *s; + size_t len; +} lsb_const_string; + + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initializes the struct to zero + * + * @param s Pointer to the struct + * + */ +LSB_EXPORT void lsb_init_const_string(lsb_const_string *s); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/string_matcher.h b/include/util/string_matcher.h new file mode 100644 index 0000000..1758e27 --- /dev/null +++ b/include/util/string_matcher.h @@ -0,0 +1,37 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** C API for the Lua string pattern matcher @file */ + +#ifndef luasandbox_util_string_matcher_h_ +#define luasandbox_util_string_matcher_h_ + +#include +#include + +#include "luasandbox.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Matches a string using a Lua string match pattern + * + * @param s String to match + * @param len Length of the string + * @param p Lua match pattern + * http://www.lua.org/manual/5.1/manual.html#pdf-string.match + * + * @return bool True if the sting matches the pattern + */ +LSB_EXPORT bool lsb_string_match(const char* s, size_t len, const char* p); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/util/util.h b/include/util/util.h new file mode 100644 index 0000000..b9da612 --- /dev/null +++ b/include/util/util.h @@ -0,0 +1,47 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Shared types and structures @file */ + +#ifndef luasandbox_util_util_h_ +#define luasandbox_util_util_h_ + +#include + +#include "luasandbox.h" +#include "output_buffer.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Hacker's Delight - Henry S. Warren, Jr. page 48 + * + * @param x + * + * @return size_t Least power of 2 greater than or equal to x + */ +LSB_EXPORT size_t lsb_lp2(unsigned long long x); + + +/** + * Computes the duration between two timespec structures. + * + * @param s Start time + * @param e End time + * + * @return double Time difference in fractional seconds + * + */ +LSB_EXPORT double lsb_timespec_delta(const struct timespec* s, + const struct timespec* e); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a603a26..2b64b9e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,10 +4,8 @@ set(LUA_SANDBOX_SRC luasandbox.c -lsb_output.c -lsb_serialize.c -lsb_serialize_protobuf.c -lsb_deserialize_protobuf.c +luasandbox_output.c +luasandbox_serialize.c ) set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) @@ -43,7 +41,7 @@ endif() set_target_properties(luasandbox PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) -target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) +target_link_libraries(luasandbox luasandboxutil ${LUA_SANDBOX_LIBS}) install(TARGETS luasandbox DESTINATION lib COMPONENT core) install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib COMPONENT core PATTERN "lua" EXCLUDE) @@ -51,4 +49,6 @@ install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION lib/${PROJECT_NAME}/modules install(DIRECTORY "${EP_BASE}/io/lib/lua/" DESTINATION lib/${PROJECT_NAME}/io_modules COMPONENT core) install(DIRECTORY "${EP_BASE}/include/" DESTINATION include COMPONENT core) +add_subdirectory(util) +add_subdirectory(heka_sandbox) add_subdirectory(test) diff --git a/src/heka_sandbox/CMakeLists.txt b/src/heka_sandbox/CMakeLists.txt new file mode 100644 index 0000000..dfd876f --- /dev/null +++ b/src/heka_sandbox/CMakeLists.txt @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set(HEKA_SANDBOX_SRC +message.c +message_matcher.c +sandbox.c +stream_reader.c +) + +add_library(luasandboxheka SHARED ${HEKA_SANDBOX_SRC}) +set_target_properties(luasandboxheka PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_link_libraries(luasandboxheka luasandbox ${LIBM_LIBRARY}) + +install(TARGETS luasandboxheka DESTINATION lib COMPONENT core) +add_subdirectory(test) diff --git a/src/heka_sandbox/message.c b/src/heka_sandbox/message.c new file mode 100644 index 0000000..04abe17 --- /dev/null +++ b/src/heka_sandbox/message.c @@ -0,0 +1,1010 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox Heka protobuf serialization/deserialization @file */ + +#include "message_impl.h" + +#include +#include +#include + +#include "../luasandbox_impl.h" // todo the API should change so this doesn't +// need access to the impl + +#include "luasandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox_output.h" +#include "sandbox_impl.h" +#include "util/heka_message.h" +#include "util/output_buffer.h" +#include "util/protobuf.h" + +/** + * Adds missing headers specified in the configuration to the message output. + * + * @param lua Pointer the Lua state. + * @param idx Lua stack index of the message table. + */ +static void set_missing_headers(lua_State *lua, int idx, lsb_heka_sandbox *hsb) +{ + lua_getfield(lua, idx, LSB_LOGGER); + int t = lua_type(lua, -1); + lua_pop(lua, 1); + if (t == LUA_TNIL && hsb->name) { + lua_pushstring(lua, hsb->name); + lua_setfield(lua, idx, LSB_LOGGER); + } + + lua_getfield(lua, idx, LSB_HOSTNAME); + t = lua_type(lua, -1); + lua_pop(lua, 1); + if (t == LUA_TNIL && hsb->hostname) { + lua_pushstring(lua, hsb->hostname); + lua_setfield(lua, 1, LSB_HOSTNAME); + } +} + + +static const char* read_string(lua_State *lua, + int wiretype, + const char *p, + const char *e) +{ + if (wiretype != LSB_PB_WT_LENGTH) { + return NULL; + } + + long long len = 0; + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || p + len > e) { + return NULL; + } + lua_pushlstring(lua, p, len); + p += len; + return p; +} + + +static const char* process_varint(lua_State *lua, + const char *name, + int wiretype, + int stack_index, + const char *p, + const char *e) +{ + if (wiretype != LSB_PB_WT_VARINT) { + return NULL; + } + long long val = 0; + p = lsb_pb_read_varint(p, e, &val); + if (!p) { + return NULL; + } + lua_pushnumber(lua, val); + lua_setfield(lua, stack_index, name); + return p; +} + + +static const char* process_fields(lua_State *lua, const char *p, const char *e) +{ + int tag = 0; + int wiretype = 0; + int has_name = 0; + int value_count = 0; + long long len = 0; + + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || p + len > e) { + return NULL; + } + e = p + len; // only process to the end of the current field record + + lua_newtable(lua); // Table to be added to the Fields array index 4 + lua_newtable(lua); // Table to hold the value(s) index 5 + do { + p = lsb_pb_read_key(p, &tag, &wiretype); + + switch (tag) { + case 1: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 4, "name"); + has_name = 1; + } + break; + + case 2: + p = process_varint(lua, "value_type", wiretype, 4, p, e); + break; + + case 3: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 4, "representation"); + } + break; + + case 4: // value_string + case 5: // value_bytes + p = read_string(lua, wiretype, p, e); + if (p) { + lua_rawseti(lua, 5, ++value_count); + } + break; + + case 6: // value_integer + { + long long val = 0; + switch (wiretype) { + case 0: + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || p + len > e) { + p = NULL; + break; + } + do { + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushnumber(lua, (lua_Number)val); + lua_rawseti(lua, 5, ++value_count); + } while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + + case 7: // value_double + { + double val = 0; + switch (wiretype) { + case 1: + if (p + sizeof(double) > e) { + p = NULL; + break; + } + memcpy(&val, p, sizeof(double)); + p += sizeof(double); + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || p + len > e || len % sizeof(double) != 0) { + p = NULL; + break; + } + do { + memcpy(&val, p, sizeof(double)); + p += sizeof(double); + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + } while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + + case 8: // value_bool + { + long long val = 0; + switch (wiretype) { + case 0: + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushboolean(lua, (int)val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || p + len > e) { + p = NULL; + break; + } + do { + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushboolean(lua, (int)val); + lua_rawseti(lua, 5, ++value_count); + } while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + default: + p = NULL; // don't allow unknown tags + break; + } + } while (p && p < e); + + lua_setfield(lua, 4, "value"); + + return has_name ? p : NULL; +} + + +/** + * Retrieve the string value for a Lua table entry (the table should be on top + * of the stack). If the entry is not found or not a string nothing is encoded. + * + * @param lsb Pointer to the sandbox. + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return int 0 on success + */ +static int +encode_string(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, + const char *name, int index) +{ + int result = 0; + lua_getfield(lsb->lua, index, name); + if (lua_isstring(lsb->lua, -1)) { + size_t len; + const char *s = lua_tolstring(lsb->lua, -1, &len); + result = lsb_pb_write_string(ob, tag, s, len); + } + lua_pop(lsb->lua, 1); + return result; +} + + +/** + * Retrieve the numeric value for a Lua table entry (the table should be on top + * of the stack). If the entry is not found or not a number nothing is encoded, + * otherwise the number is varint encoded. + * + * @param lsb Pointer to the sandbox. + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return int 0 on success + */ +static int +encode_int(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, + const char *name, int index) +{ + int result = 0; + lua_getfield(lsb->lua, index, name); + if (lua_isnumber(lsb->lua, -1)) { + unsigned long long i = (unsigned long long)lua_tonumber(lsb->lua, -1); + if ((result = lsb_pb_write_key(ob, tag, LSB_PB_WT_VARINT))) { + result = lsb_pb_write_varint(ob, i); + } + } + lua_pop(lsb->lua, 1); + return result; +} + + +/** + * Encodes the field value. + * + * @param lsb Pointer to the sandbox. + * @param ob Pointer to the output data buffer. + * @param first Boolean indicator used to add addition protobuf data in the + * correct order. + * @param representation String representation of the field i.e., "ms" + * @param value_type Protobuf value type + * + * @return int 0 on success + */ +static int +encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, + const char *representation, int value_type); + + +/** + * Encodes a field that has an array of values. + * + * @param lsb Pointer to the sandbox. + * @param ob Pointer to the output data buffer. + * @param t Lua type of the array values. + * @param representation String representation of the field i.e., "ms" + * + * @return int 0 on success + */ +static int +encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, + const char *representation, int value_type) +{ + int result = 0; + int first = (int)lua_objlen(lsb->lua, -1); + int multiple = first > 1 ? first : 0; + size_t len_pos = 0; + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (result == 0 && lua_next(lsb->lua, -2) != 0) { + if (lua_type(lsb->lua, -1) != t) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); + return 3; + } + result = encode_field_value(lsb, ob, first, representation, value_type); + if (first) { + len_pos = ob->pos; + first = 0; + } + lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for + // the next interation. + } + if (multiple && value_type == LSB_PB_INTEGER) { + // fix up the varint packed length + size_t i = len_pos - 2; + int y = 0; + // find the length byte + while (ob->buf[i] != 0 && y < LSB_MAX_VARINT_BYTES) { + --i; + ++y; + } + result = lsb_pb_update_field_length(ob, i); + if (y == LSB_MAX_VARINT_BYTES || result) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "unable set the length of the packed integer array"); + return 1; + } + } + return result; +} + + +/** + * Encodes a field that contains metadata in addition to its value. + * + * @param lsb Pointer to the sandbox. + * @param ob Pointer to the output data buffer. + * + * @return int 0 on success + */ +static int encode_field_object(lsb_lua_sandbox *lsb, lsb_output_buffer *ob) +{ + const char *representation = NULL; + lua_getfield(lsb->lua, -1, "representation"); + if (lua_isstring(lsb->lua, -1)) { + representation = lua_tostring(lsb->lua, -1); + } + + int value_type = -1; + lua_getfield(lsb->lua, -2, "value_type"); + if (lua_isnumber(lsb->lua, -1)) { + value_type = (int)lua_tointeger(lsb->lua, -1); + } + + lua_getfield(lsb->lua, -3, "value"); + int result = encode_field_value(lsb, ob, 1, representation, value_type); + lua_pop(lsb->lua, 3); // remove representation, value_type and value + return result; +} + + +static int +encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, + const char *representation, int value_type) +{ + int result = 0; + size_t len; + const char *s; + + int t = lua_type(lsb->lua, -1); + switch (t) { + case LUA_TSTRING: + switch (value_type) { + case -1: // not specified defaults to string + value_type = 0; + case 0: + case 1: + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid string value_type: %d", value_type); + return 3; + } + if (first) { // this uglyness keeps the protobuf fields in order without + // additional lookups + if (value_type == LSB_PB_BYTES) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (result) return result; + result = lsb_pb_write_varint(ob, value_type); + if (result) return result; + } + if (representation) { + result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (result) return result; + } + } + s = lua_tolstring(lsb->lua, -1, &len); + if (value_type == LSB_PB_BYTES) { + result = lsb_pb_write_string(ob, LSB_PB_VALUE_BYTES, s, len); + } else { + result = lsb_pb_write_string(ob, LSB_PB_VALUE_STRING, s, len); + } + break; + case LUA_TNUMBER: + switch (value_type) { + case -1: // not specified defaults to double + value_type = 3; + case 2: + case 3: + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid numeric value_type: %d", value_type); + return 3; + } + if (first) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (result) return result; + + result = lsb_pb_write_varint(ob, value_type); + if (result) return result; + + if (representation) { + result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (result) return result; + } + if (1 == first) { + if (value_type == LSB_PB_INTEGER) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_VARINT); + if (result) return result; + } else { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_FIXED64); + if (result) return result; + } + } else { // pack array + if (value_type == LSB_PB_INTEGER) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_LENGTH); + if (result) return result; + result = lsb_pb_write_varint(ob, 0); // length tbd later + if (result) return result; + } else { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_LENGTH); + if (result) return result; + result = lsb_pb_write_varint(ob, first * sizeof(double)); + if (result) return result; + } + } + } + if (value_type == LSB_PB_INTEGER) { + result = lsb_pb_write_varint(ob, lua_tointeger(lsb->lua, -1)); + } else { + result = lsb_pb_write_double(ob, lua_tonumber(lsb->lua, -1)); + } + break; + + case LUA_TBOOLEAN: + if (value_type != -1 && value_type != LSB_PB_BOOL) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid boolean value_type: %d", value_type); + return 3; + } + if (first) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (result) return result; + result = lsb_pb_write_varint(ob, LSB_PB_BOOL); + if (result) return result; + if (representation) { + result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (result) return result; + } + if (1 == first) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_VARINT); + if (result) return result; + } else { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_LENGTH); + if (result) return result; + result = lsb_pb_write_varint(ob, first); + if (result) return result; + } + } + result = lsb_pb_write_bool(ob, lua_toboolean(lsb->lua, -1)); + break; + + case LUA_TTABLE: + { + lua_rawgeti(lsb->lua, -1, 1); + int t = lua_type(lsb->lua, -1); + lua_pop(lsb->lua, 1); // remove the array test value + switch (t) { + case LUA_TNIL: + result = encode_field_object(lsb, ob); + break; + case LUA_TNUMBER: + case LUA_TSTRING: + case LUA_TBOOLEAN: + result = encode_field_array(lsb, ob, t, representation, value_type); + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "unsupported array type: %s", lua_typename(lsb->lua, t)); + result = 3; + break; + } + } + break; + + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_output_function(lsb->lua, -1); + size_t len_pos = 0; + if (!fp) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "user data object does not implement lsb_output"); + return 3; + } + if (first) { + result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (result) return result; + // encode userdata as a byte array + result = lsb_pb_write_varint(ob, LSB_PB_BYTES); + if (result) return result; + if (representation) { + result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (result) return result; + } + } + + result = lsb_pb_write_key(ob, LSB_PB_VALUE_BYTES, LSB_PB_WT_LENGTH); + if (result) return result; + len_pos = ob->pos; + result = lsb_pb_write_varint(ob, 0); // length tbd later + if (result) return result; + lua_pushlightuserdata(lsb->lua, ob); + if (fp(lsb->lua)) return 3; + lua_pop(lsb->lua, 1); // remove output function + result = lsb_pb_update_field_length(ob, len_pos); + } + break; + + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", + lua_typename(lsb->lua, t)); + result = 3; + break; + } + return result; +} + + +/** + * Iterates over the specified Lua table encoding the contents as user defined + * message fields. + * + * @param lsb Pointer to the sandbox. + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return int 0 on success + */ +static int +encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, + const char *name, int index) +{ + int result = 0; + lua_getfield(lsb->lua, index, name); + if (!lua_istable(lsb->lua, -1)) { + return result; + } + + lua_rawgeti(lsb->lua, -1, 1); // test for the array notation + size_t len_pos, len; + if (lua_istable(lsb->lua, -1)) { + int i = 1; + do { + result = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (result) return result; + len_pos = ob->pos; + result = lsb_pb_write_varint(ob, 0); // length tbd later + if (result) return result; + lua_getfield(lsb->lua, -1, "name"); + if (lua_isstring(lsb->lua, -1)) { + const char *s = lua_tolstring(lsb->lua, -1, &len); + result = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); + if (result) return result; + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "field name must be a string"); + return 3; + } + lua_pop(lsb->lua, 1); // remove the name + + result = encode_field_object(lsb, ob); + if (result) return result; + result = lsb_pb_update_field_length(ob, len_pos); + if (result) return result; + + lua_pop(lsb->lua, 1); // remove the current field object + lua_rawgeti(lsb->lua, -1, ++i); // grab the next field object + } while (!lua_isnil(lsb->lua, -1)); + } else { + lua_pop(lsb->lua, 1); // remove the array test value + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (result == 0 && lua_next(lsb->lua, -2) != 0) { + result = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (result) return result; + len_pos = ob->pos; + result = lsb_pb_write_varint(ob, 0); // length tbd later + if (result) return result; + if (lua_isstring(lsb->lua, -2)) { + const char *s = lua_tolstring(lsb->lua, -2, &len); + result = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); + if (result) return result; + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "field name must be a string"); + return 3; + } + result = encode_field_value(lsb, ob, 1, NULL, -1); + if (result) return result; + result = lsb_pb_update_field_length(ob, len_pos); + if (result) return result; + lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for + // the next interation. + } + } + lua_pop(lsb->lua, 1); // remove the fields table + return result; +} + + +int heka_decode_message(lua_State *lua) +{ + int n = lua_gettop(lua); + if (n != 1 || lua_type(lua, 1) != LUA_TSTRING) { + return luaL_argerror(lua, 0, "must have one string argument"); + } + + size_t len; + const char *pbstr = lua_tolstring(lua, 1, &len); + if (len < 20) { + return luaL_error(lua, "invalid message, too short"); + } + + const char *p = pbstr; + const char *lp = p; + const char *e = pbstr + len; + int wiretype = 0; + int tag = 0; + int has_uuid = 0; + int has_timestamp = 0; + int field_count = 0; + + lua_newtable(lua); // message table index 2 + do { + p = lsb_pb_read_key(p, &tag, &wiretype); + + switch (tag) { + case 1: + p = read_string(lua, wiretype, p, e); + if (p && p - lp == 18) { + lua_setfield(lua, 2, "Uuid"); + has_uuid = 1; + } else { + p = NULL; + } + break; + + case 2: + p = process_varint(lua, "Timestamp", wiretype, 2, p, e); + if (p) { + has_timestamp = 1; + } + break; + + case 3: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Type"); + } + break; + + case 4: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Logger"); + } + break; + + case 5: + p = process_varint(lua, "Severity", wiretype, 2, p, e); + break; + + case 6: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Payload"); + } + break; + + case 7: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "EnvVersion"); + } + break; + + case 8: + p = process_varint(lua, "Pid", wiretype, 2, p, e); + break; + + case 9: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Hostname"); + } + break; + + case 10: + if (wiretype != 2) { + p = NULL; + break; + } + if (field_count == 0) { + lua_newtable(lua); // Fields table index 3 + } + p = process_fields(lua, p, e); + if (p) { + lua_rawseti(lua, 3, ++field_count); + } + break; + + default: + p = NULL; // don't allow unknown tags + break; + } + if (p) lp = p; + } while (p && p < e); + + if (!p) { + return luaL_error(lua, "error in tag: %d wiretype: %d offset: %d", tag, + wiretype, (const char *)lp - pbstr); + } + + if (!(has_uuid && has_timestamp)) { + return luaL_error(lua, "missing required field uuid: %s timestamp: %s", + has_uuid ? "found" : "not found", + has_timestamp ? "found" : "not found"); + } + + if (field_count) { + lua_setfield(lua, 2, "Fields"); + } + + return 1; +} + + +int heka_encode_message(lua_State *lua) +{ + int n = lua_gettop(lua); + bool framed = false; + + switch (n) { + case 2: + luaL_checktype(lua, 2, LUA_TBOOLEAN); + framed = lua_toboolean(lua, 2); + // fall thru + case 1: + luaL_checktype(lua, 1, LUA_TTABLE); + break; + default: + return luaL_argerror(lua, n, "incorrect number of arguments"); + } + + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "encode_message() invalid upvalueindex"); + } + + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + set_missing_headers(lua, 1, hsb); + + lsb->output.pos = 0; + if (heka_encode_message_table(lsb, 1)) { + const char *err = lsb_get_error(lsb); + if (strlen(err) == 0) err = "exceeded output_limit"; + return luaL_error(lua, "encode_message() failed: %s", err); + } + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = lsb->output.pos; + if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] + > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + } + + size_t len = 0; + const char *output = lsb_get_output(lsb, &len); + + if (framed) { + char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f + int hlen = lsb_pb_output_varint(header + 3, len) + 1; + header[1] = (char)hlen; + header[hlen + 2] = '\x1f'; + luaL_Buffer b; + luaL_buffinit(lua, &b); + luaL_addlstring(&b, (char *)header, hlen + 3); + luaL_addlstring(&b, output, len); + luaL_pushresult(&b); + } else { + lua_pushlstring(lua, output, len); + } + return 1; +} + + +int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) +{ + int result = 0; + lsb_output_buffer *ob = &lsb->output; + ob->pos = 0; + + // use existing or create a type 4 uuid + lua_getfield(lsb->lua, idx, LSB_UUID); + size_t len; + const char *uuid = lua_tolstring(lsb->lua, -1, &len); + result = lsb_write_heka_uuid(ob, uuid, len); + if (result) return result; + lua_pop(lsb->lua, 1); // remove uuid + + // use existing or create a timestamp + lua_getfield(lsb->lua, idx, LSB_TIMESTAMP); + long long ts; + if (lua_isnumber(lsb->lua, -1)) { + ts = lua_tonumber(lsb->lua, -1); + } else { + ts = time(NULL) * 1000000000LL; + } + lua_pop(lsb->lua, 1); // remove timestamp + + result = lsb_pb_write_key(ob, LSB_PB_TIMESTAMP, LSB_PB_WT_VARINT); + if (result) return result; + result = lsb_pb_write_varint(ob, ts); + if (result) return result; + result = encode_string(lsb, ob, LSB_PB_TYPE, LSB_TYPE, idx); + if (result) return result; + result = encode_string(lsb, ob, LSB_PB_LOGGER, LSB_LOGGER, idx); + if (result) return result; + result = encode_int(lsb, ob, LSB_PB_SEVERITY, LSB_SEVERITY, idx); + if (result) return result; + result = encode_string(lsb, ob, LSB_PB_PAYLOAD, LSB_PAYLOAD, idx); + if (result) return result; + result = encode_string(lsb, ob, LSB_PB_ENV_VERSION, LSB_ENV_VERSION, idx); + if (result) return result; + result = encode_int(lsb, ob, LSB_PB_PID, LSB_PID, idx); + if (result) return result; + result = encode_string(lsb, ob, LSB_PB_HOSTNAME, LSB_HOSTNAME, idx); + if (result) return result; + result = encode_fields(lsb, ob, LSB_PB_FIELDS, LSB_FIELDS, idx); + if (result) return result; + result = lsb_expand_output_buffer(ob, 1); + if (result) return result; + ob->buf[ob->pos] = 0; // NULL terminate incase someone tries to treat this + // as a string + return result; +} + + +int heka_read_message(lua_State *lua, lsb_heka_message *m) +{ + int n = lua_gettop(lua); + if (n < 1 || n > 3) { + return luaL_error(lua, "read_message() incorrect number of arguments"); + } + size_t field_len; + const char *field = luaL_checklstring(lua, 1, &field_len); + int fi = luaL_optinteger(lua, 2, 0); + luaL_argcheck(lua, fi >= 0, 2, "field index must be >= 0"); + int ai = luaL_optinteger(lua, 3, 0); + luaL_argcheck(lua, ai >= 0, 3, "array index must be >= 0"); + + if (strcmp(field, LSB_UUID) == 0) { + if (m->uuid.s) { + lua_pushlstring(lua, m->uuid.s, m->uuid.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_TIMESTAMP) == 0) { + lua_pushnumber(lua, m->timestamp); + } else if (strcmp(field, LSB_TYPE) == 0) { + if (m->type.s) { + lua_pushlstring(lua, m->type.s, m->type.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_LOGGER) == 0) { + if (m->logger.s) { + lua_pushlstring(lua, m->logger.s, m->logger.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_SEVERITY) == 0) { + lua_pushinteger(lua, m->severity); + } else if (strcmp(field, LSB_PAYLOAD) == 0) { + if (m->payload.s) { + lua_pushlstring(lua, m->payload.s, m->payload.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_ENV_VERSION) == 0) { + if (m->env_version.s) { + lua_pushlstring(lua, m->env_version.s, m->env_version.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_PID) == 0) { + lua_pushinteger(lua, m->pid); + } else if (strcmp(field, LSB_HOSTNAME) == 0) { + if (m->hostname.s) { + lua_pushlstring(lua, m->hostname.s, m->hostname.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, "raw") == 0) { + lua_pushlstring(lua, m->raw.s, m->raw.len); + } else if (strcmp(field, "framed") == 0) { + { + char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f + int hlen = lsb_pb_output_varint(header + 3, m->raw.len) + 1; + header[1] = (char)hlen; + header[hlen + 2] = '\x1f'; + luaL_Buffer b; + luaL_buffinit(lua, &b); + luaL_addlstring(&b, header, hlen + 3); + luaL_addlstring(&b, m->raw.s, m->raw.len); + luaL_pushresult(&b); + } + } else { + if (field_len >= 8 + && memcmp(field, LSB_FIELDS "[", 7) == 0 + && field[field_len - 1] == ']') { + lsb_read_value v; + lsb_const_string f = { .s = field + 7, .len = field_len - 8 }; + lsb_read_heka_field(m, &f, fi, ai, &v); + switch (v.type) { + case LSB_READ_STRING: + lua_pushlstring(lua, v.u.s.s, v.u.s.len); + break; + case LSB_READ_NUMERIC: + lua_pushnumber(lua, v.u.d); + break; + case LSB_READ_BOOL: + lua_pushboolean(lua, v.u.d); + break; + default: + lua_pushnil(lua); + break; + } + } else { + lua_pushnil(lua); + } + } + return 1; +} diff --git a/src/heka_sandbox/message_impl.h b/src/heka_sandbox/message_impl.h new file mode 100644 index 0000000..6998fa3 --- /dev/null +++ b/src/heka_sandbox/message_impl.h @@ -0,0 +1,64 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher @file */ + +#ifndef luasandbox_heka_sandbox_message_impl_h_ +#define luasandbox_heka_sandbox_message_impl_h_ + +#include + +#include "luasandbox.h" +#include "util/heka_message.h" + +// these functions are intentionally not exported + +/** + * Deserialize a Heka message protobuf string into a Lua table structure. + * + * @param lua Pointer the Lua state. + * + * @return int Number of items on the stack (1 table) or throws an error on + * failure + */ +int heka_decode_message(lua_State *lua); + +/** + * Serialize a Lua table structure into a Heka message protobuf string. + * + * @param lua Pointer the Lua state. + * + * @return int Number of items on the stack (1 string) or throws an error on + * failure + */ +int heka_encode_message(lua_State *lua); + +/** + * Serialize a Lua table structure into a Heka message protobuf (using the + * sandbox output buffer, Called indirectly from inject_message so the output + * buffer can be used without round tripping the resulting data back to the + * sandbox with heka_encode_message. + * + * @param lsb Pointer to the sandbox. + * @param idx Lua stack index of the message table. + * + * @return int 0 on success + */ +int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx); + + +/** + * Breakout of the common code for the read_message API + * + * @param lua Pointer to the lua_State + * @param m Heka message to extract the data from + * + * @return int Number of items on the stack (1 value) or throws an error on + * failure + */ +int heka_read_message(lua_State *lua, lsb_heka_message *m); + +#endif diff --git a/src/heka_sandbox/message_matcher.c b/src/heka_sandbox/message_matcher.c new file mode 100644 index 0000000..682b423 --- /dev/null +++ b/src/heka_sandbox/message_matcher.c @@ -0,0 +1,588 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight/Heka message matcher implementation @file */ + +#include "heka_sandbox/message_matcher.h" + +#include +#include +#include + +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox/lualib.h" +#include "util/string_matcher.h" + +static const char *grammar = + "local l = require 'lpeg'\n" + "local dt = require 'date_time'\n" + "l.locale(l)\n" + "\n" + "local sp = l.space^0\n" + "local eqne = l.Cg(l.P'==' + '!=', 'op')\n" + "local string_match = l.Cg(l.P'=~' + '!~', 'op')\n" + "local relational = l.Cg(eqne + '>=' + '>' + '<=' + '<', 'op')\n" + "local op_and = l.C(l.P'&&')\n" + "local op_or = l.C(l.P'||')\n" + "\n" + "local string_vars = l.Cg(l.P'" LSB_TYPE + "' + '" LSB_LOGGER + "' + '" LSB_HOSTNAME + "' + '" LSB_ENV_VERSION + "' + '" LSB_PAYLOAD + "' + '" LSB_UUID + "', 'variable')\n" + "local numeric_vars = l.Cg(l.P'" LSB_SEVERITY + "' + '" LSB_PID + "', 'variable')\n" + "local boolean = l.Cg(l.P'TRUE' / function() return true end + l.P'FALSE' / function() return false end, 'value')\n" + "local null = l.P'NIL'\n" + "local index = l.P'[' * (l.digit^1 / tonumber) * l.P']' + l.Cc'0' / tonumber\n" + "local fields = l.P'Fields[' * l.Cg((l.P(1) - ']')^0, 'variable') * ']' * l.Cg(index, 'fi') * l.Cg(index, 'ai')\n" + "\n" + "local string_value = l.Cg(l.P'\"' * l.Cs(((l.P(1) - l.S'\\\\\"') + l.P'\\\\\"' / '\"')^0) * '\"'\n" + " + l.P'\\'' * l.Cs(((l.P(1) - l.S'\\\\\\'') + l.P'\\\\\\'' / '\\'')^0) * '\\'', 'value')\n" + "local string_test = l.Ct(string_vars * sp * (relational * sp * string_value + string_match * sp * string_value))\n" + "\n" + "local sign = l.S'+-'^-1\n" + "local number = l.P'0' + l.R'19' * l.digit^0\n" + "local decimal = (l.P'.'^-1 * l.digit^1)^-1\n" + "local exponent = (l.S'eE' * sign * l.digit^1)^-1\n" + "local numeric_value = l.Cg((sign * number * decimal * exponent) / tonumber, 'value')\n" + "local numeric_test = l.Ct(numeric_vars * sp * relational * sp * numeric_value)\n" + "\n" + "local ts_value = l.Cg(dt.rfc3339 / dt.time_to_ns, 'value')\n" + "local ts_quoted = l.P'\"' * ts_value * '\"' + l.P\"'\" * ts_value * \"'\"\n" + "local ts_test = l.Ct(l.Cg(l.P'Timestamp', 'variable') * sp * relational * sp * (numeric_value + ts_quoted))\n" + "\n" + "local field_test = l.Ct(fields * sp * (relational * sp * (string_value + numeric_value)\n" + " + string_match * sp * string_value\n" + " + eqne * sp * (boolean + null)))\n" + "\n" + "local Exp, Term, Test = l.V'Exp', l.V'Term', l.V'Test'\n" + "g = l.P{\n" + "Exp,\n" + "Exp = l.Ct(Term * (op_or * sp * Term)^0);\n" + "Term = l.Ct(Test * (op_and * sp * Test)^0) * sp;\n" + "Test = (string_test + ts_test + numeric_test + field_test + l.Ct(l.Cg(boolean, 'op'))) * sp + l.P'(' * Exp * ')' * sp;\n" + "}\n" + "\n" + "local function postfix(t, output, stack, stack_ptr)\n" + " local sp = stack_ptr\n" + " for j,v in ipairs(t) do\n" + " if type(v) == 'string' then\n" + " stack_ptr = stack_ptr + 1\n" + " stack[stack_ptr] = v\n" + " elseif type(v) == 'table' then\n" + " if v.op then\n" + " output[#output+1] = v\n" + " else\n" + " postfix(v, output, stack, stack_ptr)\n" + " end\n" + " end\n" + " end\n" + " for i=stack_ptr, sp+1, -1 do\n" + " output[#output+1] = stack[i]\n" + " end\n" + " stack_ptr = sp\n" + " return output\n" + "end\n" + "\n" + "local grammar = l.Ct(sp * g * -1)\n" + "\n" + "function parse(s)\n" + " local output = {}\n" + " local stack = {}\n" + " local stack_ptr = 0\n" + " local t = grammar:match(s)\n" + " local len = 0\n" + " if t then\n" + " t = postfix(t, output, stack, stack_ptr)\n" + " len = #t\n" + " end\n" + " return t, len -- node array and length\n" + "end\n"; + + +typedef enum { + OP_EQ, + OP_NE, + OP_GTE, + OP_GT, + OP_LTE, + OP_LT, + OP_RE, + OP_NRE, + OP_TRUE, + OP_FALSE, + OP_OR, + OP_AND +} match_operation; + + +typedef enum { + TYPE_NIL, + TYPE_STRING, + TYPE_NUMERIC, + TYPE_BOOLEAN, + TYPE_STRING_MATCH +} match_type; + + +typedef struct match_node { + struct match_node *left; + struct match_node *right; + + char *variable; + size_t variable_len; + + union { + char *s; + double d; + } value; + size_t value_len; + match_type value_type; + + match_operation op; + int id; + int fi; + int ai; +} match_node; + + +struct lsb_message_match_builder { + lua_State *parser; +}; + + +struct lsb_message_matcher { + int nodes_size; + match_node nodes[]; +}; + + +static bool string_test(match_node *mn, lsb_const_string *val) +{ + switch (mn->op) { + case OP_EQ: + if (val->len != mn->value_len) return false; + return strncmp(val->s, mn->value.s, val->len) == 0; + case OP_NE: + if (val->len != mn->value_len) return true; + return strncmp(val->s, mn->value.s, val->len) != 0; + case OP_LT: + { + int cmp = strncmp(val->s, mn->value.s, val->len); + if (cmp == 0) { + return val->len < mn->value_len; + } + return cmp < 0; + } + case OP_LTE: + return strncmp(val->s, mn->value.s, val->len) <= 0; + case OP_GT: + { + int cmp = strncmp(val->s, mn->value.s, val->len); + if (cmp == 0) { + return val->len > mn->value_len; + } + return cmp > 0; + } + case OP_GTE: + { + int cmp = strncmp(val->s, mn->value.s, val->len); + if (cmp == 0) { + return val->len == mn->value_len; + } + return cmp > 0; + } + case OP_RE: + return lsb_string_match(val->s, val->len, mn->value.s); + case OP_NRE: + return !lsb_string_match(val->s, val->len, mn->value.s); + default: + break; + } + return false; +} + + +static bool numeric_test(match_node *mn, double val) +{ + switch (mn->op) { + case OP_EQ: + return val == mn->value.d; + case OP_NE: + return val != mn->value.d; + case OP_LT: + return val < mn->value.d; + case OP_LTE: + return val <= mn->value.d; + case OP_GT: + return val > mn->value.d; + case OP_GTE: + return val >= mn->value.d; + default: + break; + } + return false; +} + + +static bool eval_node(match_node *mn, lsb_heka_message *m) +{ + switch (mn->op) { + case OP_TRUE: + return true; + case OP_FALSE: + return false; + default: + switch (mn->id) { + case LSB_PB_TIMESTAMP: + return numeric_test(mn, m->timestamp); + case LSB_PB_TYPE: + return string_test(mn, &m->type); + case LSB_PB_LOGGER: + return string_test(mn, &m->logger); + case LSB_PB_SEVERITY: + return numeric_test(mn, m->severity); + case LSB_PB_PAYLOAD: + return string_test(mn, &m->payload); + case LSB_PB_ENV_VERSION: + return string_test(mn, &m->env_version); + case LSB_PB_PID: + return numeric_test(mn, m->pid); + case LSB_PB_HOSTNAME: + return string_test(mn, &m->hostname); + case LSB_PB_UUID: + return string_test(mn, &m->uuid); + default: + { + lsb_read_value val; + lsb_const_string variable = { .s = mn->variable, + .len = mn->variable_len }; + + switch (mn->value_type) { + case TYPE_STRING: + case TYPE_STRING_MATCH: + if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) + && val.type == LSB_READ_STRING) { + return string_test(mn, &val.u.s); + } + break; + case TYPE_NUMERIC: + case TYPE_BOOLEAN: + if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) + && (val.type == LSB_READ_NUMERIC || val.type == LSB_READ_BOOL)) { + return numeric_test(mn, val.u.d); + } + break; + case TYPE_NIL: + if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val)) { + return mn->op == OP_NE; + } + return mn->op == OP_EQ; + } + } + break; + } + break; + } + return false; +} + + +static bool eval_tree(match_node *mn, lsb_heka_message *m) +{ + bool match; + if (mn->left) { + match = eval_tree(mn->left, m); + } else { + match = eval_node(mn, m); + } + + if (match && mn->op == OP_OR) { + return match; // short circuit + } + + if (!match && mn->op == OP_AND) { + return match; // short circuit + } + + if (mn->right) { + match = eval_tree(mn->right, m); + } + return match; +} + + +static void load_op_node(lua_State *L, match_node *mn) +{ + const char *op = lua_tostring(L, -1); + if (strcmp(op, "||") == 0) { + mn->op = OP_OR; + } else if (strcmp(op, "&&") == 0) { + mn->op = OP_AND; + } else { + fprintf(stderr, "message_matcher unknown op: %s", op); + exit(EXIT_FAILURE); + } +} + + +static void load_expression_node(lua_State *L, match_node *mn) +{ + size_t len; + const char *tmp; + lua_getfield(L, -1, "op"); + tmp = lua_tolstring(L, -1, &len); + if (strcmp(tmp, "==") == 0) { + mn->op = OP_EQ; + } else if (strcmp(tmp, "!=") == 0) { + mn->op = OP_NE; + } else if (strcmp(tmp, ">=") == 0) { + mn->op = OP_GTE; + } else if (strcmp(tmp, ">") == 0) { + mn->op = OP_GT; + } else if (strcmp(tmp, "<=") == 0) { + mn->op = OP_LTE; + } else if (strcmp(tmp, "<") == 0) { + mn->op = OP_LT; + } else if (strcmp(tmp, "=~") == 0) { + mn->value_type = TYPE_STRING_MATCH; + mn->op = OP_RE; + } else if (strcmp(tmp, "!~") == 0) { + mn->value_type = TYPE_STRING_MATCH; + mn->op = OP_NRE; + } else if (strcmp(tmp, "TRUE") == 0) { + mn->op = OP_TRUE; + lua_pop(L, 1); + return; // no other args + } else if (strcmp(tmp, "FALSE") == 0) { + mn->op = OP_FALSE; // no other args + lua_pop(L, 1); + return; + } else { + fprintf(stderr, "message_matcher invalid op: %s", tmp); + exit(EXIT_FAILURE); + } + lua_pop(L, 1); + + lua_getfield(L, -1, "fi"); + if (lua_type(L, -1) == LUA_TNUMBER) { + mn->id = LSB_PB_FIELDS; + } + mn->fi = lua_tointeger(L, -1); + lua_pop(L, 1); + + lua_getfield(L, -1, "ai"); + mn->ai = lua_tointeger(L, -1); + lua_pop(L, 1); + + lua_getfield(L, -1, "variable"); + tmp = lua_tolstring(L, -1, &len); + if (mn->id == LSB_PB_FIELDS) { + mn->variable = malloc(len + 1); + mn->variable_len = len; + memcpy(mn->variable, tmp, len + 1); + } else { + if (strcmp(tmp, LSB_TIMESTAMP) == 0) { + mn->id = LSB_PB_TIMESTAMP; + } else if (strcmp(tmp, LSB_TYPE) == 0) { + mn->id = LSB_PB_TYPE; + } else if (strcmp(tmp, LSB_LOGGER) == 0) { + mn->id = LSB_PB_LOGGER; + } else if (strcmp(tmp, LSB_SEVERITY) == 0) { + mn->id = LSB_PB_SEVERITY; + } else if (strcmp(tmp, LSB_PAYLOAD) == 0) { + mn->id = LSB_PB_PAYLOAD; + } else if (strcmp(tmp, LSB_ENV_VERSION) == 0) { + mn->id = LSB_PB_ENV_VERSION; + } else if (strcmp(tmp, LSB_PID) == 0) { + mn->id = LSB_PB_PID; + } else if (strcmp(tmp, LSB_HOSTNAME) == 0) { + mn->id = LSB_PB_HOSTNAME; + } else if (strcmp(tmp, LSB_UUID) == 0) { + mn->id = LSB_PB_UUID; + } else { + fprintf(stderr, "broken message_matcher grammar: invalid header"); + exit(EXIT_FAILURE); + } + } + lua_pop(L, 1); + + lua_getfield(L, -1, "value"); + switch (lua_type(L, -1)) { + case LUA_TSTRING: + tmp = lua_tolstring(L, -1, &len); + mn->value_type = TYPE_STRING; + mn->value_len = len; + mn->value.s = malloc(len + 1); + memcpy(mn->value.s, tmp, len + 1); + break; + case LUA_TNUMBER: + mn->value_type = TYPE_NUMERIC; + mn->value.d = lua_tonumber(L, -1); + break; + case LUA_TBOOLEAN: + mn->value_type = TYPE_BOOLEAN; + mn->value.d = lua_toboolean(L, -1); + break; + case LUA_TNIL: + mn->value_type = TYPE_NIL; + break; + default: + fprintf(stderr, "message_matcher invalid value"); + exit(EXIT_FAILURE); + } + lua_pop(L, 1); +} + + +lsb_message_match_builder* +lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath) +{ + +#if _WIN32 + if (_putenv("TZ=UTC") != 0) { + return NULL; + } +#else + if (setenv("TZ", "UTC", 1) != 0) { + return NULL; + } +#endif + + lsb_message_match_builder *mmb = malloc(sizeof *mmb); + mmb->parser = luaL_newstate(); + if (!mmb->parser) { + free(mmb); + return NULL; + } + + lua_createtable(mmb->parser, 0, 1); + lua_pushstring(mmb->parser, lua_path); + lua_setfield(mmb->parser, -2, "path"); + lua_pushstring(mmb->parser, lua_cpath); + lua_setfield(mmb->parser, -2, "cpath"); + lua_setfield(mmb->parser, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + + luaL_openlibs(mmb->parser); + + if (luaL_dostring(mmb->parser, grammar)) { + free(mmb); + return NULL; + } + return mmb; +} + + +void lsb_destroy_message_match_builder(lsb_message_match_builder *mmb) +{ + if (mmb->parser) { + lua_close(mmb->parser); + mmb->parser = NULL; + } + free(mmb); +} + + +lsb_message_matcher* +lsb_create_message_matcher(const lsb_message_match_builder *mmb, + const char *exp) +{ + lua_getglobal(mmb->parser, "parse"); + if (!lua_isfunction(mmb->parser, -1)) { + // todo logger fprintf(stderr, "message_matcher error: %s", + // lua_tostring(mmb->parser, -1)); + return NULL; + } + lua_pushstring(mmb->parser, exp); + if (lua_pcall(mmb->parser, 1, 2, 0)) { + // todo logger fprintf(stderr, "message_matcher error: %s", + // lua_tostring(mmb->parser, -1)); + return NULL; + } + + if (lua_type(mmb->parser, 1) != LUA_TTABLE) { + // todo logger fprintf(stderr, "parse failed"); + return NULL; + } + int size = lua_tointeger(mmb->parser, 2); + lsb_message_matcher *mm = calloc(sizeof(lsb_message_matcher) + + (sizeof(match_node) * size), 1); + if (!mm) { + // todo logger fprintf(stderr, "calloc failed"); + return NULL; + } + mm->nodes_size = size; + + // load in reverse order so the root node will be first + for (int i = size, j = 0; i > 0; --i, ++j) { + lua_rawgeti(mmb->parser, 1, i); + switch (lua_type(mmb->parser, -1)) { + case LUA_TSTRING: + load_op_node(mmb->parser, &mm->nodes[j]); + break; + case LUA_TTABLE: + load_expression_node(mmb->parser, &mm->nodes[j]); + break; + default: + free(mm); + //todo logeer fprintf(stderr, + //"message_matcher error: invalid table returned"); + return NULL; + } + lua_pop(mmb->parser, 1); + } + lua_pop(mmb->parser, 2); + + // turn the postfix stack into an executable tree + match_node **stack = calloc(sizeof(match_node *) * size, 1); + if (!stack) { + free(mm); + // todo logger fprintf(stderr, "message_matcher stack allocation failed"); + return NULL; + } + + int top = 0; + for (int i = size - 1; i >= 0; --i) { + if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { + stack[top++] = &mm->nodes[i]; + } else { + mm->nodes[i].right = stack[--top]; + mm->nodes[i].left = stack[--top]; + stack[top++] = &mm->nodes[i]; + } + } + free(stack); + return mm; +} + + +void lsb_destroy_message_matcher(lsb_message_matcher *mm) +{ + for (int i = 0; i < mm->nodes_size; ++i) { + free(mm->nodes[i].variable); + switch (mm->nodes[i].value_type) { + case TYPE_STRING: + case TYPE_STRING_MATCH: + free(mm->nodes[i].value.s); + break; + default: + // no action required + break; + } + } + free(mm); +} + + +bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m) +{ + return eval_tree(mm->nodes, m); +} diff --git a/src/heka_sandbox/sandbox.c b/src/heka_sandbox/sandbox.c new file mode 100644 index 0000000..7db094b --- /dev/null +++ b/src/heka_sandbox/sandbox.c @@ -0,0 +1,735 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Heka sandbox implementation @file */ + +#include "heka_sandbox/sandbox.h" + +#include +#include +#include + +#include "heka_sandbox/message_matcher.h" +#include "luasandbox.h" +#include "luasandbox_output.h" +#include "message_impl.h" +#include "sandbox_impl.h" +#include "stream_reader_impl.h" +#include "util/protobuf.h" +#include "util/running_stats.h" + +#ifdef _WIN32 +#include +#else +#include +#endif + +static const char *pm_func_name = "process_message"; +static const char *im_func_name = "inject_message"; + +static int read_message(lua_State *lua) +{ + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == lsb) { + return luaL_error(lua, "read_message() invalid lightuserdata"); + } + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + + if (!hsb->msg || !hsb->msg->raw.s) { + lua_pushnil(lua); + return 1; + } + return heka_read_message(lua, hsb->msg); +} + + +static int inject_message_input(lua_State *lua) +{ + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == lsb) { + return luaL_error(lua, "%s() invalid lightuserdata", im_func_name); + } + + const char *scp = NULL; + double ncp = NAN; + int t = lua_type(lua, 2); + switch (t) { + case LUA_TNUMBER: + ncp = lua_tonumber(lua, 2); + break; + case LUA_TSTRING: + scp = lua_tostring(lua, 2); + break; + case LUA_TNONE: + case LUA_TNIL: + break; + default: + return luaL_error(lua, "%s() unsupported checkpoint type: %s", im_func_name, + lua_typename(lua, t)); + } + + lsb_const_string output; + t = lua_type(lua, 1); + switch (t) { + case LUA_TUSERDATA: + { + heka_stream_reader *hsr = luaL_checkudata(lua, 1, + mozsvc_heka_stream_reader); + if (hsr->msg.raw.s) { + output.len = hsr->msg.raw.len; + output.s = hsr->msg.raw.s; + } else { + return luaL_error(lua, "%s() attempted to inject a nil message", + im_func_name); + } + } + break; + case LUA_TSTRING: + { + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + output.s = lua_tolstring(lua, 1, &output.len); + bool ok = lsb_decode_heka_message(&m, output.s, output.len, NULL); + lsb_free_heka_message(&m); + if (!ok) { + return luaL_error(lua, "%s() attempted to inject a invalid protobuf " + "string", im_func_name); + } + } + break; + case LUA_TTABLE: + if (heka_encode_message_table(lsb, 1)) { + const char *err = lsb_get_error(lsb); + if (strlen(err) == 0) err = "exceeded output_limit"; + return luaL_error(lua, "%s() failed: %s", im_func_name, err); + } + output.len = 0; + output.s = lsb_get_output(lsb, &output.len); + break; + default: + return luaL_error(lua, "%s() unsupported message type: %s", + im_func_name, lua_typename(lua, t)); + } + + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + if (hsb->cb.iim(hsb->parent, output.s, output.len, ncp, scp) != 0) { + return luaL_error(lua, "%s() failed: rejected by the callback", + im_func_name); + } + return 0; +} + + +static int inject_message_analysis(lua_State *lua) +{ + luaL_checktype(lua, 1, LUA_TTABLE); + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == lsb) { + return luaL_error(lua, "%s() invalid lightuserdata", im_func_name); + } + + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + lua_pushstring(lua, hsb->name); + lua_setfield(lua, 1, LSB_LOGGER); + lua_pushstring(lua, hsb->hostname); + lua_setfield(lua, 1, LSB_HOSTNAME); + + if (heka_encode_message_table(lsb, 1)) { + return luaL_error(lua, "%s() failed: %s", im_func_name, lsb_get_error(lsb)); + } + + size_t output_len = 0; + const char *output = lsb_get_output(lsb, &output_len); + if (hsb->cb.aim(hsb->parent, output, output_len) != 0) { + return luaL_error(lua, "%s() failed: rejected by the callback", + im_func_name); + } + return 0; +} + + +static int inject_payload(lua_State *lua) +{ + static const char *default_type = "txt"; + + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "%s invalid lightuserdata", __FUNCTION__); + } + + int n = lua_gettop(lua); + if (n > 2) { + lsb_output(lsb, 3, n, 1); + lua_pop(lua, n - 2); + } + size_t len = 0; + const char *output = lsb_get_output(lsb, &len); + if (!len) return 0; + + if (n > 0) { + if (lua_type(lua, 1) != LUA_TSTRING) { + return luaL_error(lua, "%s() payload_type argument must be a string", + __FUNCTION__); + } + } + + if (n > 1) { + if (lua_type(lua, 2) != LUA_TSTRING) { + return luaL_error(lua, "%s() payload_name argument must be a string", + __FUNCTION__); + } + } + + // build up a heka message table + lua_createtable(lua, 0, 2); // message + lua_createtable(lua, 0, 2); // Fields + if (n > 0) { + lua_pushvalue(lua, 1); + } else { + lua_pushstring(lua, default_type); + } + lua_setfield(lua, -2, "payload_type"); + + if (n > 1) { + lua_pushvalue(lua, 2); + lua_setfield(lua, -2, "payload_name"); + } + lua_setfield(lua, -2, LSB_FIELDS); + lua_pushstring(lua, "inject_payload"); + lua_setfield(lua, -2, LSB_TYPE); + lua_pushlstring(lua, output, len); + lua_setfield(lua, -2, LSB_PAYLOAD); + if (lua_gettop(lua) > 1) lua_replace(lua, 1); + + inject_message_analysis(lua); + return 0; +} + + +static int update_checkpoint(lua_State *lua) +{ + static const char *func_name = "update_checkpoint"; + + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "%s() invalid upvalueindex", func_name); + } + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + int result = 0; + int n = lua_gettop(lua); + switch (n) { + case 2: // async case + luaL_checktype(lua, 2, LUA_TNUMBER); + hsb->stats.pm_failures += lua_tonumber(lua, 2); + // fall thru + case 1: + luaL_checktype(lua, 1, LUA_TLIGHTUSERDATA); + result = hsb->cb.ucp(hsb->parent, lua_touserdata(lua, 1)); + break; + case 0: // batch case + result = hsb->cb.ucp(hsb->parent, NULL); + break; + default: + return luaL_error(lua, "%s() invalid number of args: %d", func_name, n); + } + if (result) { + return luaL_error(lua, "%s() failed: rejected by the callback", func_name); + } + return result; +} + + +static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) +{ + static const char *io[] = { + NULL, "", "dofile", "load", "loadfile", "loadstring", "newproxy", "print", + NULL, "os", "getenv", "exit", "setlocale", + NULL, "string", "dump" + }; + + static const char *analysis[] = { + NULL, "", "collectgarbage", "coroutine", "dofile", "load", "loadfile", + "loadstring", "newproxy", "print", + NULL, "os", "getenv", "execute", "exit", "remove", "rename", "setlocale", + "tmpname", + NULL, "string", "dump" + }; + + const char **list; + size_t list_size; + + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + switch (hsb->type) { + case 'i': + case 'o': + list_size = sizeof io / sizeof*io; + list = io; + break; + default: + list_size = sizeof analysis / sizeof*analysis; + list = analysis; + lua_newtable(lua); + lua_pushboolean(lua, true); + lua_setfield(lua, -2, "io"); + lua_setfield(lua, 1, "disable_modules"); + break; + } + + lua_newtable(lua); + for (unsigned i = 0, j = 1; i < list_size; ++i) { + if (list[i]) { + lua_pushstring(lua, list[i]); + lua_rawseti(lua, -2, j++); + } else { + lua_newtable(lua); + lua_setfield(lua, -2, list[++i]); + j = 1; + } + } + lua_setfield(lua, 1, "remove_entries"); + + lua_getfield(lua, 1, LSB_HEKA_MAX_MESSAGE_SIZE); + if (lua_type(lua, -1) != LUA_TNUMBER || lua_tointeger(lua, -1) <= 0) { + lua_pushnumber(lua, 64 * 1024); + lua_setfield(lua, 1, LSB_HEKA_MAX_MESSAGE_SIZE); + } + lua_pop(lua, 1); // remove max_message_size + + lua_getfield(lua, 1, LSB_LOGGER); + size_t len; + const char *tmp = lua_tolstring(lua, -1, &len); + if (tmp) { + hsb->name = malloc(len + 1); + if (hsb->name) strcpy(hsb->name, tmp); + } + lua_pop(lua, 1); // remove the Logger + + lua_getfield(lua, 1, LSB_HOSTNAME); + tmp = lua_tolstring(lua, -1, &len); + if (tmp) { + hsb->hostname = malloc(len + 1); + if (hsb->hostname) strcpy(hsb->hostname, tmp); + } else { + char hostname[256] = { 0 }; + if (gethostname(hostname, sizeof hostname)) { + hostname[sizeof hostname - 1] = 0; + } + len = strlen(hostname); + hsb->hostname = malloc(len + 1); + if (hsb->hostname) strcpy(hsb->hostname, hostname); + } + lua_pop(lua, 1); // remove the Hostname + + lua_pop(lua, 1); // remove the lsb_config table +} + + +lsb_heka_sandbox* lsb_heka_create_input(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger logger, + lsb_heka_inject_message_input im) +{ + if (!lua_file) { + if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + return NULL; + } + + if (!im) { + if (logger) logger(__FUNCTION__, 3, "inject_message callback must be " + "specified"); + return NULL; + } + + lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); + if (!hsb) { + if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + return NULL; + } + + hsb->type = 'i'; + hsb->parent = parent; + hsb->msg = NULL; + hsb->cb.iim = im; + hsb->name = NULL; + hsb->hostname = NULL; + + hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); + if (!hsb->lsb) { + free(hsb); + return NULL; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + set_restrictions(lua, hsb); + +// todo link print to the logger or a no-op + lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); + lsb_add_function(hsb->lsb, read_message, "read_message"); + lsb_add_function(hsb->lsb, inject_message_input, "inject_message"); +// inject_payload is intentionally excluded from input plugins +// you can construct whatever you need with inject_message + +// preload the Heka stream reader module + luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); + lua_pushstring(lua, mozsvc_heka_stream_reader_table); + lua_pushcfunction(lua, luaopen_heka_stream_reader); + lua_rawset(lua, -3); + lua_pop(lua, 1); // remove the preloaded table + + if (lsb_init(hsb->lsb, state_file)) { + if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return NULL; + } + +// remove output function + lua_pushnil(lua); + lua_setglobal(lua, "output"); + + return hsb; +} + + +static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, + lua_State *lua, int nargs, bool profile) +{ + struct timespec ts, ts1; + + hsb->msg = msg; + if (profile) { + clock_gettime(CLOCK_MONOTONIC, &ts); // todo fix on Mac & Win + } + if (lua_pcall(lua, nargs, 2, 0) != 0) { + char err[LSB_ERROR_SIZE]; + size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", pm_func_name, + lua_tostring(lua, -1)); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + if (profile) { + clock_gettime(CLOCK_MONOTONIC, &ts); // todo fix on Mac & Win + lsb_update_running_stats(&hsb->stats.pm, lsb_timespec_delta(&ts1, &ts)); + } + hsb->msg = NULL; + + if (lua_type(lua, 1) != LUA_TNUMBER) { + char err[LSB_ERROR_SIZE]; + size_t len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a numeric status code", + pm_func_name); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + + int status = (int)lua_tointeger(lua, 1); + switch (lua_type(lua, 2)) { + case LUA_TNONE: + case LUA_TNIL: + lsb_set_error(hsb->lsb, NULL); + break; + case LUA_TSTRING: + lsb_set_error(hsb->lsb, lua_tostring(lua, 2)); + break; + default: + { + char err[LSB_ERROR_SIZE]; + int len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a nil or string error message", + pm_func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + break; + } + lua_pop(lua, 2); + lsb_pcall_teardown(hsb->lsb); + + if (status > 0) { + char err[LSB_ERROR_SIZE]; + size_t len = snprintf(err, LSB_ERROR_SIZE, + "%s() received a termination status code", + pm_func_name); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + return status; +} + + +int lsb_heka_process_message_input(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + double cp_numeric, + const char *cp_string, + bool profile) +{ + if (!hsb || hsb->type != 'i') return 1; + + if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", pm_func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + + if (!isnan(cp_numeric)) { + lua_pushnumber(lua, cp_numeric); + } else if (cp_string) { + lua_pushstring(lua, cp_string); + } else { + lua_pushnil(lua); + } + return process_message(hsb, msg, lua, 1, profile); +} + + +lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger logger, + lsb_heka_inject_message_analysis im) +{ + if (!lua_file) { + if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + return NULL; + } + + if (!im) { + if (logger) logger(__FUNCTION__, 3, "inject_message callback must be " + "specified"); + return NULL; + } + + + lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); + if (!hsb) { + if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + return NULL; + } + + hsb->type = 'a'; + hsb->parent = parent; + hsb->msg = NULL; + hsb->cb.aim = im; + hsb->name = NULL; + hsb->hostname = NULL; + + hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); + if (!hsb->lsb) { + free(hsb); + return NULL; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + set_restrictions(lua, hsb); + +// todo link print to the logger or a no-op + lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); + lsb_add_function(hsb->lsb, read_message, "read_message"); + lsb_add_function(hsb->lsb, inject_message_analysis, "inject_message"); + lsb_add_function(hsb->lsb, inject_payload, "inject_payload"); +// rename output to add_to_payload + lua_getglobal(lua, "output"); + lua_setglobal(lua, "add_to_payload"); + lua_pushnil(lua); + lua_setglobal(lua, "output"); + + if (lsb_init(hsb->lsb, state_file)) { + if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return NULL; + } + return hsb; +} + + +int lsb_heka_process_message_analysis(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + bool profile) +{ + if (!hsb || hsb->type != 'a') return 1; + + if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", pm_func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + return process_message(hsb, msg, lua, 0, profile); +} + + +lsb_heka_sandbox* lsb_heka_create_output(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger logger, + lsb_heka_update_checkpoint ucp) +{ + if (!lua_file) { + if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + return NULL; + } + + if (!ucp) { + if (logger) logger(__FUNCTION__, 3, "update_checkpoint callback must be " + "specified"); + return NULL; + } + + + lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); + if (!hsb) { + if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + return NULL; + } + + hsb->type = 'o'; + hsb->parent = parent; + hsb->msg = NULL; + hsb->cb.ucp = ucp; + hsb->name = NULL; + hsb->hostname = NULL; + + hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); + if (!hsb->lsb) { + free(hsb); + return NULL; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + set_restrictions(lua, hsb); + +// todo link print to the logger or a no-op + lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); + lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); + lsb_add_function(hsb->lsb, update_checkpoint, "update_checkpoint"); + + if (lsb_init(hsb->lsb, state_file)) { + if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return NULL; + } + +// remove output function + lua_pushnil(lua); + lua_setglobal(lua, "output"); + + return hsb; +} + + +void lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) +{ + if (!hsb) return; + + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); +} + + +int lsb_heka_process_message_output(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + void *sequence_id, + bool profile) +{ + if (!hsb || hsb->type != 'o') return 1; + + if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", pm_func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + + int nargs = 0; + if (sequence_id) { + nargs = 1; + lua_pushlightuserdata(lua, sequence_id); + } + + return process_message(hsb, msg, lua, nargs, profile); +} + + +int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) +{ + static const char *func_name = "timer_event"; + + if (!hsb || (hsb->type != 'o' && hsb->type != 'a')) return 1; + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + + if (lsb_pcall_setup(hsb->lsb, func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + // todo change if we need more than 1 sec resolution + lua_pushnumber(lua, t * 1e9); + lua_pushboolean(lua, shutdown); + + struct timespec ts, ts1; + clock_gettime(CLOCK_MONOTONIC, &ts); // todo fix on Mac & Win + if (lua_pcall(lua, 2, 0, 0) != 0) { + char err[LSB_ERROR_SIZE]; + size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, + lua_tostring(lua, -1)); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + clock_gettime(CLOCK_MONOTONIC, &ts1); // todo fix on Mac & Win + lsb_update_running_stats(&hsb->stats.te, lsb_timespec_delta(&ts, &ts1)); + lsb_pcall_teardown(hsb->lsb); + lua_gc(lua, LUA_GCCOLLECT, 0); + return 0; +} + + +const char* lsb_heka_get_error(lsb_heka_sandbox *hsb) +{ + if (!hsb) return ""; + return lsb_get_error(hsb->lsb); +} diff --git a/src/heka_sandbox/sandbox_impl.h b/src/heka_sandbox/sandbox_impl.h new file mode 100644 index 0000000..23abebf --- /dev/null +++ b/src/heka_sandbox/sandbox_impl.h @@ -0,0 +1,44 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight Heka sandbox private implementation @file */ + +#ifndef luasandbox_heka_sandbox_sandbox_impl_h_ +#define luasandbox_heka_sandbox_sandbox_impl_h_ + +#include "luasandbox.h" +#include "heka_sandbox/sandbox.h" +#include "util/heka_message.h" +#include "util/running_stats.h" + +struct heka_stats { + long long im_cnt; + long long im_bytes; + + long long pm_cnt; + long long pm_failures; + + lsb_running_stats pm; + lsb_running_stats te; +}; + + +struct lsb_heka_sandbox { + void *parent; + lsb_lua_sandbox *lsb; + lsb_heka_message *msg; + char *name; + char *hostname; + union { + lsb_heka_inject_message_input iim; + lsb_heka_inject_message_analysis aim; + lsb_heka_update_checkpoint ucp; + } cb; + char type; + struct heka_stats stats; +}; + +#endif diff --git a/src/heka_sandbox/stream_reader.c b/src/heka_sandbox/stream_reader.c new file mode 100644 index 0000000..40de720 --- /dev/null +++ b/src/heka_sandbox/stream_reader.c @@ -0,0 +1,242 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight Heka stream reader implementation @file */ + +#include "stream_reader_impl.h" + +#include +#include + +#include "heka_sandbox/sandbox.h" +#include "message_impl.h" +#include "luasandbox/lauxlib.h" + +const char* mozsvc_heka_stream_reader = "mozsvc.heka_stream_reader"; +const char *mozsvc_heka_stream_reader_table = "heka_stream_reader"; + +static int hsr_new(lua_State* lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n == 1, 0, "incorrect number of arguments"); + size_t len; + const char* name = luaL_checklstring(lua, 1, &len); + luaL_argcheck(lua, len < 255, 1, "name is too long"); + + size_t nbytes = sizeof(heka_stream_reader); + heka_stream_reader* hsr = lua_newuserdata(lua, nbytes); + + size_t mms = 0; + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + lua_getfield(lua, -1, LSB_HEKA_MAX_MESSAGE_SIZE); + mms = (size_t)lua_tointeger(lua, -1); + lua_pop(lua, 1); // remove limit + } else { + free(hsr); + return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); + } + lua_pop(lua, 1); // remove config + + if (lsb_init_heka_message(&hsr->msg, 8)) { + free(hsr); + return luaL_error(lua, "failed to init the message struct"); + } + if (lsb_init_input_buffer(&hsr->buf, mms)) { + lsb_free_heka_message(&hsr->msg); + free(hsr); + return luaL_error(lua, "failed to init the input buffer"); + } + hsr->name = malloc(len + 1); + if (!hsr->name) { + lsb_free_input_buffer(&hsr->buf); + lsb_free_heka_message(&hsr->msg); + free(hsr); + return luaL_error(lua, "memory allocation failed"); + } + strcpy(hsr->name, name); + + luaL_getmetatable(lua, mozsvc_heka_stream_reader); + lua_setmetatable(lua, -2); + return 1; +} + + +static heka_stream_reader* check_hsr(lua_State* lua, int args) +{ + heka_stream_reader* hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); + luaL_argcheck(lua, args == lua_gettop(lua), 0, + "incorrect number of arguments"); + return hsr; +} + + +static int hsr_decode_message(lua_State* lua) +{ + heka_stream_reader* hsr = check_hsr(lua, 2); + lsb_input_buffer* b = &hsr->buf; + b->readpos = b->scanpos = b->msglen = 0; + + size_t len; + if (lua_type(lua, 2) == LUA_TSTRING) { + const char* s = lua_tolstring(lua, 2, &len); + if (len > 0) { + if (lsb_expand_input_buffer(b, len)) { + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); + } + memcpy(b->buf, s, len); + } else { + return luaL_error(lua, "empty protobuf string"); + } + } else { + return luaL_error(lua, "buffer must be string"); + } + + if (!lsb_decode_heka_message(&hsr->msg, b->buf, len, NULL)) { + return luaL_error(lua, "invalid protobuf string"); + } + return 0; +} + + +static int hsr_find_message(lua_State* lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n > 1 && n < 4, 0,"incorrect number of arguments"); + + heka_stream_reader* hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); + lsb_input_buffer* b = &hsr->buf; + FILE* fh = NULL; + + switch (lua_type(lua, 2)) { + case LUA_TNIL: // scan the existing buffer + break; + case LUA_TSTRING: // add data to the buffer + { + size_t len; + const char* s = lua_tolstring(lua, 2, &len); + if (len > 0) { + if (lsb_expand_input_buffer(b, len)) { + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); + } + memcpy(b->buf + b->readpos, s, len); + b->readpos += len; + } + } + break; + case LUA_TUSERDATA: // add data from the provided file handle to the buffer + fh = *(FILE**)luaL_checkudata(lua, 2, "FILE*"); + if (!fh) luaL_error(lua, "attempt to use a closed file"); + break; + default: + return luaL_error(lua, "buffer must be a nil, string, userdata (FILE*)"); + } + + bool decode = true; + if (n == 3 && lua_type(lua, 3) == LUA_TBOOLEAN) { + decode = (bool)lua_toboolean(lua, 3); + } + + size_t pos_r = b->readpos; + size_t pos_s = b->scanpos; + size_t discarded = 0; + bool found = lsb_find_heka_message(&hsr->msg, b, decode, &discarded, NULL); + + if (found) { + lua_pushboolean(lua, 1); // found + lua_pushinteger(lua, b->scanpos - pos_s); // consumed + } else { + lua_pushboolean(lua, 0); // not found + if (b->readpos == 0) { + lua_pushinteger(lua, pos_r - pos_s); // consumed everything in the buf + } else { + lua_pushinteger(lua, b->scanpos - pos_s); + } + } + + if (fh) { // update bytes read + if (found) { + lua_pushinteger(lua, 0); + } else { + if (lsb_expand_input_buffer(&hsr->buf, 0)) { + luaL_error(lua, "%s buffer reallocation failed", hsr->name); + } + size_t nread = fread(hsr->buf.buf + hsr->buf.readpos, + 1, + hsr->buf.size - hsr->buf.readpos, + fh); + hsr->buf.readpos += nread; + lua_pushnumber(lua, nread); + } + } else { // update bytes needed + if (found) { + if (b->scanpos != b->readpos) { + lua_pushinteger(lua, 0); + } else { + lua_pushinteger(lua, b->size); + } + } else { + if (b->msglen + LSB_MAX_HDR_SIZE > b->size) { + lua_pushinteger(lua, b->msglen); + } else { + lua_pushinteger(lua, b->scanpos + b->size - b->readpos); + } + } + } + return 3; +} + + +static int hsr_read_message(lua_State* lua) +{ + int n = lua_gettop(lua); + if (n < 1 || n > 4) { + return luaL_error(lua, "read_message() incorrect number of arguments"); + } + heka_stream_reader* hsr = check_hsr(lua, n); + lua_remove(lua, 1); // remove the hsr user data + return heka_read_message(lua, &hsr->msg); +} + + +static int hsr_gc(lua_State* lua) +{ + heka_stream_reader* hsr = check_hsr(lua, 1); + free(hsr->name); + lsb_free_heka_message(&hsr->msg); + lsb_free_input_buffer(&hsr->buf); + return 1; +} + + +static const struct luaL_reg heka_stream_readerlib_f[] = +{ + { "new", hsr_new } + , { NULL, NULL } +}; + + +static const struct luaL_reg heka_stream_readerlib_m[] = +{ + { "find_message", hsr_find_message } + , { "decode_message", hsr_decode_message } + , { "read_message", hsr_read_message } + , { "__gc", hsr_gc } + , { NULL, NULL } +}; + + +int luaopen_heka_stream_reader(lua_State* lua) +{ + luaL_newmetatable(lua, mozsvc_heka_stream_reader); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, heka_stream_readerlib_m); + luaL_register(lua, mozsvc_heka_stream_reader_table, heka_stream_readerlib_f); + return 1; +} diff --git a/src/heka_sandbox/stream_reader_impl.h b/src/heka_sandbox/stream_reader_impl.h new file mode 100644 index 0000000..2ed37c9 --- /dev/null +++ b/src/heka_sandbox/stream_reader_impl.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight Heka stream reader structures @file */ + +#ifndef luasandbox_heka_sandbox_stream_reader_h_ +#define luasandbox_heka_sandbox_stream_reader_h_ + +#include "luasandbox/lua.h" + +#include "util/input_buffer.h" +#include "util/heka_message.h" + +extern const char *mozsvc_heka_stream_reader; +extern const char *mozsvc_heka_stream_reader_table; + +typedef struct heka_stream_reader +{ + char *name; + lsb_heka_message msg; + lsb_input_buffer buf; +} heka_stream_reader; + +int luaopen_heka_stream_reader(lua_State *lua); + +#endif diff --git a/src/heka_sandbox/test/CMakeLists.txt b/src/heka_sandbox/test/CMakeLists.txt new file mode 100644 index 0000000..baaa6a1 --- /dev/null +++ b/src/heka_sandbox/test/CMakeLists.txt @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +configure_file(test.h.in test.h ESCAPE_QUOTES) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) + +add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +add_executable(test_heka_sandbox test_sandbox.c) +target_link_libraries(test_heka_sandbox luasandboxheka luasandboxutil luasandbox luasb) +add_test(NAME test_heka_sandbox COMMAND test_heka_sandbox) + +add_executable(test_message_matcher test_message_matcher.c) +target_link_libraries(test_message_matcher luasandboxheka luasandboxutil luasb) +add_test(NAME test_message_matcher COMMAND test_message_matcher) diff --git a/src/heka_sandbox/test/lua/aim.lua b/src/heka_sandbox/test/lua/aim.lua new file mode 100644 index 0000000..eac1d27 --- /dev/null +++ b/src/heka_sandbox/test/lua/aim.lua @@ -0,0 +1,40 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +-- Table tests +local msgs = { + {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, +} + +local err_msgs = { + {err = "bad argument #1 to '?' (table expected, got nil)"}, +} + +for i, v in ipairs(msgs) do + inject_message(v) +end + +for i, v in ipairs(err_msgs) do + local ok, err = pcall(inject_message, v.msg) + if ok then error(string.format("test: %d should have failed", i)) end + assert(v.err == err, string.format("test: %d expected: %s received: %s", i, v.err, err)) +end + +add_to_payload("foo bar") +inject_payload() + +add_to_payload("foo") +inject_payload("dat", "test", " bar") + +local ok, err = pcall(add_to_payload, add_to_payload) +if ok then error("cannot output functions") end +local eerr = "bad argument #1 to '?' (unsupported type)" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_payload, "txt", "name", add_to_payload) +if ok then error("cannot output functions") end +eerr = "bad argument #3 to '?' (unsupported type)" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) diff --git a/src/heka_sandbox/test/lua/analysis.lua b/src/heka_sandbox/test/lua/analysis.lua new file mode 100644 index 0000000..26fa72c --- /dev/null +++ b/src/heka_sandbox/test/lua/analysis.lua @@ -0,0 +1,19 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +function process_message() + return 0 +end + +function timer_event(ns, shutdown) + if ns == 1e9 then + assert(shutdown, "not shutting down") + return + end + + if ns == 2e9 then error("boom") end + + if shutdown then error("should not have a shutdown signal") end +end + diff --git a/src/test/lua/decode_message.lua b/src/heka_sandbox/test/lua/decode_message.lua similarity index 97% rename from src/test/lua/decode_message.lua rename to src/heka_sandbox/test/lua/decode_message.lua index 0595730..65bcea2 100644 --- a/src/test/lua/decode_message.lua +++ b/src/heka_sandbox/test/lua/decode_message.lua @@ -1,3 +1,7 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + -- minimal message, required fields only local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\000" local msg = decode_message(test) diff --git a/src/test/lua/decode_message_benchmark.lua b/src/heka_sandbox/test/lua/decode_message_benchmark.lua similarity index 75% rename from src/test/lua/decode_message_benchmark.lua rename to src/heka_sandbox/test/lua/decode_message_benchmark.lua index fc22337..c69da71 100644 --- a/src/test/lua/decode_message_benchmark.lua +++ b/src/heka_sandbox/test/lua/decode_message_benchmark.lua @@ -1,6 +1,10 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + local test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" -function process(tc) +function process_message() local msg = decode_message(test) return 0 end diff --git a/src/heka_sandbox/test/lua/encode_message.lua b/src/heka_sandbox/test/lua/encode_message.lua new file mode 100644 index 0000000..0cc6f7e --- /dev/null +++ b/src/heka_sandbox/test/lua/encode_message.lua @@ -0,0 +1,43 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" +require "table" + +local msgs = { + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl\074\002\sh" + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h"}, + rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\001l\074\001\h" + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + rv = "\030\002\008\028\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl\074\002\sh", + framed = true, + }, +} + +for i, v in ipairs(msgs) do + local rv + if v.framed then + rv = encode_message(v.msg, v.framed) + else + rv = encode_message(v.msg) + end + + if v.rv ~= rv then + local et = {string.byte(v.rv, 1, -1)} + local rt = {string.byte(rv, 1, -1)} + assert(v.rv == rv, string.format("test: %d\nexpected: %s\nreceived: %s", i, table.concat(et, " "), table.concat(rt, " "))) + end + +end + +local ok, err = pcall(encode_message, "foo") +if ok then error("encode_message should not accept a string") end +local eerr = "bad argument #1 to '?' (table expected, got string)" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) diff --git a/src/heka_sandbox/test/lua/iim.lua b/src/heka_sandbox/test/lua/iim.lua new file mode 100644 index 0000000..836a652 --- /dev/null +++ b/src/heka_sandbox/test/lua/iim.lua @@ -0,0 +1,68 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" +require "bloom_filter"; +local bf = bloom_filter.new(10, 0.01) + +-- Table tests +local msgs = { + {{Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, nil}, + {{Timestamp = 1, Uuid = "00000000-0000-0000-0000-000000000000"}, nil}, + {{Timestamp = 2, Uuid = "00000000-0000-0000-0000-000000000000", Logger = "logger", Hostname = "hostname", Type = "type", Payload = "payload", EnvVersion = "envversion", Pid = 99, Severity = 5}, 99}, + {{Timestamp = 3, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}}, "foo.log:123"}, +} + +local err_msgs = { + {msg = {Timestamp = 1e9, Fields = {counts={2,"ten",4}}}, err = "inject_message() failed: array has mixed types"}, + {msg = {Timestamp = 1e9, Fields = {counts={}}}, err = "inject_message() failed: unsupported type: nil"}, + {err = "inject_message() unsupported message type: nil"}, + {msg = {Timestamp = 1e9, Fields = {counts={{1},{2}}}}, err = "inject_message() failed: unsupported array type: table"}, + {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=2}}}, err = "inject_message() failed: invalid string value_type: 2"}, + {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=3}}}, err = "inject_message() failed: invalid string value_type: 3"}, + {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=4}}}, err = "inject_message() failed: invalid string value_type: 4"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=1, value_type=0}}}, err = "inject_message() failed: invalid numeric value_type: 0"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=1, value_type=1}}}, err = "inject_message() failed: invalid numeric value_type: 1"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=1, value_type=4}}}, err = "inject_message() failed: invalid numeric value_type: 4"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=0}}}, err = "inject_message() failed: invalid boolean value_type: 0"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=1}}}, err = "inject_message() failed: invalid boolean value_type: 1"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, + {msg = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}}, err = "inject_message() failed: user data object does not implement lsb_output"}, +} + +for i, v in ipairs(msgs) do + inject_message(v[1], v[2]) +end + +for i, v in ipairs(err_msgs) do + local ok, err = pcall(inject_message, v.msg) + if ok then error(string.format("test: %d should have failed", i)) end + assert(v.err == err, string.format("test: %d expected: %s received: %s", i, v.err, err)) +end + +-- Stream Reader tests +require "heka_stream_reader" +local hsr = heka_stream_reader.new("test") +ok, err = pcall(inject_message, hsr) +local eerr = "inject_message() attempted to inject a nil message" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +hsr:decode_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\004") +inject_message(hsr) + +-- String tests +inject_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\005") + +ok, err = pcall(inject_message, "\000") +if ok then error(string.format("string test should have failed")) end +local eerr = "inject_message() attempted to inject a invalid protobuf string" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_message, {}) +if ok then error(string.format("test should have failed")) end +eerr = "inject_message() failed: rejected by the callback" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + + diff --git a/src/heka_sandbox/test/lua/input.lua b/src/heka_sandbox/test/lua/input.lua new file mode 100644 index 0000000..e027b35 --- /dev/null +++ b/src/heka_sandbox/test/lua/input.lua @@ -0,0 +1,18 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local msg = {Timestamp = 8} + +function process_message(cp) + if cp == 0 then + return -2, "host specific failure" + elseif cp == 1 then + return -1, "failed" + elseif cp == 2 then + return 0, "ok" + elseif cp == "string" then + return 0, "string" + end + return 0, "no cp" +end diff --git a/src/heka_sandbox/test/lua/output.lua b/src/heka_sandbox/test/lua/output.lua new file mode 100644 index 0000000..edb5eba --- /dev/null +++ b/src/heka_sandbox/test/lua/output.lua @@ -0,0 +1,17 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +function process_message(sequence_id) + if not sequence_id then + update_checkpoint() + return 0 + else + update_checkpoint(sequence_id, 7) + return -5 + end +end + +function timer_event(ns) +end + diff --git a/src/heka_sandbox/test/lua/pm_no_return.lua b/src/heka_sandbox/test/lua/pm_no_return.lua new file mode 100644 index 0000000..aa5c3dc --- /dev/null +++ b/src/heka_sandbox/test/lua/pm_no_return.lua @@ -0,0 +1,7 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +function process_message() +end + diff --git a/src/heka_sandbox/test/lua/read_message.lua b/src/heka_sandbox/test/lua/read_message.lua new file mode 100644 index 0000000..1133bf4 --- /dev/null +++ b/src/heka_sandbox/test/lua/read_message.lua @@ -0,0 +1,54 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +local tests = { + {"Timestamp", 1e9}, + {"Uuid", "abcdefghijklmnop"}, + {"Type", "type"}, + {"Logger", "logger"}, + {"Payload", "payload"}, + {"EnvVersion", "env_version"}, + {"Hostname", "hostname"}, + {"Severity", 9}, + {"Pid", 0}, + {"raw", 208}, + {"framed", 214}, + {"Fields[string]", "string"}, + {"Fields[notfound]", nil}, + {"Fields[string", nil}, + {"morethan8", nil}, + {"lt8", nil}, +} + +local fields = { + {"number" , 0 , 0 , 1}, + {"numbers" , 0 , 2 , 3}, + {"bool" , 0 , 0 , true}, + {"bools" , 0 , 1 , false}, + {"strings" , 0 , 0 , "s1"}, + {"strings" , 0 , 2 , "s3"}, + {"strings" , 10, 0 , nil}, + {"strings" , 0 , 10, nil}, + {"notfound" , 0 , 0 , nil}, +} + + +function process_message() + for i, v in ipairs(tests) do + local r = read_message(v[1]) + if v[1] == "raw" or v[1] == "framed" then + assert(v[2] == string.len(r), string.format("test: %d expected: %d received: %d", i, string.len(v[2]), string.len(r))) + else + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + end + + for i, v in ipairs(fields) do + local r = read_message(string.format("Fields[%s]", v[1]), v[2], v[3]) + assert(v[4] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[4]), tostring(r))) + end + return 0 +end diff --git a/src/heka_sandbox/test/test.h.in b/src/heka_sandbox/test/test.h.in new file mode 100644 index 0000000..4ded823 --- /dev/null +++ b/src/heka_sandbox/test/test.h.in @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Unit test setup @file */ + +#ifndef test_h_ +#define test_h_ + +#include "${CMAKE_SOURCE_DIR}/src/test/mu_test.h" + +#include +#include +#include +#include + +#define TEST_LUA_PATH "${CMAKE_SOURCE_DIR}/modules/?.lua" + +#ifdef _WIN32 +#define snprintf _snprintf +#define TEST_LUA_CPATH "${CMAKE_BINARY_DIR}/ep_base/lib/lua/?.dll" +#else +#define TEST_LUA_CPATH "${CMAKE_BINARY_DIR}/ep_base/lib/lua/?.so" +#endif + +#endif diff --git a/src/heka_sandbox/test/test_message_matcher.c b/src/heka_sandbox/test/test_message_matcher.c new file mode 100644 index 0000000..493ef7e --- /dev/null +++ b/src/heka_sandbox/test/test_message_matcher.c @@ -0,0 +1,300 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight/Heka message matcher unit tests @file */ + +#include +#include +#include +#include +#include + +#include "heka_sandbox/message_matcher.h" +#include "util/heka_message.h" + +#include "test.h" + +// {"Logger":"GoSpec","Uuid":"xxx","Pid":32157,"Severity":6,"EnvVersion":"0.8","Fields":[{""value":["bar"],"name":"foo","value_type":0},{"value":[64],"name":"number","value_type":2},{"value":["data"],"name":"bytes","value_type":1},{"value":[999,1024],"name":"int","value_type":2},{"value":[99.9],"name":"double","value_type":3},{"value":[true],"name":"bool","value_type":4},{"value":["alternate"],"name":"foo","value_type":0},{"value":["name=test;type=web;"],"name":"Payload","value_type":0},{"representation":"date-time","value":["Mon Jan 02 15:04:05 -0700 2006"],"name":"Timestamp","value_type":0},{"value":[0],"name":"zero","value_type":2},{"value":["43"],"name":"string","value_type":0}],"Payload":"Test Payload","Timestamp":1.428773426113e+18,"Hostname":"trink-x230","Type":"TEST"} + +char pb[] = "\x0a\x10\x27\x88\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x0c\x54\x65\x73\x74\x20\x50\x61\x79\x6c\x6f\x61\x64\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; +size_t pblen = sizeof(pb); + + +static char* test_stub() +{ + return NULL; +} + + +static char* test_true_matcher() +{ + char* tests[] = { + "TRUE" + , "Timestamp > 1.428773420000e+18" + , "Timestamp < 1.428773426999e18" + , "Timestamp == 1428773426113040228" + , "Timestamp > '2015-04-11T17:30:26Z'" + , "(Severity == 7 || Payload == 'Test Payload') && Type == 'TEST'" + , "EnvVersion == \"0.8\"" + , "EnvVersion == '0.8'" + , "EnvVersion != '0.9'" + , "EnvVersion > '0.7'" + , "EnvVersion >= '0.8'" + , "EnvVersion < '0.9'" + , "EnvVersion <= '0.8'" + , "Hostname != ''" + , "Logger == 'GoSpec'" + , "Pid != 0" + , "Severity != 5" + , "Severity < 7" + , "Severity <= 7" + , "Severity <= 6" + , "Severity == 6" + , "Severity > 5" + , "Severity >= 5" + , "Severity >= 6" + , "Timestamp > 0" + , "Type != 'test'" + , "Type == 'TEST' && Severity == 6" + , "Type == 'test' && Severity == 7 || Payload == 'Test Payload'" + , "Type == 'TEST'" + , "Type == 'foo' || Type == 'bar' || Type == 'TEST'" + , "Fields[foo] == 'bar'" + , "Fields[foo][0] == 'bar'" + , "Fields[foo][0][0] == 'bar'" + , "Fields[foo][1] == 'alternate'" + , "Fields[foo][1][0] == 'alternate'" + , "Fields[foo] <= 'barx'" + , "Fields[foo] < 'barx'" + , "Fields[foo] >= 'bar'" + , "Fields[foo] > 'baq'" + , "Fields[foo] != 'bara'" + , "Fields[bytes] == 'data'" + , "Fields[int] == 999" + , "Fields[int][0][1] == 1024" + , "Fields[double] == 99.9" + , "Fields[bool] == TRUE" + , "Fields[int] != NIL" + , "Fields[int][0][1] != NIL" + , "Fields[int][0][2] == NIL" + , "Fields[missing] == NIL" + , "Type =~ 'TEST'" + , "Type !~ 'bogus'" + , "Type =~ 'TEST' && Payload =~ 'Payload'" + , "Fields[foo][1] =~ 'alt'" + , "Fields[Payload] =~ 'name=%w+'" + , "Type =~ 'ST'" + , "Type =~ '^TE'" + , "Type =~ 'ST$'" + , "Type !~ '^te'" + , "Type !~ 'st$'" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 16); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); + mu_assert(mmb, "failed to create mmb"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m), "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_destroy_message_match_builder(mmb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_false_matcher() +{ + char* tests[] = { + "FALSE" + , "Timestamp == 1e9" + , "Timestamp > '2015-04-11T17:30:27Z'" + , "Type == 'test'&&(Severity==7||Payload=='Test Payload')" + , "EnvVersion == '0.9'" + , "EnvVersion != '0.8'" + , "EnvVersion > '0.9'" + , "EnvVersion >= '0.9'" + , "EnvVersion < '0.8'" + , "EnvVersion <= '0.7'" + , "Severity == 5" + , "Severity != 6" + , "Severity < 6" + , "Severity <= 5" + , "Severity > 6" + , "Severity >= 7" + , "Fields[foo] == 'ba'" + , "Fields[foo][1] == 'bar'" + , "Fields[foo][0][1] == 'bar'" + , "Fields[bool] == FALSE" + , "Fields[foo] > 'bara'" + , "Fields[foo] >= 'bara'" + , "Fields[foo] == 'bara'" + , "Type =~ 'Test'" + , "Type !~ 'TEST'" + , "Payload =~ '^Payload'" + , "Type == \"te'st\"" + , "Type == 'te\"st'" + , "Fields[int] =~ '999'" + , "Fields[zero] == \"0\"" + , "Fields[string] == 43" + , "Fields[int] == NIL" + , "Fields[int][0][1] == NIL" + , "Fields[missing] != NIL" + , "Type =~ '^te'" + , "Type =~ 'st$'" + , "Type !~ '^TE'" + , "Type !~ 'ST$'" + , "Logger =~ '.' && Type =~ '^anything'" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); + mu_assert(mmb, "failed to create mmb"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m) == false, "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_destroy_message_match_builder(mmb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_malformed_matcher() +{ + char* tests[] = { + "" + , "bogus" + , "Type = 'test'" // invalid operator + , "Pid == 'test='" // Pid is not a string + , "Type == 'test' && (Severity==7 || Payload == 'Test Payload'" // missing paren + , "Invalid == 'bogus'" // unknown variable name + , "Fields[]" // empty name key + , "Fields[test][]" // empty field index + , "Fields[test][a]" // non numeric field index + , "Fields[test][0][]" // empty array index + , "Fields[test][0][a]" // non numeric array index + , "Fields[test][0][0][]" // extra index dimension + , "Fields[test][xxxx" // unmatched bracket + , "Pid =~ '6'" // string match not allowed on numeric + , "Pid !~ '6'" // string match not allowed on numeric + , "Type =~ 'test" // unmatched quote + , "Type != 'test\"" // mis matched quote types + , "Pid =~ 6" // incorrect type for the operator + , "NIL" // invalid use of constant + , "Type == NIL" // existence check only works on fields + , "Fields[test] > NIL" // existence check only works with equals and not equals + , "TRUE FALSE" // missing operator + , "Timestamp == '20150411T173026'" // non rfc3339 timestamp + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); + mu_assert(mmb, "failed to create mmb"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + mu_assert(mm == NULL, "created malformed matcher"); + } + lsb_destroy_message_match_builder(mmb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* benchmark_matcher_create() +{ + int iter = 100000; + const char* exp = "Type == 'TEST' && Severity == 6"; + + lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); + mu_assert(mmb, "failed to create mmb"); + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_message_matcher* mm = lsb_create_message_matcher(mmb, exp); + mu_assert(mm, "lsb_create_message_matcher failed"); + lsb_destroy_message_matcher(mm); + } + t = clock() - t; + lsb_destroy_message_match_builder(mmb); + printf("benchmark_matcher_create: %g\n", ((double)t) / CLOCKS_PER_SEC + / iter); + return NULL; +} + +static char* benchmark_match() +{ + int iter = 1000000; + char* tests[] = { + "Type == 'TEST' && Severity == 6" + , "Fields[foo] == 'bar' && Severity == 6" + , "Fields[number] == 64 && Severity == 6" + , "Fields[missing] == NIL" + , "Fields[int] != NIL" + , "Type =~ '^[Tt]EST' && Severity == 6" + , "Payload =~ '^Test'" + , "Payload =~ 'load$'" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); + mu_assert(mmb, "failed to create mmb"); + + for (int i = 0; tests[i]; i++) { + lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + mu_assert(mm, "lsb_create_message_matcher failed: %s", tests[i]); + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(lsb_eval_message_matcher(mm, &m), + "lsb_eval_message_matcher failed"); + } + t = clock() - t; + lsb_destroy_message_matcher(mm); + printf("matcher: '%s': %g\n", tests[i], ((double)t) / CLOCKS_PER_SEC + / iter); + } + lsb_destroy_message_match_builder(mmb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_true_matcher); + mu_run_test(test_false_matcher); + mu_run_test(test_malformed_matcher); + + mu_run_test(benchmark_matcher_create); + mu_run_test(benchmark_match); + return NULL; +} + + +int main() +{ + char* result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/heka_sandbox/test/test_sandbox.c b/src/heka_sandbox/test/test_sandbox.c new file mode 100644 index 0000000..afa81e4 --- /dev/null +++ b/src/heka_sandbox/test/test_sandbox.c @@ -0,0 +1,503 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Heka sandbox unit tests @file */ + +#include "test.h" + +#include +#include +#include +#include +#include + +#include "heka_sandbox/sandbox.h" + +// {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} +static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; + +char *e = NULL; + +void dlog(const char *component, int level, const char *fmt, ...) +{ + + va_list args; + va_start(args, fmt); + fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, + component ? component : "unnamed"); + vfprintf(stderr, fmt, args); + fwrite("\n", 1, 1, stderr); + va_end(args); +} + + +static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, + const char *cp_string) +{ + static int cnt = 0; + struct im_result { + const char *pb; + size_t pb_len; + double cp_numeric; + const char *cp_string; + }; + + struct im_result results[] = { + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 67, .cp_numeric = 99, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 156, .cp_numeric = NAN, .cp_string = "foo.log:123" }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return 1; + } + + if (parent) { + fprintf(stderr, "test: %d parent set\n", cnt); + } + + if (pb_len != results[cnt].pb_len) { + fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" + PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); + return 1; + } + + if (memcmp(pb, results[cnt].pb, pb_len)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = 0; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); + } + fprintf(stderr, "\nreceived: "); + for (size_t i = 0; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; + } + + bool ncp_failed = false; + if (isnan(results[cnt].cp_numeric)) { + if (!isnan(cp_numeric)) { + ncp_failed = true; + } + } else if (results[cnt].cp_numeric != cp_numeric) { + ncp_failed = true; + } + if (ncp_failed) { + fprintf(stderr, "test: %d cp_numeric expected: %g received: %g\n", cnt, + results[cnt].cp_numeric, cp_numeric); + return 1; + } + + bool ncs_failed = false; + if (!results[cnt].cp_string) { + if (cp_string) { + ncs_failed = true; + } + } else if (!cp_string || strcmp(results[cnt].cp_string, cp_string)) { + ncs_failed = true; + } + if (ncs_failed) { + fprintf(stderr, "test: %d cp_string expected: %s received: %s\n", cnt, + results[cnt].cp_string ? results[cnt].cp_string : "NULL", + cp_string ? cp_string : "NULL"); + return 1; + } + cnt++; + return 0; +} + + +static int aim(void *parent, const char *pb, size_t pb_len) +{ + static int cnt = 0; + struct im_result { + const char *pb; + size_t pb_len; + double cp_numeric; + const char *cp_string; + }; + + struct im_result results[] = { + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x1f\x7d\x09\x9e\xf5\x9d\x40\x1d\xa8\xaf\x6a\xff\xc3\x21\xeb\x42\x10\x80\x88\xe4\xaa\xa0\xa9\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 88, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x5b\x7d\xee\xa0\x02\xbc\x45\xbb\xaf\xa9\xcc\x2c\xdd\x65\xde\x45\x10\x80\x88\xdc\xad\xcd\xbf\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 110, .cp_numeric = NAN, .cp_string = NULL }, + }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return 1; + } + + if (parent) { + fprintf(stderr, "test: %d parent set\n", cnt); + } + + if (pb_len != results[cnt].pb_len) { + fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" + PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); + return 1; + } + + if (cnt == 0) { + if (memcmp(pb, results[cnt].pb, pb_len)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = 0; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); + } + fprintf(stderr, "\nreceived: "); + for (size_t i = 0; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; + } + } else { + lsb_heka_message m; + lsb_init_heka_message(&m, 2); + bool rv = lsb_decode_heka_message(&m, pb, pb_len, dlog); + lsb_free_heka_message(&m); + if (!rv) return 1; + } + cnt++; + return 0; +} + + +static int ucp(void *parent, void *sequence_id) +{ + static int cnt = 0; + if (parent) return 1; + void *results[] = { NULL, (void *)99 }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return 1; + } + + if (results[cnt] != sequence_id) { + fprintf(stderr, "expected: %p received: %p\n", results[cnt], sequence_id); + return 1; + } + cnt++; + return 0; +} + + +static char* test_create_input_sandbox() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, + iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_input(NULL, "notfourd.lua", NULL, NULL, NULL, + iim); + mu_assert(!hsb, "lsb_heka_create_input succeeded"); + + hsb = lsb_heka_create_input(NULL, NULL, NULL, NULL, NULL, iim); + mu_assert(!hsb, "lsb_heka_create_input succeeded"); + + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, NULL, NULL); + mu_assert(!hsb, "lsb_heka_create_input succeeded"); + lsb_heka_destroy_sandbox(hsb); // test NULL + return NULL; +} + + +static char* test_create_analysis_sandbox() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, + aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_analysis(NULL, "notfound.lua", NULL, NULL, NULL, aim); + mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); + + hsb = lsb_heka_create_analysis(NULL, NULL, NULL, NULL, NULL, aim); + mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); + + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, NULL, + NULL); + mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); + return NULL; +} + + +static char* test_create_output_sandbox() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_output(NULL, "notfound.lua", NULL, NULL, NULL, ucp); + mu_assert(!hsb, "lsb_heka_create_output succeeded"); + + hsb = lsb_heka_create_output(NULL, NULL, NULL, NULL, NULL, ucp); + mu_assert(!hsb, "lsb_heka_create_output succeeded"); + + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, NULL, NULL); + mu_assert(!hsb, "lsb_heka_create_output succeeded"); + return NULL; +} + + +static char* test_timer_event() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, + aim); + mu_assert(0 == lsb_heka_timer_event(hsb, 0, false), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(0 == lsb_heka_timer_event(hsb, 1, true), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(1 == lsb_heka_timer_event(hsb, 2, false), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_analysis(NULL, "lua/input.lua", NULL, NULL, NULL, aim); + mu_assert(hsb, "lsb_heka_create_analysis succeeded"); + mu_assert_rv(1, lsb_heka_timer_event(hsb, 0, false)); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp("timer_event() function was not found", err) == 0, + "received: %s", err); + lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_pm_input() +{ + struct pm_result { + double ncp; + const char *scp; + int rv; + const char *err; + }; + + struct pm_result results[] = { + { .ncp = 0, .scp = NULL, .rv = -2, .err = "host specific failure" }, + { .ncp = 1, .scp = NULL, .rv = -1, .err = "failed" }, + { .ncp = 2, .scp = NULL, .rv = 0, .err = "ok" }, + { .ncp = NAN, .scp = "string", .rv = 0, .err = "string" }, + { .ncp = NAN, .scp = NULL, .rv = 0, .err = "no cp" }, + }; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ + int rv = lsb_heka_process_message_input(hsb, &m, results[i].ncp, + results[i].scp, false); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", + results[i].err, err); + mu_assert(results[i].rv == rv, "test: %u expected: %d received: %d", i, + results[i].rv, + rv); + } + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_pm_analysis() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, + aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + mu_assert_rv(0, lsb_heka_process_message_analysis(hsb, &m, false)); + const char *err = lsb_heka_get_error(hsb); + const char *eerr = ""; + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_pm_no_return() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, dlog, + aim); + mu_assert_rv(1, lsb_heka_process_message_analysis(hsb, &m, false)); + const char *err = lsb_heka_get_error(hsb); + const char *eerr = "process_message() must return a numeric status code"; + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_pm_output() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + + mu_assert_rv(0, lsb_heka_process_message_output(hsb, &m, NULL, false)); + const char *err = lsb_heka_get_error(hsb); + const char *eerr = ""; + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + + mu_assert_rv(-5, lsb_heka_process_message_output(hsb, &m, (void *)99, false)); + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_im_input() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/iim.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_im_analysis() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/aim.lua", NULL, + "Hostname = 'foo.com';Logger = 'aim'", dlog, + aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_encode_message() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, + "Hostname = 'sh';Logger = 'sl'", dlog, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_decode_message() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/decode_message.lua", NULL, NULL, + dlog, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_read_message() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, dlog), "failed"); + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, dlog, + aim); + mu_assert(hsb, "lsb_heka_create_analysist failed"); + int rv = lsb_heka_process_message_analysis(hsb, &m, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* benchmark_decode_message() +{ + int iter = 100000; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/decode_message_benchmark.lua", NULL, + NULL, dlog, ucp); + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(0 == lsb_heka_process_message_output(hsb, &m, NULL, false), "%s", + lsb_heka_get_error(hsb)); + } + t = clock() - t; + printf("benchmark_decode_message() %g seconds\n", ((double)t) + / CLOCKS_PER_SEC / iter); + + mu_assert(hsb, "lsb_heka_create_output failed"); + lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_create_input_sandbox); + mu_run_test(test_create_analysis_sandbox); + mu_run_test(test_create_output_sandbox); + mu_run_test(test_timer_event); + mu_run_test(test_pm_input); + mu_run_test(test_pm_analysis); + mu_run_test(test_pm_no_return); + mu_run_test(test_pm_output); + mu_run_test(test_im_input); + mu_run_test(test_im_analysis); + mu_run_test(test_encode_message); + mu_run_test(test_decode_message); + mu_run_test(test_read_message); + + mu_run_test(benchmark_decode_message); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + free(e); + + return result != 0; +} diff --git a/src/lsb_deserialize_protobuf.c b/src/lsb_deserialize_protobuf.c deleted file mode 100644 index 58d7d26..0000000 --- a/src/lsb_deserialize_protobuf.c +++ /dev/null @@ -1,387 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua sandbox Heka protobuf deserialization @file */ - -#include - -#include "luasandbox/lauxlib.h" -#include "luasandbox.h" - -#define UUID_SIZE 16 -#define MAX_VARINT_BYTES 10 - -static unsigned const char* -read_key(unsigned const char* p, int* tag, int* wiretype) -{ - *wiretype = 7 & *p; - *tag = *p >> 3; - return ++p; -} - - -static unsigned const char* -read_length(unsigned const char* p, unsigned const char* e, size_t* vi) -{ - *vi = 0; - unsigned i, shift = 0; - for (i = 0; p != e && i < MAX_VARINT_BYTES; i++) { - *vi |= ((unsigned long long)p[i] & 0x7f) << shift; - shift += 7; - if ((p[i] & 0x80) == 0) break; - } - if (i == MAX_VARINT_BYTES) { - return NULL; - } - return p + i + 1; -} - - -static unsigned const char* -read_varint(unsigned const char* p, unsigned const char* e, long long* vi) -{ - *vi = 0; - unsigned i, shift = 0; - for (i = 0; p != e && i < MAX_VARINT_BYTES; i++) { - *vi |= ((unsigned long long)p[i] & 0x7f) << shift; - shift += 7; - if ((p[i] & 0x80) == 0) break; - } - if (i == MAX_VARINT_BYTES) { - return NULL; - } - return p + i + 1; -} - - -static unsigned const char* -read_string(lua_State* lua, - int wiretype, - unsigned const char* p, - unsigned const char* e) -{ - if (wiretype != 2) { - return NULL; - } - - size_t len = 0; - p = read_length(p, e, &len); - if (!p || p + len > e) { - return NULL; - } - lua_pushlstring(lua, (const char*)p, len); - p += len; - return p; -} - - -static unsigned const char* -process_varint(lua_State* lua, - const char* name, - int wiretype, - int stack_index, - unsigned const char* p, - unsigned const char* e) -{ - if (wiretype != 0) { - return NULL; - } - long long val = 0; - p = read_varint(p, e, &val); - if (!p) { - return NULL; - } - lua_pushnumber(lua, (lua_Number)val); - lua_setfield(lua, stack_index, name); - return p; -} - - -static unsigned const char* -process_fields(lua_State* lua, - unsigned const char* p, - unsigned const char* e) -{ - int tag = 0; - int wiretype = 0; - int has_name = 0; - int value_count = 0; - size_t len = 0; - - p = read_length(p, e, &len); - if (!p || p + len > e) { - return NULL; - } - e = p + len; // only process to the end of the current field record - - lua_newtable(lua); // Table to be added to the Fields array index 4 - lua_newtable(lua); // Table to hold the value(s) index 5 - do { - p = read_key(p, &tag, &wiretype); - - switch (tag) { - case 1: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 4, "name"); - has_name = 1; - } - break; - - case 2: - p = process_varint(lua, "value_type", wiretype, 4, p, e); - break; - - case 3: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 4, "representation"); - } - break; - - case 4: // value_string - case 5: // value_bytes - p = read_string(lua, wiretype, p, e); - if (p) { - lua_rawseti(lua, 5, ++value_count); - } - break; - - case 6: // value_integer - { - long long val = 0; - switch (wiretype) { - case 0: - p = read_varint(p, p + len, &val); - if (!p) break; - lua_pushnumber(lua, (lua_Number)val); - lua_rawseti(lua, 5, ++value_count); - break; - case 2: - p = read_length(p, e, &len); - if (!p || p + len > e) { - p = NULL; - break; - } - do { - p = read_varint(p, p + len, &val); - if (!p) break; - lua_pushnumber(lua, (lua_Number)val); - lua_rawseti(lua, 5, ++value_count); - } - while (p < e); - break; - default: - p = NULL; - break; - } - } - break; - - case 7: // value_double - { - double val = 0; - switch (wiretype) { - case 1: - if (p + sizeof(double) > e) { - p = NULL; - break; - } - memcpy(&val, p, sizeof(double)); - p += sizeof(double); - lua_pushnumber(lua, val); - lua_rawseti(lua, 5, ++value_count); - break; - case 2: - p = read_length(p, e, &len); - if (!p || p + len > e || len % sizeof(double) != 0) { - p = NULL; - break; - } - do { - memcpy(&val, p, sizeof(double)); - p += sizeof(double); - lua_pushnumber(lua, val); - lua_rawseti(lua, 5, ++value_count); - } - while (p < e); - break; - default: - p = NULL; - break; - } - } - break; - - case 8: // value_bool - { - long long val = 0; - switch (wiretype) { - case 0: - p = read_varint(p, p + len, &val); - if (!p) break; - lua_pushboolean(lua, (int)val); - lua_rawseti(lua, 5, ++value_count); - break; - case 2: - p = read_length(p, e, &len); - if (!p || p + len > e) { - p = NULL; - break; - } - do { - p = read_varint(p, p + len, &val); - if (!p) break; - lua_pushboolean(lua, (int)val); - lua_rawseti(lua, 5, ++value_count); - } - while (p < e); - break; - default: - p = NULL; - break; - } - } - break; - default: - p = NULL; // don't allow unknown tags - break; - } - } - while (p && p < e); - - lua_setfield(lua, 4, "value"); - - return has_name ? p : NULL; -} - - -int lsb_decode_protobuf(lua_State* lua) -{ - int n = lua_gettop(lua); - if (n != 1 || lua_type(lua, 1) != LUA_TSTRING) { - return luaL_argerror(lua, 0, "must have one string argument"); - } - - size_t len; - const char* pbstr = lua_tolstring(lua, 1, &len); - if (len < 20) { - return luaL_error(lua, "invalid message, too short"); - } - - unsigned const char* p = (unsigned const char*)pbstr; - unsigned const char* lp = p; - unsigned const char* e = (unsigned const char*)pbstr + len; - int wiretype = 0; - int tag = 0; - int has_uuid = 0; - int has_timestamp = 0; - int field_count = 0; - - lua_newtable(lua); // message table index 2 - do { - p = read_key(p, &tag, &wiretype); - - switch (tag) { - case 1: - p = read_string(lua, wiretype, p, e); - if (p && p - lp == 18) { - lua_setfield(lua, 2, "Uuid"); - has_uuid = 1; - } else { - p = NULL; - } - break; - - case 2: - p = process_varint(lua, "Timestamp", wiretype, 2, p, e); - if (p) { - has_timestamp = 1; - } - break; - - case 3: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 2, "Type"); - } - break; - - case 4: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 2, "Logger"); - } - break; - - case 5: - p = process_varint(lua, "Severity", wiretype, 2, p, e); - break; - - case 6: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 2, "Payload"); - } - break; - - case 7: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 2, "EnvVersion"); - } - break; - - case 8: - p = process_varint(lua, "Pid", wiretype, 2, p, e); - break; - - case 9: - p = read_string(lua, wiretype, p, e); - if (p) { - lua_setfield(lua, 2, "Hostname"); - } - break; - - case 10: - if (wiretype != 2) { - p = NULL; - break; - } - if (field_count == 0) { - lua_newtable(lua); // Fields table index 3 - } - p = process_fields(lua, p, e); - if (p) { - lua_rawseti(lua, 3, ++field_count); - } - break; - - default: - p = NULL; // don't allow unknown tags - break; - } - if (p) lp = p; - } - while (p && p < e); - - if (!p) { - return luaL_error(lua, "error in tag: %d wiretype: %d offset: %d", tag, - wiretype, (const char*)lp - pbstr); - } - - if (!(has_uuid && has_timestamp)) { - return luaL_error(lua, "missing required field uuid: %s timestamp: %s", - has_uuid ? "found" : "not found", - has_timestamp ? "found" : "not found"); - } - - if (field_count) { - lua_setfield(lua, 2, "Fields"); - } - - return 1; -} diff --git a/src/lsb_output.c b/src/lsb_output.c deleted file mode 100644 index 08c2836..0000000 --- a/src/lsb_output.c +++ /dev/null @@ -1,235 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua sandbox output buffer implementation @file */ - -#include "luasandbox_output.h" - -#include -#include -#include -#include - -#include "luasandbox/lauxlib.h" -#include "lsb_private.h" -#include "luasandbox_serialize.h" -#include "lsb_serialize_protobuf.h" - -static const char* output_function = "lsb_output"; - - -static void update_output_stats(lua_sandbox* lsb) -{ - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = lsb->output.pos; - if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] - > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; - } -} - - -void lsb_add_output_function(lua_State* lua, lua_CFunction fp) -{ - lua_pushstring(lua, output_function); - lua_pushcfunction(lua, fp); - lua_rawset(lua, -3); -} - - -lua_CFunction lsb_get_output_function(lua_State* lua, int index) -{ - lua_CFunction fp = NULL; - lua_getfenv(lua, index); - lua_pushstring(lua, output_function); - lua_rawget(lua, -2); - fp = lua_tocfunction(lua, -1); - lua_pop(lua, 2); // environment and field - return fp; -} - - -int lsb_appendc(lsb_output_data* output, char ch) -{ - size_t needed = 2; - if (output->size - output->pos < needed) { - if (lsb_realloc_output(output, needed)) return 1; - } - output->data[output->pos++] = ch; - output->data[output->pos] = 0; - return 0; -} - - -int lsb_appendf(lsb_output_data* output, const char* fmt, ...) -{ - va_list args; - int result = 0; - int remaining = 0; - char* ptr = NULL, *old_ptr = NULL; - do { - ptr = output->data + output->pos; - remaining = (int)(output->size - output->pos); - va_start(args, fmt); - int needed = vsnprintf(ptr, remaining, fmt, args); - va_end(args); - if (needed == -1) { - // Windows and Unix have different return values for this function - // -1 on Unix is a format error - // -1 on Windows means the buffer is too small and the required len - // is not returned - needed = remaining; - } - if (needed >= remaining) { - if (output->maxsize - && (output->size >= output->maxsize - || output->pos + needed >= output->maxsize)) { - return 1; - } - size_t newsize = output->size * 2; - while ((size_t)needed >= newsize - output->pos) { - newsize *= 2; - } - if (output->maxsize && newsize > output->maxsize) { - newsize = output->maxsize; - } - void* p = malloc(newsize); - if (p != NULL) { - memcpy(p, output->data, output->pos); - old_ptr = output->data; - output->data = p; - output->size = newsize; - } else { - return 1; // Out of memory condition. - } - } else { - output->pos += needed; - break; - } - } - while (1); - free(old_ptr); - return result; -} - - -int lsb_appends(lsb_output_data* output, const char* str, size_t len) -{ - size_t needed = len + 1; - if (output->size - output->pos < needed) { - if (lsb_realloc_output(output, needed)) return 1; - } - memcpy(output->data + output->pos, str, len); - output->pos += len; - output->data[output->pos] = 0; - return 0; -} - - -int lsb_realloc_output(lsb_output_data* output, size_t needed) -{ - if (output->maxsize && needed + output->pos > output->maxsize) { - return 1; - } - size_t newsize = output->size * 2; - while (needed >= newsize - output->pos) { - newsize *= 2; - } - if (output->maxsize && newsize > output->maxsize) { - newsize = output->maxsize; - } - - void* ptr = realloc(output->data, newsize); - if (!ptr) return 1; - output->data = ptr; - output->size = newsize; - return 0; -} - - -void lsb_output(lua_sandbox* lsb, int start, int end, int append) -{ - if (!append) { - lsb->output.pos = 0; - } - - int result = 0; - for (int i = start; result == 0 && i <= end; ++i) { - switch (lua_type(lsb->lua, i)) { - case LUA_TNUMBER: - if (lsb_output_double(&lsb->output, lua_tonumber(lsb->lua, i))) { - result = 1; - } - break; - case LUA_TSTRING: - { - size_t len; - const char* s = lua_tolstring(lsb->lua, i, &len); - if (lsb_appends(&lsb->output, s, len)) { - result = 1; - } - } - break; - case LUA_TNIL: - if (lsb_appends(&lsb->output, "nil", 3)) { - result = 1; - } - break; - case LUA_TBOOLEAN: - if (lsb_appendf(&lsb->output, "%s", - lua_toboolean(lsb->lua, i) - ? "true" : "false")) { - result = 1; - } - break; - case LUA_TUSERDATA: - { - lua_CFunction fp = lsb_get_output_function(lsb->lua, i); - if (!fp) { - luaL_argerror(lsb->lua, i, "unknown userdata type"); - return; // never reaches here but the compiler doesn't know it - } - lua_pushvalue(lsb->lua, i); - lua_pushlightuserdata(lsb->lua, &lsb->output); - result = fp(lsb->lua); - lua_pop(lsb->lua, 2); // remove the copy of the value and the output - } - break; - default: - luaL_argerror(lsb->lua, i, "unsupported type"); - break; - } - } - update_output_stats(lsb); - if (result != 0) { - if (lsb->error_message[0] == 0) { - luaL_error(lsb->lua, "output_limit exceeded"); - } - luaL_error(lsb->lua, lsb->error_message); - } -} - - -int lsb_output_protobuf(lua_sandbox* lsb, int index, int append) -{ - if (lua_type(lsb->lua, index) != LUA_TTABLE) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, "takes a table argument"); - return 1; - } - - if (!append) { - lsb->output.pos = 0; - } - - size_t last_pos = lsb->output.pos; - if (lsb_serialize_table_as_pb(lsb, index) != 0) { - lsb->output.pos = last_pos; - return 1; - } - - update_output_stats(lsb); - return 0; -} diff --git a/src/lsb_private.h b/src/lsb_private.h deleted file mode 100644 index 7b482c4..0000000 --- a/src/lsb_private.h +++ /dev/null @@ -1,49 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Lua sandbox private functions @file */ - -#ifndef lsb_private_h_ -#define lsb_private_h_ - -#include "luasandbox.h" -#include "luasandbox/lua.h" - -#ifdef _WIN32 -#define snprintf _snprintf -#endif - -#define LSB_OUTPUT_SIZE 1024 - -struct lsb_output_data -{ - size_t maxsize; - size_t size; - size_t pos; - char* data; -}; - -struct lua_sandbox { - lua_State* lua; - void* parent; - lsb_state state; - lsb_output_data output; - char* lua_file; - size_t usage[LSB_UT_MAX][LSB_US_MAX]; - char error_message[LSB_ERROR_SIZE]; -}; - -/** - * Utility function to retrieve a user data output function - * - * @param lua - * @param index - * - * @return lua_CFunction - */ -lua_CFunction lsb_get_output_function(lua_State* lua, int index); - -#endif diff --git a/src/lsb_serialize_protobuf.c b/src/lsb_serialize_protobuf.c deleted file mode 100644 index e66657f..0000000 --- a/src/lsb_serialize_protobuf.c +++ /dev/null @@ -1,640 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua sandbox Heka protobuf serialization @file */ - -#include "lsb_serialize_protobuf.h" - -#include -#include -#include - -#include "luasandbox_output.h" -#include "lsb_private.h" - -#define UUID_SIZE 16 -#define MAX_VARINT_BYTES 10 - -/** - * Writes a varint encoded number to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Number to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int pb_write_varint(lsb_output_data* d, unsigned long long i) -{ - size_t needed = MAX_VARINT_BYTES; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - - if (i == 0) { - d->data[d->pos++] = 0; - return 0; - } - - while (i) { - d->data[d->pos++] = (i & 0x7F) | 0x80; - i >>= 7; - } - d->data[d->pos - 1] &= 0x7F; // end the varint - return 0; -} - - -/** - * Writes a double to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Double to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int pb_write_double(lsb_output_data* d, double i) -{ - size_t needed = sizeof(double); - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - - // todo add big endian support if necessary - memcpy(&d->data[d->pos], &i, needed); - d->pos += needed; - return 0; -} - - -/** - * Writes a bool to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Number to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int pb_write_bool(lsb_output_data* d, int i) -{ - size_t needed = 1; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - - if (i) { - d->data[d->pos++] = 1; - } else { - d->data[d->pos++] = 0; - } - return 0; -} - - -/** - * Writes a field tag (tag id/wire type) to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param wire_type Field wire type. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int pb_write_tag(lsb_output_data* d, unsigned char id, - unsigned char wire_type) -{ - size_t needed = 1; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - - d->data[d->pos++] = wire_type | (id << 3); - return 0; -} - - -/** - * Updates the field length in the output buffer once the size is known, this - * allows for single pass encoding. - * - * @param d Pointer to the output data buffer. - * @param len_pos Position in the output buffer where the length should be - * written. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int update_field_length(lsb_output_data* d, size_t len_pos) -{ - size_t len = d->pos - len_pos - 1; - if (len < 128) { - d->data[len_pos] = (char)len; - return 0; - } - size_t l = len, cnt = 0; - while (l) { - l >>= 7; - ++cnt; // compute the number of bytes needed for the varint length - } - size_t needed = cnt - 1; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - size_t end_pos = d->pos + needed; - memmove(&d->data[len_pos + cnt], &d->data[len_pos + 1], len); - d->pos = len_pos; - if (pb_write_varint(d, len)) return 1; - d->pos = end_pos; - return 0; -} - - -/** - * Writes a string to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param s String to output. - * @param len Length of s. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int -pb_write_string(lsb_output_data* d, char id, const char* s, size_t len) -{ - if (pb_write_tag(d, id, 2)) { - return 1; - } - if (pb_write_varint(d, len)) { - return 1; - } - - size_t needed = len; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - memcpy(&d->data[d->pos], s, len); - d->pos += len; - return 0; -} - - -/** - * Retrieve the string value for a Lua table entry (the table should be on top - * of the stack). If the entry is not found or not a string nothing is encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int -encode_string(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, - int index) -{ - int result = 0; - lua_getfield(lsb->lua, index, name); - if (lua_isstring(lsb->lua, -1)) { - size_t len; - const char* s = lua_tolstring(lsb->lua, -1, &len); - result = pb_write_string(d, id, s, len); - } - lua_pop(lsb->lua, 1); - return result; -} - - -/** - * Retrieve the numeric value for a Lua table entry (the table should be on top - * of the stack). If the entry is not found or not a number nothing is encoded, - * otherwise the number is varint encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int -encode_int(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, - int index) -{ - int result = 0; - lua_getfield(lsb->lua, index, name); - if (lua_isnumber(lsb->lua, -1)) { - unsigned long long i = (unsigned long long)lua_tonumber(lsb->lua, -1); - if (!(result = pb_write_tag(d, id, 0))) { - result = pb_write_varint(d, i); - } - } - lua_pop(lsb->lua, 1); - return result; -} - - -/** - * Encodes the field value. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param first Boolean indicator used to add addition protobuf data in the - * correct order. - * @param representation String representation of the field i.e., "ms" - * @param value_type Protobuf value type - * - * @return int Zero on success, non-zero if out of memory. - */ -static int -encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, - const char* representation, int value_type); - - -/** - * Encodes a field that has an array of values. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param ltype Lua type of the array values. - * @param representation String representation of the field i.e., "ms" - * - * @return int Zero on success, non-zero if out of memory. - */ -static int -encode_field_array(lua_sandbox* lsb, lsb_output_data* d, int t, - const char* representation, int value_type) -{ - int result = 0, first = (int)lua_objlen(lsb->lua, -1); - int multiple = first > 1 ? first : 0; - size_t len_pos = 0; - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - if (lua_type(lsb->lua, -1) != t) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); - return 1; - } - result = encode_field_value(lsb, d, first, representation, value_type); - if (first) { - len_pos = d->pos; - first = 0; - } - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. - } - if (multiple && value_type == 2) { // fix up the varint packed length - size_t i = len_pos - 2; - int y = 0; - while (d->data[i] != 0 && y < MAX_VARINT_BYTES) { // find the length byte - --i; - ++y; - } - if (y == MAX_VARINT_BYTES || update_field_length(d, i)) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "unable set the length of the packed integer array"); - return 1; - } - } - return result; -} - - -/** - * Encodes a field that contains metadata in addition to its value. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int encode_field_object(lua_sandbox* lsb, lsb_output_data* d) -{ - int result = 0; - int value_type = -1; - const char* representation = NULL; - lua_getfield(lsb->lua, -1, "representation"); - if (lua_isstring(lsb->lua, -1)) { - representation = lua_tostring(lsb->lua, -1); - } - lua_getfield(lsb->lua, -2, "value_type"); - if (lua_isnumber(lsb->lua, -1)) { - value_type = (int)lua_tointeger(lsb->lua, -1); - } - lua_getfield(lsb->lua, -3, "value"); - result = encode_field_value(lsb, d, 1, representation, value_type); - lua_pop(lsb->lua, 3); // remove representation, value_type and value - return result; -} - - -static int -encode_field_value(lua_sandbox* lsb, lsb_output_data* d, int first, - const char* representation, int value_type) -{ - int result = 1; - size_t len; - const char* s; - - int t = lua_type(lsb->lua, -1); - switch (t) { - case LUA_TSTRING: - switch (value_type) { - case -1: // not specified defaults to string - value_type = 0; - case 0: - case 1: - break; - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "invalid string value_type: %d", value_type); - return 1; - } - if (first) { // this uglyness keeps the protobuf fields in order without - // additional lookups - if (value_type == 1) { - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, value_type)) return 1; - } - if (representation) { - if (pb_write_string(d, 3, representation, strlen(representation))) { - return 1; - } - } - } - s = lua_tolstring(lsb->lua, -1, &len); - if (value_type == 1) { - result = pb_write_string(d, 5, s, len); - } else { - result = pb_write_string(d, 4, s, len); - } - break; - case LUA_TNUMBER: - switch (value_type) { - case -1: // not specified defaults to double - value_type = 3; - case 2: - case 3: - break; - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "invalid numeric value_type: %d", value_type); - return 1; - } - if (first) { - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, value_type)) return 1; - if (representation) { - if (pb_write_string(d, 3, representation, - strlen(representation))) { - return 1; - } - } - if (1 == first) { - if (value_type == 2) { - if (pb_write_tag(d, 6, 0)) return 1; - } else { - if (pb_write_tag(d, 7, 1)) return 1; - } - } else { // pack array - if (value_type == 2) { - if (pb_write_tag(d, 6, 2)) return 1; - if (pb_write_varint(d, 0)) return 1; // length tbd later - } else { - if (pb_write_tag(d, 7, 2)) return 1; - if (pb_write_varint(d, first * sizeof(double))) return 1; - } - } - } - if (value_type == 2) { - result = pb_write_varint(d, lua_tointeger(lsb->lua, -1)); - } else { - result = pb_write_double(d, lua_tonumber(lsb->lua, -1)); - } - break; - - case LUA_TBOOLEAN: - if (value_type != -1 && value_type != 4) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "invalid boolean value_type: %d", value_type); - return 1; - } - if (first) { - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, 4)) return 1; - if (representation) { - if (pb_write_string(d, 3, representation, - strlen(representation))) { - return 1; - } - } - if (1 == first) { - if (pb_write_tag(d, 8, 0)) return 1; - } else { - if (pb_write_tag(d, 8, 2)) return 1; - if (pb_write_varint(d, first)) return 1; - } - } - result = pb_write_bool(d, lua_toboolean(lsb->lua, -1)); - break; - - case LUA_TTABLE: - { - lua_rawgeti(lsb->lua, -1, 1); - int t = lua_type(lsb->lua, -1); - lua_pop(lsb->lua, 1); // remove the array test value - switch (t) { - case LUA_TNIL: - result = encode_field_object(lsb, d); - break; - case LUA_TNUMBER: - case LUA_TSTRING: - case LUA_TBOOLEAN: - result = encode_field_array(lsb, d, t, representation, value_type); - break; - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "unsupported array type: %s", lua_typename(lsb->lua, t)); - result = 1; - break; - } - } - break; - - case LUA_TUSERDATA: - { - lua_CFunction fp = lsb_get_output_function(lsb->lua, -1); - size_t len_pos = 0; - if (!fp) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "user data object does not implement lsb_output"); - return 1; - } - if (first) { - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, 1)) return 1; // encode userdata as a byte array - if (representation) { - if (pb_write_string(d, 3, representation, strlen(representation))) { - return 1; - } - } - } - - if (pb_write_tag(d, 5, 2)) return 1; - len_pos = d->pos; - if (pb_write_varint(d, 0)) return 1; // length tbd later - lua_pushlightuserdata(lsb->lua, d); - if (fp(lsb->lua)) return 1; - lua_pop(lsb->lua, 1); // remove output function - result = update_field_length(d, len_pos); - } - break; - - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", - lua_typename(lsb->lua, t)); - result = 1; - break; - } - return result; -} - - -/** - * Iterates over the specified Lua table encoding the contents as user defined - * message fields. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -static int -encode_fields(lua_sandbox* lsb, lsb_output_data* d, char id, const char* name, - int index) -{ - int result = 0; - lua_getfield(lsb->lua, index, name); - if (!lua_istable(lsb->lua, -1)) { - return result; - } - - lua_rawgeti(lsb->lua, -1, 1); // test for the array notation - size_t len_pos, len; - if (lua_istable(lsb->lua, -1)) { - int i = 1; - do { - if (pb_write_tag(d, id, 2)) return 1; - len_pos = d->pos; - if (pb_write_varint(d, 0)) return 1; // length tbd later - lua_getfield(lsb->lua, -1, "name"); - if (lua_isstring(lsb->lua, -1)) { - const char* s = lua_tolstring(lsb->lua, -1, &len); - if (pb_write_string(d, 1, s, len)) return 1; - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "field name must be a string"); - return 1; - } - lua_pop(lsb->lua, 1); // remove the name - - if (encode_field_object(lsb, d)) return 1; - if (update_field_length(d, len_pos)) return 1; - - lua_pop(lsb->lua, 1); // remove the current field object - lua_rawgeti(lsb->lua, -1, ++i); // grab the next field object - } - while (!lua_isnil(lsb->lua, -1)); - } else { - lua_pop(lsb->lua, 1); // remove the array test value - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - if (pb_write_tag(d, id, 2)) return 1; - len_pos = d->pos; - if (pb_write_varint(d, 0)) return 1; // length tbd later - if (lua_isstring(lsb->lua, -2)) { - const char* s = lua_tolstring(lsb->lua, -2, &len); - if (pb_write_string(d, 1, s, len)) return 1; - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "field name must be a string"); - return 1; - } - if (encode_field_value(lsb, d, 1, NULL, -1)) return 1; - if (update_field_length(d, len_pos)) return 1; - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. - } - } - lua_pop(lsb->lua, 1); // remove the fields table - return result; -} - - -int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index) -{ - lsb_output_data* d = &lsb->output; - d->pos = 0; - size_t needed = 18; - if (needed > d->size) { - if (lsb_realloc_output(d, needed)) return 1; - } - - // use existing or create a type 4 uuid - lua_getfield(lsb->lua, index, "Uuid"); - size_t len; - const char* uuid = lua_tolstring(lsb->lua, -1, &len); - - d->data[d->pos++] = 2 | (1 << 3); - d->data[d->pos++] = UUID_SIZE; - if (uuid && len == UUID_SIZE) { - memcpy(d->data + d->pos, uuid, UUID_SIZE); - d->pos += UUID_SIZE; - } else { - for (int x = 0; x < UUID_SIZE; ++x) { - d->data[d->pos++] = rand() % 256; - } - d->data[8] = (d->data[8] & 0x0F) | 0x40; - d->data[10] = (d->data[10] & 0x0F) | 0xA0; - } - lua_pop(lsb->lua, 1); // remove uuid - - // use existing or create a timestamp - lua_getfield(lsb->lua, index, "Timestamp"); - unsigned long long ts; - if (lua_isnumber(lsb->lua, -1)) { - ts = (unsigned long long)lua_tonumber(lsb->lua, -1); - } else { - ts = (unsigned long long)(time(NULL) * 1e9); - } - lua_pop(lsb->lua, 1); // remove timestamp - - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, ts)) return 1; - - if (encode_string(lsb, d, 3, "Type", index)) return 1; - if (encode_string(lsb, d, 4, "Logger", index)) return 1; - if (encode_int(lsb, d, 5, "Severity", index)) return 1; - if (encode_string(lsb, d, 6, "Payload", index)) return 1; - if (encode_string(lsb, d, 7, "EnvVersion", index)) return 1; - if (encode_int(lsb, d, 8, "Pid", index)) return 1; - if (encode_string(lsb, d, 9, "Hostname", index)) return 1; - if (encode_fields(lsb, d, 10, "Fields", index)) return 1; - // if we go above 15 pb_write_tag will need to start varint encoding - needed = 1; - if (needed > d->size - d->pos) { - if (lsb_realloc_output(d, needed)) return 1; - } - d->data[d->pos] = 0; // NULL terminate incase someone tries to treat this - // as a string - - return 0; -} diff --git a/src/lsb_serialize_protobuf.h b/src/lsb_serialize_protobuf.h deleted file mode 100644 index 1c6a709..0000000 --- a/src/lsb_serialize_protobuf.h +++ /dev/null @@ -1,24 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Lua sandbox Heka protobuf serialization implementation @file */ - -#ifndef lsb_serialize_protobuf_h_ -#define lsb_serialize_protobuf_h_ - -#include "luasandbox.h" - -/** - * Serialize a specific Lua table structure as Protobuf - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero on failure. - */ -int lsb_serialize_table_as_pb(lua_sandbox* lsb, int index); - -#endif diff --git a/src/luasandbox.c b/src/luasandbox.c index 4981553..7ae2c03 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -19,24 +19,10 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" #include "luasandbox/lualib.h" -#include "luasandbox_output.h" -#include "lsb_private.h" +#include "util/output_buffer.h" +#include "luasandbox_impl.h" #include "luasandbox_serialize.h" -#include "lsb_serialize_protobuf.h" - -static const char* standard_config = "{" - "memory_limit = %u," - "instruction_limit = %u," - "output_limit = %u," - "path = [[%s]]," - "cpath = [[%s]]," - "remove_entries = {" - "[''] = {'collectgarbage','coroutine','dofile','load','loadfile'" - ",'loadstring','newproxy','print'}," - "os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'}" - "}," - "disable_modules = {io = 1}" - "}"; + static jmp_buf g_jbuf; @@ -50,16 +36,16 @@ static const luaL_Reg preload_module_list[] = { { NULL, NULL } }; -static int libsize(const luaL_Reg* l) +static int libsize(const luaL_Reg *l) { int size = 0; for (; l->name; l++) size++; return size; } -static void preload_modules(lua_State* lua) +static void preload_modules(lua_State *lua) { - const luaL_Reg* lib = preload_module_list; + const luaL_Reg *lib = preload_module_list; luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", libsize(preload_module_list)); for (; lib->func; lib++) { @@ -77,35 +63,35 @@ static void preload_modules(lua_State* lua) * * See: http://www.lua.org/manual/5.1/manual.html#lua_Alloc * -* @param ud Pointer to the lua_sandbox +* @param ud Pointer to the lsb_lua_sandbox * @param ptr Pointer to the memory block being allocated/reallocated/freed. * @param osize The original size of the memory block. * @param nsize The new size of the memory block. * * @return void* A pointer to the memory block. */ -static void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize) +static void* memory_manager(void *ud, void *ptr, size_t osize, size_t nsize) { - lua_sandbox* lsb = (lua_sandbox*)ud; + lsb_lua_sandbox *lsb = (lsb_lua_sandbox *)ud; - void* nptr = NULL; + void *nptr = NULL; if (nsize == 0) { free(ptr); lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= osize; } else { size_t new_state_memory = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize; + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize; if (0 == lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] || new_state_memory <= lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]) { nptr = realloc(ptr, nsize); if (nptr != NULL) { lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = - new_state_memory; + new_state_memory; if (lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] > lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; } } } @@ -115,13 +101,13 @@ static void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize) #endif -static size_t instruction_usage(lua_sandbox* lsb) +static size_t instruction_usage(lsb_lua_sandbox *lsb) { return lua_gethookcount(lsb->lua) - lua_gethookcountremaining(lsb->lua); } -static void instruction_manager(lua_State* lua, lua_Debug* ar) +static void instruction_manager(lua_State *lua, lua_Debug *ar) { if (LUA_HOOKCOUNT == ar->event) { luaL_error(lua, "instruction_limit exceeded"); @@ -129,13 +115,12 @@ static void instruction_manager(lua_State* lua, lua_Debug* ar) } -static int output(lua_State* lua) +static int output(lua_State *lua) { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == lsb) { return luaL_error(lua, "output() invalid lightuserdata"); } - lua_sandbox* lsb = (lua_sandbox*)luserdata; int n = lua_gettop(lua); if (n == 0) { @@ -146,7 +131,21 @@ static int output(lua_State* lua) } -static int unprotected_panic(lua_State* lua) +static int read_config(lua_State *lua) +{ + luaL_checkstring(lua, 1); + luaL_argcheck(lua, lua_gettop(lua) == 1, 0, "too many arguments"); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + lua_getfield(lua, -1, lua_tostring(lua, 1)); + } else { + lua_pushnil(lua); + } + return 1; +} + + +static int unprotected_panic(lua_State *lua) { (void)lua; longjmp(g_jbuf, 1); @@ -154,96 +153,203 @@ static int unprotected_panic(lua_State* lua) } -static size_t get_usage_config(lua_State* lua, int idx, const char* item) +static int get_usage_config(lua_State *lua, int idx, const char *item) { lua_getfield(lua, idx, item); - size_t u = (size_t)lua_tointeger(lua, -1); // defaults to zero + int i = lua_tointeger(lua, -1); lua_pop(lua, 1); - return u; + return i; } -static int expand_path(const char* path, int n, const char* fmt, char* opath) +static int check_string(lua_State *L, int idx, const char *name, + const char *dflt) { - // not an exhaustive check, just making sure the Lua markers are accounted for - for (int i = 0; (path[i]); ++i) { - if (path[i] == '\'' || path[i] == ';' || path[i] == '?' - || !isprint(path[i])) { - return 1; + lua_getfield(L, idx, name); + int t = lua_type(L, -1); + switch (t) { + case LUA_TSTRING: + break; + case LUA_TNIL: + if (dflt) { + lua_pushstring(L, dflt); // add the default to the config + lua_setglobal(L, name); } + break; + default: + lua_pushfstring(L, "%s must be set to a string", name); + return 1; } - int result = snprintf(opath, n, fmt, path); - if (result < 0 || result > n - 1) { + return 0; +} + + +static int check_int(lua_State *L, int idx, const char *name, int val) +{ + lua_getfield(L, idx, name); + int t = lua_type(L, -1); + lua_Integer i; + switch (t) { + case LUA_TNUMBER: + i = lua_tointeger(L, -1); + if (i < 0) { + lua_pushfstring(L, "%s must be set to a positive integer", name); + return 1; + } + if (i > INT_MAX) { + lua_pushfstring(L, "%s cannot exceed INT_MAX %d", name, INT_MAX); + return 1; + } + break; + case LUA_TNIL: // add the default to the config + lua_pushinteger(L, val); + lua_setglobal(L, name); + break; // use the default + default: + lua_pushfstring(L, "%s must be set to a number", name); return 1; + break; } + lua_pop(L, 1); return 0; } -lua_sandbox* lsb_create(void* parent, - const char* lua_file, - const char* require_path, - unsigned memory_limit, - unsigned instruction_limit, - unsigned output_limit) +static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) { - if (!lua_file) { + lua_State *L = luaL_newstate(); + if (!L) { + if (logger) logger(__FUNCTION__, 3, "lua_State creation failed"); return NULL; } - char config[1024 * 2]; - char lpath[260] = { 0 }; - char cpath[260] = { 0 }; + if (!cfg) cfg = ""; // use the default settings - if (require_path) { -#if defined(_WIN32) - if (expand_path(require_path, sizeof(lpath), "%s\\?.lua", lpath)) { - return NULL; - } - if (expand_path(require_path, sizeof(cpath), "%s\\?.dll", cpath)) { - return NULL; - } -#else - if (expand_path(require_path, sizeof(lpath), "%s/?.lua", lpath)) { - return NULL; - } - if (expand_path(require_path, sizeof(cpath), "%s/?.so", cpath)) { - return NULL; - } -#endif - } + int ret = luaL_dostring(L, cfg); + if (ret) goto cleanup; - int result = snprintf(config, sizeof(config), standard_config, memory_limit, - instruction_limit, output_limit, lpath, cpath); + ret = check_int(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); + if (ret) goto cleanup; - if (result < 0 || result > (int)sizeof(config) - 1) { + ret = check_int(L, LUA_GLOBALSINDEX, LSB_MEMORY_LIMIT, 8 * 1024 * 1024); + if (ret) goto cleanup; + + ret = check_int(L, LUA_GLOBALSINDEX, LSB_INSTRUCTION_LIMIT, 1000000); + if (ret) goto cleanup; + + ret = check_string(L, LUA_GLOBALSINDEX, LSB_LUA_PATH, NULL); + if (ret) goto cleanup; + + ret = check_string(L, LUA_GLOBALSINDEX, LSB_LUA_CPATH, NULL); + if (ret) goto cleanup; + +cleanup: + if (ret) { + if (logger) { + logger(__FUNCTION__, 3, "config error: %s", lua_tostring(L, -1)); + } + lua_close(L); return NULL; } - - return lsb_create_custom(parent, lua_file, config); + return L; } -lua_sandbox* lsb_create_custom(void* parent, - const char* lua_file, - const char* config) +static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) { - if (!lua_file) { - return NULL; + lua_newtable(sb); + lua_pushnil(cfg); + while (lua_next(cfg, -2) != 0) { + int kt = lua_type(cfg, -2); + int vt = lua_type(cfg, -1); + switch (kt) { + case LUA_TNUMBER: + case LUA_TSTRING: + switch (vt) { + case LUA_TSTRING: + { + size_t len; + const char *tmp = lua_tolstring(cfg, -1, &len); + if (tmp) { + lua_pushlstring(sb, tmp, len); + if (kt == LUA_TSTRING) { + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + } else { + lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + } + } + } + break; + case LUA_TNUMBER: + lua_pushnumber(sb, lua_tonumber(cfg, -1)); + if (kt == LUA_TSTRING) { + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + } else { + lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + } + break; + case LUA_TBOOLEAN: + lua_pushboolean(sb, lua_toboolean(cfg, -1)); + if (kt == LUA_TSTRING) { + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + } else { + lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + } + break; + case LUA_TTABLE: + copy_table(sb, cfg, logger); + break; + default: + if (logger) { + logger(__FUNCTION__, 4, "skipping config value type: %s", + lua_typename(cfg, vt)); + } + break; + } + break; + default: + if (logger) { + logger(__FUNCTION__, 4, "skipping config key type: %s", + lua_typename(cfg, kt)); + } + break; + } + lua_pop(cfg, 1); } - bool seeded = false; + switch (lua_type(cfg, -2)) { + case LUA_TSTRING: + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + break; + case LUA_TNUMBER: + lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + break; + } +} + + +static bool set_tz() +{ #if _WIN32 if (_putenv("TZ=UTC") != 0) { - return NULL; + return false; } - // todo use CryptGenRandom to seed srand #else if (setenv("TZ", "UTC", 1) != 0) { - return NULL; + return false; } +#endif + return true; +} + - FILE* fh = fopen("/dev/urandom", "r"); +static void set_random_seed() +{ + bool seeded = false; +#ifdef _WIN32 + // todo use CryptGenRandom to seed srand +#else + FILE *fh = fopen("/dev/urandom", "r"); if (fh) { unsigned seed; unsigned char advance; @@ -262,12 +368,33 @@ lua_sandbox* lsb_create_custom(void* parent, if (!seeded) { srand((unsigned)time(NULL)); } +} - lua_sandbox* lsb = malloc(sizeof(lua_sandbox)); - memset(lsb->usage, 0, sizeof(lsb->usage)); + +lsb_lua_sandbox* lsb_create(void *parent, + const char *lua_file, + const char *cfg, + lsb_logger logger) +{ + if (!lua_file) { + if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + return NULL; + } + + if (!set_tz()) { + if (logger) logger(__FUNCTION__, 3, "fail to set the TZ to UTC"); + return NULL; + } + + set_random_seed(); + + lsb_lua_sandbox *lsb = malloc(sizeof*lsb); if (!lsb) { + if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); return NULL; } + memset(lsb->usage, 0, sizeof(lsb->usage)); + #ifdef LUA_JIT lsb->lua = luaL_newstate(); #else @@ -275,44 +402,46 @@ lua_sandbox* lsb_create_custom(void* parent, #endif if (!lsb->lua) { + if (logger) logger(__FUNCTION__, 3, "lua state creation failed"); free(lsb); return NULL; } - // create config table - lua_pushfstring(lsb->lua, "return %s", config); - if (luaL_dostring(lsb->lua, lua_tostring(lsb->lua, -1)) - || lua_type(lsb->lua, -1) != LUA_TTABLE) { + // add the config to the lsb_config registry table + lua_State *lua_cfg = load_sandbox_config(cfg, logger); + if (!lua_cfg) { lua_close(lsb->lua); - lsb->lua = NULL; free(lsb); return NULL; } - size_t memory_limit = get_usage_config(lsb->lua, -1, "memory_limit"); - size_t instruction_limit = get_usage_config(lsb->lua, -1, "instruction_limit"); - size_t output_limit = get_usage_config(lsb->lua, -1, "output_limit"); - lua_setfield(lsb->lua, LUA_REGISTRYINDEX, "lsb_config"); - lua_pop(lsb->lua, 1); // remove configuration string + lua_pushnil(lua_cfg); + lua_pushvalue(lua_cfg, LUA_GLOBALSINDEX); + copy_table(lsb->lua, lua_cfg, logger); + lua_pop(lua_cfg, 2); + lua_close(lua_cfg); + size_t ml = get_usage_config(lsb->lua, -1, "memory_limit"); + size_t il = get_usage_config(lsb->lua, -1, "instruction_limit"); + size_t ol = get_usage_config(lsb->lua, -1, "output_limit"); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + lua_pushcclosure(lsb->lua, &read_config, 0); + lua_setglobal(lsb->lua, "read_config"); + + lua_pushlightuserdata(lsb->lua, (void *)lsb); + lua_pushcclosure(lsb->lua, &output, 1); + lua_setglobal(lsb->lua, "output"); lsb->parent = parent; - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = memory_limit; - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] = instruction_limit; - lsb->usage[LSB_UT_OUTPUT][LSB_US_LIMIT] = output_limit; + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = ml; + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] = il; + lsb->usage[LSB_UT_OUTPUT][LSB_US_LIMIT] = ol; lsb->state = LSB_UNKNOWN; lsb->error_message[0] = 0; - lsb->output.pos = 0; - lsb->output.maxsize = output_limit; - if (output_limit && output_limit < LSB_OUTPUT_SIZE) { - lsb->output.size = output_limit; - } else { - lsb->output.size = LSB_OUTPUT_SIZE; - } - lsb->output.data = malloc(lsb->output.size); - size_t len = strlen(lua_file); - lsb->lua_file = malloc(len + 1); + lsb->lua_file = malloc(strlen(lua_file) + 1); + lsb->state_file = NULL; - if (!lsb->output.data || !lsb->lua_file) { - free(lsb->output.data); + if (!lsb->lua_file || lsb_init_output_buffer(&lsb->output, ol)) { + if (logger) logger(__FUNCTION__, 3, "memory allocation failed failed"); + lsb_free_output_buffer(&lsb->output); free(lsb->lua_file); lua_close(lsb->lua); lsb->lua = NULL; @@ -324,11 +453,26 @@ lua_sandbox* lsb_create_custom(void* parent, } -int lsb_init(lua_sandbox* lsb, const char* data_file) +int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) { if (!lsb) { - return 0; + return 1; } + + if (lsb->state != LSB_UNKNOWN) { + lsb_terminate(lsb, "already initialized"); + return 1; + } + + if (state_file && strlen(state_file) > 0) { + lsb->state_file = malloc(strlen(state_file) + 1); + if (!lsb->state_file) { + lsb_terminate(lsb, "memory allocation failure"); + return 1; + } + strcpy(lsb->state_file, state_file); + } + #ifndef LUA_JIT size_t mem_limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; @@ -359,10 +503,6 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) return 1; } - lua_pushlightuserdata(lsb->lua, (void*)lsb); - lua_pushcclosure(lsb->lua, &output, 1); - lua_setglobal(lsb->lua, "output"); - if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, (int)lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); @@ -389,15 +529,17 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) } else { lua_gc(lsb->lua, LUA_GCCOLLECT, 0); lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = - (unsigned)instruction_usage(lsb); + (unsigned)instruction_usage(lsb); if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; } lsb->state = LSB_RUNNING; - if (data_file != NULL && strlen(data_file) > 0) { - if (lsb_restore_global_data(lsb, data_file)) return 3; + if (lsb->state_file != NULL) { + if (lsb_restore_global_data(lsb)) { + return 3; + } } } lua_atpanic(lsb->lua, pf); @@ -405,31 +547,30 @@ int lsb_init(lua_sandbox* lsb, const char* data_file) } -char* lsb_destroy(lua_sandbox* lsb, const char* data_file) +char* lsb_destroy(lsb_lua_sandbox *lsb) { - char* err = NULL; + char *err = NULL; if (!lsb) { return err; } - if (lsb->lua && data_file && strlen(data_file) > 0) { - if (lsb_preserve_global_data(lsb, data_file) != 0) { - size_t len = strlen(lsb->error_message); - err = malloc(len + 1); - if (err != NULL) { - strcpy(err, lsb->error_message); - } + if (lsb_preserve_global_data(lsb) != 0) { + size_t len = strlen(lsb->error_message); + err = malloc(len + 1); + if (err != NULL) { + strcpy(err, lsb->error_message); } } lsb_terminate(lsb, NULL); - free(lsb->output.data); + lsb_free_output_buffer(&lsb->output); + free(lsb->state_file); free(lsb->lua_file); free(lsb); return err; } -size_t lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, +size_t lsb_usage(lsb_lua_sandbox *lsb, lsb_usage_type utype, lsb_usage_stat ustat) { if (!lsb || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { @@ -453,7 +594,7 @@ size_t lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, } -const char* lsb_get_error(lua_sandbox* lsb) +const char* lsb_get_error(lsb_lua_sandbox *lsb) { if (lsb) { return lsb->error_message; @@ -462,7 +603,7 @@ const char* lsb_get_error(lua_sandbox* lsb) } -void lsb_set_error(lua_sandbox* lsb, const char* err) +void lsb_set_error(lsb_lua_sandbox *lsb, const char *err) { if (lsb) { if (err) { @@ -475,25 +616,25 @@ void lsb_set_error(lua_sandbox* lsb, const char* err) } -lua_State* lsb_get_lua(lua_sandbox* lsb) +lua_State* lsb_get_lua(lsb_lua_sandbox *lsb) { return lsb->lua; } -const char* lsb_get_lua_file(lua_sandbox* lsb) +const char* lsb_get_lua_file(lsb_lua_sandbox *lsb) { return lsb->lua_file; } -void* lsb_get_parent(lua_sandbox* lsb) +void* lsb_get_parent(lsb_lua_sandbox *lsb) { return lsb->parent; } -lsb_state lsb_get_state(lua_sandbox* lsb) +lsb_state lsb_get_state(lsb_lua_sandbox *lsb) { if (lsb) { return lsb->state; @@ -502,28 +643,16 @@ lsb_state lsb_get_state(lua_sandbox* lsb) } -void lsb_add_function(lua_sandbox* lsb, lua_CFunction func, - const char* func_name) +void lsb_add_function(lsb_lua_sandbox *lsb, lua_CFunction func, + const char *func_name) { - lua_pushlightuserdata(lsb->lua, (void*)lsb); + lua_pushlightuserdata(lsb->lua, (void *)lsb); lua_pushcclosure(lsb->lua, func, 1); lua_setglobal(lsb->lua, func_name); } -const char* lsb_get_output(lua_sandbox* lsb, size_t* len) -{ - if (len) { - *len = lsb->output.pos; - } - if (lsb->output.pos == 0) return ""; - - lsb->output.pos = 0; - return lsb->output.data; -} - - -int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name) +int lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name) { if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, @@ -545,19 +674,19 @@ int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name) } -void lsb_pcall_teardown(lua_sandbox* lsb) +void lsb_pcall_teardown(lsb_lua_sandbox *lsb) { lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = - (unsigned)instruction_usage(lsb); + (unsigned)instruction_usage(lsb); if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; } } -void lsb_terminate(lua_sandbox* lsb, const char* err) +void lsb_terminate(lsb_lua_sandbox *lsb, const char *err) { if (err) { strncpy(lsb->error_message, err, LSB_ERROR_SIZE); diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h new file mode 100644 index 0000000..084ac8f --- /dev/null +++ b/src/luasandbox_impl.h @@ -0,0 +1,31 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua sandbox private functions @file */ + +#ifndef luasandbox_impl_h_ +#define luasandbox_impl_h_ + +#include "luasandbox.h" +#include "luasandbox/lua.h" +#include "util/output_buffer.h" + +#ifdef _WIN32 +#define snprintf _snprintf +#endif + +struct lsb_lua_sandbox { + lua_State *lua; + void *parent; + char *lua_file; + char *state_file; + lsb_state state; + lsb_output_buffer output; + size_t usage[LSB_UT_MAX][LSB_US_MAX]; + char error_message[LSB_ERROR_SIZE]; +}; + +#endif diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c new file mode 100644 index 0000000..181551c --- /dev/null +++ b/src/luasandbox_output.c @@ -0,0 +1,119 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox output buffer implementation @file */ + +#include "luasandbox_output.h" + +#include +#include +#include +#include + +#include "luasandbox/lauxlib.h" +#include "luasandbox_impl.h" +#include "luasandbox_serialize.h" + +static const char *output_function = "lsb_output"; + +void lsb_add_output_function(lua_State *lua, lua_CFunction fp) +{ + lua_pushstring(lua, output_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +lua_CFunction lsb_get_output_function(lua_State *lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, output_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + + +void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) +{ + if (!append) { + lsb->output.pos = 0; + } + + int result = 0; + for (int i = start; result == 0 && i <= end; ++i) { + switch (lua_type(lsb->lua, i)) { + case LUA_TNUMBER: + if (lsb_outputd(&lsb->output, lua_tonumber(lsb->lua, i))) { + result = 1; + } + break; + case LUA_TSTRING: + { + size_t len; + const char *s = lua_tolstring(lsb->lua, i, &len); + if (lsb_outputs(&lsb->output, s, len)) { + result = 1; + } + } + break; + case LUA_TNIL: + if (lsb_outputs(&lsb->output, "nil", 3)) { + result = 1; + } + break; + case LUA_TBOOLEAN: + if (lsb_outputf(&lsb->output, "%s", lua_toboolean(lsb->lua, i) + ? "true" : "false")) { + result = 1; + } + break; + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_output_function(lsb->lua, i); + if (!fp) { + luaL_argerror(lsb->lua, i, "unknown userdata type"); + return; // never reaches here but the compiler doesn't know it + } + lua_pushvalue(lsb->lua, i); + lua_pushlightuserdata(lsb->lua, &lsb->output); + result = fp(lsb->lua); + lua_pop(lsb->lua, 2); // remove the copy of the value and the output + } + break; + default: + luaL_argerror(lsb->lua, i, "unsupported type"); + break; + } + } + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = lsb->output.pos; + if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] + > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + } + + if (result != 0) { + if (lsb->error_message[0] == 0) { + luaL_error(lsb->lua, "output_limit exceeded"); + } + luaL_error(lsb->lua, lsb->error_message); + } +} + + +const char* lsb_get_output(lsb_lua_sandbox *lsb, size_t *len) +{ + if (len) { + *len = lsb->output.pos; + } + if (lsb->output.pos == 0) return ""; + + lsb->output.pos = 0; + return lsb->output.buf; +} diff --git a/src/lsb_serialize.c b/src/luasandbox_serialize.c similarity index 69% rename from src/lsb_serialize.c rename to src/luasandbox_serialize.c index 3bdef0e..5ecaa3d 100644 --- a/src/lsb_serialize.c +++ b/src/luasandbox_serialize.c @@ -14,7 +14,7 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lualib.h" -#include "lsb_private.h" +#include "luasandbox_impl.h" #ifdef _MSC_VER #pragma warning( disable : 4056 ) @@ -22,7 +22,7 @@ typedef struct { - const void* ptr; + const void *ptr; size_t name_pos; } table_ref; @@ -30,19 +30,19 @@ typedef struct { size_t size; size_t pos; - table_ref* array; + table_ref *array; } table_ref_array; typedef struct { - FILE* fh; - lsb_output_data keys; + FILE *fh; + lsb_output_buffer keys; table_ref_array tables; - const void* globals; + const void *globals; } serialization_data; -static const char* preservation_version = "_PRESERVATION_VERSION"; -static const char* serialize_function = "lsb_serialize"; +static const char *preservation_version = "_PRESERVATION_VERSION"; +static const char *serialize_function = "lsb_serialize"; /** * Serializes a Lua table structure. @@ -54,7 +54,7 @@ static const char* serialize_function = "lsb_serialize"; * @return int Zero on success, non-zero on failure. */ static int -serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent); +serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); /** * Serializes a Lua data value. @@ -65,7 +65,9 @@ serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent); * * @return int */ -static int serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output); +static int serialize_data(lsb_lua_sandbox *lsb, + int index, + lsb_output_buffer *output); /** * Serializes a table key value pair. @@ -77,7 +79,7 @@ static int serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output); * @return int Zero on success, non-zero on failure. */ static int -serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent); +serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); /** * Returns the serialization function if the userdata implemented one. @@ -87,7 +89,7 @@ serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent); * * @return lua_CFunction NULL if not found */ -static lua_CFunction get_serialize_function(lua_State* lua, int index) +static lua_CFunction get_serialize_function(lua_State *lua, int index) { lua_CFunction fp = NULL; lua_getfenv(lua, index); @@ -109,8 +111,8 @@ static lua_CFunction get_serialize_function(lua_State* lua, int index) * @return int */ static int -ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index, - lua_CFunction* fp) +ignore_value_type(lsb_lua_sandbox *lsb, serialization_data *data, int index, + lua_CFunction *fp) { switch (lua_type(lsb->lua, index)) { case LUA_TTABLE: @@ -147,7 +149,7 @@ ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index, * * @return table_ref* NULL if not found. */ -static table_ref* find_table_ref(table_ref_array* tra, const void* ptr) +static table_ref* find_table_ref(table_ref_array *tra, const void *ptr) { for (size_t i = 0; i < tra->pos; ++i) { if (ptr == tra->array[i].ptr) { @@ -168,11 +170,11 @@ static table_ref* find_table_ref(table_ref_array* tra, const void* ptr) * @return table_ref* Pointer to the table reference or NULL if out of memory. */ static table_ref* -add_table_ref(table_ref_array* tra, const void* ptr, size_t name_pos) +add_table_ref(table_ref_array *tra, const void *ptr, size_t name_pos) { if (tra->pos == tra->size) { size_t newsize = tra->size * 2; - void* p = realloc(tra->array, newsize * sizeof(table_ref)); + void *p = realloc(tra->array, newsize * sizeof(table_ref)); if (p != NULL) { tra->array = p; tra->size = newsize; @@ -193,7 +195,7 @@ add_table_ref(table_ref_array* tra, const void* ptr, size_t name_pos) * * @return int Version numebr */ -static int get_preservation_version(lua_State* lua) +static int get_preservation_version(lua_State *lua) { int ver = 0; lua_getglobal(lua, preservation_version); @@ -212,7 +214,7 @@ static int get_preservation_version(lua_State* lua) static int -serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent) +serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) { int result = 0; lua_checkstack(lsb->lua, 2); @@ -227,7 +229,7 @@ serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent) static int -serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output) +serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *output) { output->pos = 0; switch (lua_type(lsb->lua, index)) { @@ -252,7 +254,7 @@ serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output) lua_pushstring(lsb->lua, "%q"); lua_pushvalue(lsb->lua, index - 3); if (lua_pcall(lsb->lua, 2, 1, 0) == 0) { - if (lsb_appendf(output, "%s", lua_tostring(lsb->lua, -1))) { + if (lsb_outputf(output, "%s", lua_tostring(lsb->lua, -1))) { lua_pop(lsb->lua, 1); // Remove the string table. return 1; } @@ -270,9 +272,8 @@ serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output) lua_pop(lsb->lua, 2); // Remove the pcall result and the string table. break; case LUA_TBOOLEAN: - if (lsb_appendf(output, "%s", - lua_toboolean(lsb->lua, index) - ? "true" : "false")) { + if (lsb_outputf(output, "%s", lua_toboolean(lsb->lua, index) + ? "true" : "false")) { return 1; } break; @@ -287,7 +288,7 @@ serialize_data(lua_sandbox* lsb, int index, lsb_output_data* output) static int -serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) +serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) { int kindex = -2, vindex = -1, result = 0; lua_CFunction fp = NULL; @@ -299,19 +300,19 @@ serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) } size_t pos = data->keys.pos; - if (lsb_appendf(&data->keys, "%s[%s]", data->keys.data + parent, - lsb->output.data)) { + if (lsb_outputf(&data->keys, "%s[%s]", data->keys.buf + parent, + lsb->output.buf)) { return 1; } if (lua_type(lsb->lua, vindex) == LUA_TTABLE) { - const void* ptr = lua_topointer(lsb->lua, vindex); - table_ref* seen = find_table_ref(&data->tables, ptr); + const void *ptr = lua_topointer(lsb->lua, vindex); + table_ref *seen = find_table_ref(&data->tables, ptr); if (seen == NULL) { seen = add_table_ref(&data->tables, ptr, pos); if (seen != NULL) { data->keys.pos += 1; - fprintf(data->fh, "%s = {}\n", data->keys.data + pos); + fprintf(data->fh, "%s = {}\n", data->keys.buf + pos); result = serialize_table(lsb, data, pos); } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, @@ -319,55 +320,60 @@ serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) return 1; } } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); + fprintf(data->fh, "%s = ", data->keys.buf + pos); data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + seen->name_pos); + fprintf(data->fh, "%s\n", data->keys.buf + seen->name_pos); } } else if (lua_type(lsb->lua, vindex) == LUA_TUSERDATA) { - void* ud = lua_touserdata(lsb->lua, vindex); - table_ref* seen = find_table_ref(&data->tables, ud); + void *ud = lua_touserdata(lsb->lua, vindex); + table_ref *seen = find_table_ref(&data->tables, ud); if (seen == NULL) { seen = add_table_ref(&data->tables, ud, pos); if (seen != NULL) { data->keys.pos += 1; - lua_pushlightuserdata(lsb->lua, data->keys.data + pos); + lua_pushlightuserdata(lsb->lua, data->keys.buf + pos); lua_pushlightuserdata(lsb->lua, &lsb->output); lsb->output.pos = 0; result = fp(lsb->lua); lua_pop(lsb->lua, 2); // remove the key and the output if (result == 0) { - size_t n = fwrite(lsb->output.data, 1, lsb->output.pos, data->fh); + size_t n = fwrite(lsb->output.buf, 1, lsb->output.pos, data->fh); if (n != lsb->output.pos) { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_serialize failed %s", data->keys.data + pos); + "lsb_serialize failed %s", data->keys.buf + pos); return 1; } } } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_serialize out of memory %s", data->keys.data + pos); + "lsb_serialize out of memory %s", data->keys.buf + pos); return 1; } } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); + fprintf(data->fh, "%s = ", data->keys.buf + pos); data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + + fprintf(data->fh, "%s\n", data->keys.buf + seen->name_pos); } } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); + fprintf(data->fh, "%s = ", data->keys.buf + pos); data->keys.pos = pos; result = serialize_data(lsb, vindex, &lsb->output); if (result == 0) { - fprintf(data->fh, "%s\n", lsb->output.data); + fprintf(data->fh, "%s\n", lsb->output.buf); } } return result; } -int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) +int lsb_preserve_global_data(lsb_lua_sandbox *lsb) { + + if (!lsb->lua || !lsb->state_file) { + return 0; + } + // make sure the string library is loaded before we start lua_getglobal(lsb->lua, LUA_STRLIBNAME); if (!lua_istable(lsb->lua, -1)) { @@ -384,11 +390,11 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) lua_pushvalue(lsb->lua, LUA_GLOBALSINDEX); - FILE* fh = fopen(data_file, "wb"); + FILE *fh = fopen(lsb->state_file, "wb"); if (fh == NULL) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_preserve_global_data could not open: %s", - data_file); + lsb->state_file); if (len >= LSB_ERROR_SIZE || len < 0) { lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } @@ -398,7 +404,6 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) int result = 0; serialization_data data; data.fh = fh; - data.keys.maxsize = 0; // Clear the sandbox limits during preservation. #ifdef LUA_JIT @@ -413,13 +418,10 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) lsb->output.maxsize = 0; // end clear - data.keys.size = LSB_OUTPUT_SIZE; - data.keys.pos = 0; - data.keys.data = malloc(data.keys.size); data.tables.size = 64; data.tables.pos = 0; data.tables.array = malloc(data.tables.size * sizeof(table_ref)); - if (data.tables.array == NULL || data.keys.data == NULL) { + if (data.tables.array == NULL || lsb_init_output_buffer(&data.keys, 0)) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_preserve_global_data out of memory"); result = 1; @@ -428,7 +430,7 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) preservation_version, preservation_version, get_preservation_version(lsb->lua)); - lsb_appends(&data.keys, "_G", 2); + lsb_outputs(&data.keys, "_G", 2); data.keys.pos += 1; data.globals = lua_topointer(lsb->lua, -1); lua_checkstack(lsb->lua, 2); @@ -442,10 +444,10 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) // was added the stack should only contain table _G. } free(data.tables.array); - free(data.keys.data); + lsb_free_output_buffer(&data.keys); fclose(fh); if (result != 0) { - remove(data_file); + remove(lsb->state_file); } // Uncomment if we start preserving state when not destroying the sandbox @@ -477,9 +479,9 @@ int lsb_preserve_global_data(lua_sandbox* lsb, const char* data_file) } -static int file_exists(const char* fn) +static int file_exists(const char *fn) { - FILE* fh = fopen(fn, "r"); + FILE *fh = fopen(fn, "r"); if (fh) { fclose(fh); return 1; @@ -488,9 +490,9 @@ static int file_exists(const char* fn) } -int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) +int lsb_restore_global_data(lsb_lua_sandbox *lsb) { - if (!file_exists(data_file)) { + if (!lsb || !lsb->state_file || !file_exists(lsb->state_file)) { return 0; } @@ -504,7 +506,7 @@ int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) lua_sethook(lsb->lua, NULL, 0, 0); int err = 0; - if ((err = luaL_dofile(lsb->lua, data_file)) != 0) { + if ((err = luaL_dofile(lsb->lua, lsb->state_file)) != 0) { if (LUA_ERRFILE != err) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_restore_global_data %s", @@ -523,13 +525,13 @@ int lsb_restore_global_data(lua_sandbox* lsb, const char* data_file) lua_gc(lsb->lua, LUA_GCCOLLECT, 0); lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; #endif return 0; } -void lsb_add_serialize_function(lua_State* lua, lua_CFunction fp) +void lsb_add_serialize_function(lua_State *lua, lua_CFunction fp) { lua_pushstring(lua, serialize_function); lua_pushcfunction(lua, fp); @@ -537,131 +539,42 @@ void lsb_add_serialize_function(lua_State* lua, lua_CFunction fp) } -int lsb_serialize_binary(const void* src, size_t len, lsb_output_data* output) +int lsb_serialize_binary(lsb_output_buffer *ob, const void *src, size_t len) { - const char* uc = (const char*)src; + const char *uc = (const char *)src; for (unsigned i = 0; i < len; ++i) { switch (uc[i]) { case '\n': - if (lsb_appends(output, "\\n", 2)) return 1; + if (lsb_outputs(ob, "\\n", 2)) return 1; break; case '\r': - if (lsb_appends(output, "\\r", 2)) return 1; + if (lsb_outputs(ob, "\\r", 2)) return 1; break; case '"': - if (lsb_appends(output, "\\\"", 2)) return 1; + if (lsb_outputs(ob, "\\\"", 2)) return 1; break; case '\\': - if (lsb_appends(output, "\\\\", 2)) return 1; + if (lsb_outputs(ob, "\\\\", 2)) return 1; break; default: - if (lsb_appendc(output, uc[i])) return 1; + if (lsb_outputc(ob, uc[i])) return 1; break; } } return 0; } -static int fast_double(lsb_output_data* output, double d) -{ - if (d > INT_MAX) { - return lsb_appendf(output, "%0.17g", d); - } - - const int precision = 8; - const unsigned magnitude = 100000000; - char buffer[20]; - char* p = buffer; - int negative = 0; - - if (d < 0) { - negative = 1; - d = -d; - } - - int number = (int)d; - double tmp = (d - number) * magnitude; - unsigned fraction = (unsigned)tmp; - double diff = tmp - fraction; - - if (diff > 0.5) { - ++fraction; - if (fraction >= magnitude) { - fraction = 0; - ++number; - } - } else if (diff == 0.5 && ((fraction == 0) || (fraction & 1))) { - // bankers rounding - ++fraction; - } - - // output decimal fraction - if (fraction != 0) { - int nodigits = 1; - char c = 0; - for (int x = 0; x < precision; ++x) { - c = fraction % 10; - if (!(c == 0 && nodigits)) { - *p++ = c + '0'; - nodigits = 0; - } - fraction /= 10; - } - *p++ = '.'; - } - - // output number - do { - *p++ = (number % 10) + '0'; - number /= 10; - } - while (number > 0); - - size_t remaining = output->size - output->pos; - size_t needed = (p - buffer) + negative; - if (needed >= remaining) { - if (lsb_realloc_output(output, needed)) return 1; - } - - if (negative) { - output->data[output->pos++] = '-'; - } - do { - --p; - output->data[output->pos++] = *p; - } - while (p != buffer); - output->data[output->pos] = 0; - - return 0; -} - - -int lsb_serialize_double(lsb_output_data* output, double d) -{ - if (isnan(d)) { - return lsb_appends(output, "0/0", 3); - } - if (d == INFINITY) { - return lsb_appends(output, "1/0", 3); - } - if (d == -INFINITY) { - return lsb_appends(output, "-1/0", 4); - } - return fast_double(output, d); -} - -int lsb_output_double(lsb_output_data* output, double d) +int lsb_serialize_double(lsb_output_buffer *ob, double d) { if (isnan(d)) { - return lsb_appends(output, "nan", 3); + return lsb_outputs(ob, "0/0", 3); } if (d == INFINITY) { - return lsb_appends(output, "inf", 3); + return lsb_outputs(ob, "1/0", 3); } if (d == -INFINITY) { - return lsb_appends(output, "-inf", 4); + return lsb_outputs(ob, "-1/0", 4); } - return fast_double(output, d); + return lsb_outputfd(ob, d); } diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 836d2ae..3160e97 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -2,10 +2,10 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -add_executable(test_lua_sandbox test_lua_sandbox.c) -target_link_libraries(test_lua_sandbox luasandbox) +add_executable(test_luasandbox test_luasandbox.c) +target_link_libraries(test_luasandbox luasandbox) -add_test(NAME test_move_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) +add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) if(WIN32) add_test(NAME test_move_lua_lib COMMAND cmake -E copy ${CMAKE_BINARY_DIR}/ep_base/bin/${CMAKE_SHARED_MODULE_PREFIX}luasb${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) @@ -18,9 +18,9 @@ elseif(APPLE) ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_LIBRARY_PREFIX}luasandbox${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/../lib) endif() -add_test(NAME test_move_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) -add_test(NAME test_move_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -add_test(NAME test_sandbox COMMAND test_lua_sandbox) +add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) +add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +add_test(NAME test_luasandbox COMMAND test_luasandbox) if(APPLE) set_tests_properties(test_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=../lib) endif() diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 2de5bac..a2b63e0 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -3,76 +3,14 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "circular_buffer" -require "cjson" -require "hyperloglog" local cbuf = circular_buffer.new(1440, 3, 60) -local benchmark = {Timestamp = 1e9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} function process(tc) if tc == 0 then -- lua types write_output(1.2, " string ", nil, " ", true, " ", false) elseif tc == 1 then -- cbuf write_output("annotation\n", cbuf) - elseif tc == 2 then -- heka message - local hm = {Uuid = "ignored", Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9} - write_message(hm) - elseif tc == 3 then -- heka message field - local hm = {Timestamp = 1e9, Fields = {count=1}} - write_message(hm) - elseif tc == 4 then -- heka message field array - local hm = {Timestamp = 1e9, Fields = {counts={2,3,4}}} - write_message(hm) - elseif tc == 5 then -- hekamessage field metadata - local hm = {Timestamp = 1e9, Fields = {count={value=5,representation="count"}}} - write_message(hm) - elseif tc == 6 then -- heka message field metadata array - local hm = {Timestamp = 1e9, Fields = {counts={value={6,7,8},representation="count"}}} - write_message(hm) - elseif tc == 7 then -- heka message field all types - local hm = benchmark - write_message(hm) - elseif tc == 8 then -- heka message force memmove - local hm = {Timestamp = 1e9, Fields = {string="0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"}} - write_message(hm) - elseif tc == 9 then -- heka negative values - local hm = {Timestamp = -1, Pid = -1, Severity = -1} - write_message(hm) - elseif tc == 10 then -- heka varint encoding - local hm = {Timestamp = 1e9, Fields = { count = {value=1, value_type=2, representation="integer"}}} - write_message(hm) - elseif tc == 11 then -- heka varint packed encoding - local hm = {Timestamp = 1e9, Fields = { count = {value={1,2}, value_type=2, representation="integer"}}} - write_message(hm) - elseif tc == 12 then -- heka field array message - local hm = {Timestamp = 1e9, Fields = {{name = "count", value=1, value_type=2, representation="integer"}}} - write_message(hm) - elseif tc == 13 then -- heka field array message multiple fields - local hm = {Timestamp = 1e9, Fields = {{name = "count", value=1, value_type=2, representation="integer"}, {name = "count", value=2, value_type=2, representation="integer"}}} - write_message(hm) - elseif tc == 14 then -- heka field array message value defaults to a double - local hm = {Timestamp = 1e9, Fields = {{name = "count", value=1, representation="double"}}} - write_message(hm) - elseif tc == 15 then -- heka field array message value defaults to a double packed - local hm = {Timestamp = 1e9, Fields = {{name = "count", value={1,1}, representation="double"}}} - write_message(hm) - elseif tc == 16 then -- heka array of byte fields - local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 1, representation="list"}}} - write_message(hm) - elseif tc == 17 then -- heka array of explicit string fields - local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 0, representation="list"}}} - write_message(hm) - elseif tc == 18 then -- heka array of byte fields no representation - local hm = {Timestamp = 1e9, Fields = {{name = "names", value={"s1","s2"}, value_type = 1}}} - write_message(hm) - elseif tc == 19 then - local hll = hyperloglog.new() - local hm = {Timestamp = 1e9, Fields = {hll = {value=hll, representation="hll"}}} - write_message(hm) - elseif tc == 20 then - local cb = circular_buffer.new(2, 1, 60) - local hm = {Timestamp = 1e9, Fields = {cb = {value=cb, representation="cbuf"}}} - write_message(hm) end return 0 end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 4aa9f47..c48dab5 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -14,66 +14,17 @@ function process(tc) escape = escape .. escape end output(escape) - elseif tc == 2 then -- heka error mis-match field array - local hm = {Timestamp = 1e9, Fields = {counts={2,"ten",4}}} - write_message(hm) - elseif tc == 3 then -- heka error nil field - local hm = {Timestamp = 1e9, Fields = {counts={}}} - write_message(hm) - elseif tc == 4 then -- unsupported userdata + elseif tc == 2 then -- unsupported userdata write_output(lpeg.P"") - elseif tc == 5 then -- overflow + elseif tc == 3 then -- overflow local cb = circular_buffer.new(1000, 1, 60); write_output(cb) - elseif tc == 6 then -- invalid array type - local hm = {Timestamp = 1e9, Fields = {counts={{1},{2}}}} - write_message(hm) - elseif tc == 7 then -- overflow cjson encode buffer + elseif tc == 4 then -- overflow cjson encode buffer local t = {} for i=1, 10 do t[#t+1] = "this is a test" end cjson.encode(t) - elseif tc == 8 then -- invalid type - write_message("string") - elseif tc == 9 then -- invalid value_type integer - local hm = {Timestamp = 1e9, Fields = {counts={value="s", value_type=2}}} - write_message(hm) - elseif tc == 10 then -- invalid value_type double - local hm = {Timestamp = 1e9, Fields = {counts={value="s", value_type=3}}} - write_message(hm) - elseif tc == 11 then -- invalid value_type bool - local hm = {Timestamp = 1e9, Fields = {counts={value="s", value_type=4}}} - write_message(hm) - elseif tc == 12 then -- invalid value_type string - local hm = {Timestamp = 1e9, Fields = {counts={value=1, value_type=0}}} - write_message(hm) - elseif tc == 13 then -- invalid value_type bytes - local hm = {Timestamp = 1e9, Fields = {counts={value=1, value_type=1}}} - write_message(hm) - elseif tc == 14 then -- invalid value_type bool - local hm = {Timestamp = 1e9, Fields = {counts={value=1, value_type=4}}} - write_message(hm) - elseif tc == 15 then -- invalid value_type string - local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=0}}} - write_message(hm) - elseif tc == 16 then -- invalid value_type bytes - local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=1}}} - write_message(hm) - elseif tc == 17 then -- invalid value_type integer - local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}} - write_message(hm) - elseif tc == 18 then -- invalid value_type double - local hm = {Timestamp = 1e9, Fields = {counts={value=true, value_type=3}}} - write_message(hm) - elseif tc == 19 then -- bloom_filter doesn't implement lsb_output - local bf = bloom_filter.new(10, 0.01) - local hm = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}} - write_message(hm) - elseif tc == 20 then -- cuckoo_filter doesn't implement lsb_output - local cf = cuckoo_filter.new(10) - local hm = {Timestamp = 1e9, Fields = {cf = {value=cf, representation="cf"}}} - write_message(hm) end return 0 end diff --git a/src/test/lua/read_config.lua b/src/test/lua/read_config.lua new file mode 100644 index 0000000..a72779e --- /dev/null +++ b/src/test/lua/read_config.lua @@ -0,0 +1,17 @@ +assert(type(read_config) == "function") +assert(read_config("memory_limit") == 65765) +assert(read_config("instruction_limit") == 1000) +assert(read_config("output_limit") == 1024) + +local array = read_config("array") +assert(type(array) == "table") +assert(array[1] == "foo") +assert(array[2] == 99) + +local hash = read_config("hash") +assert(type(hash) == "table") +assert(hash.foo == "bar") +assert(hash.hash1.subfoo == "subbar") + +assert(type(read_config("path")) == "string") +assert(type(read_config("cpath")) == "string") diff --git a/src/test/lua/sandbox_config.lua b/src/test/lua/sandbox_config.lua index 1631aaf..796fa94 100644 --- a/src/test/lua/sandbox_config.lua +++ b/src/test/lua/sandbox_config.lua @@ -29,6 +29,7 @@ local tests = { getfenv=1, getmetatable=1, ipairs=1, + read_config=1, math=1, module=1, next=1, diff --git a/src/test/mu_test.h b/src/test/mu_test.h new file mode 100644 index 0000000..c338ac8 --- /dev/null +++ b/src/test/mu_test.h @@ -0,0 +1,66 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Unit test macros and global data @file */ + +#ifndef luasandbox_test_mu_test_h_ +#define luasandbox_test_mu_test_h_ + +#include + +#ifdef _WIN32 +#define snprintf _snprintf +#endif + +#ifdef _MSC_VER +#define PRIuSIZE "Iu" +#else +#define PRIuSIZE "zu" +#endif + +#define MU_ERR_LEN 1024 + +#define mu_assert(cond, ...) \ +do { \ + if (!(cond)) { \ + int cnt = snprintf(mu_err, MU_ERR_LEN, "line: %d (%s) ", __LINE__, #cond); \ + if (cnt > 0 && cnt < MU_ERR_LEN) { \ + cnt = snprintf(mu_err+cnt, MU_ERR_LEN-cnt, __VA_ARGS__); \ + if (cnt > 0 && cnt < MU_ERR_LEN) { \ + return mu_err; \ + } \ + } \ + mu_err[MU_ERR_LEN - 1] = 0; \ + return mu_err; \ + } \ +} while (0) + +#define mu_assert_rv(rv, fn) \ +do { \ + int result = fn; \ + if (rv != result) { \ + int cnt = snprintf(mu_err, MU_ERR_LEN, "line: %d %s expected: %d " \ + " received: %d", __LINE__, #fn, rv, result); \ + if (cnt > 0 && cnt < MU_ERR_LEN) { \ + return mu_err; \ + } \ + mu_err[MU_ERR_LEN - 1] = 0; \ + return mu_err; \ + } \ +} while (0) + +#define mu_run_test(test) \ +do { \ + char *message = test(); \ + mu_tests_run++; \ + if (message) \ + return message; \ +} while (0) + +int mu_tests_run = 0; +char mu_err[MU_ERR_LEN] = { 0 }; + +#endif \ No newline at end of file diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index bd5e432..b7d5c76 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -5,6 +5,9 @@ _G["dataRef"]:set_header(1, "Column_1", "count", "sum") _G["dataRef"]:set_header(2, "Column_2", "count", "sum") _G["dataRef"]:set_header(3, "Column_3", "count", "sum") _G["dataRef"]:fromstring("2 2 nan nan nan nan nan nan nan nan nan") +if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end +_G["delta"]:set_header(1, "Column_1", "count", "sum") +_G["delta"]:fromstring("1 1 2 nan 0 2") _G["rate"] = 0.12345678 _G["kvp"] = {} _G["kvp"]["a"] = "foo" @@ -16,9 +19,7 @@ _G["kvp"]["r"][4] = 92.002 _G["kvp"]["r"][5] = 91.10001 _G["kvp"]["r"]["key"] = "val" _G["kvp"]["b"] = "bar" -if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end -_G["delta"]:set_header(1, "Column_1", "count", "sum") -_G["delta"]:fromstring("1 1 2 nan 0 2") +_G["data"] = _G["dataRef"] _G["uuids"] = {} _G["uuids"][1] = {} _G["uuids"][1]["type"] = "test" @@ -27,8 +28,6 @@ _G["uuids"][2] = {} _G["uuids"][2]["type"] = "test1" _G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" _G["inf"] = 1/0 -_G["rates"] = _G["kvp"]["r"] -_G["data"] = _G["dataRef"] _G["cycleb"] = {} _G["cycleb"]["a"] = {} _G["cycleb"]["a"]["b"] = _G["cycleb"] @@ -70,6 +69,7 @@ _G["nested"]["cb"]:set_header(4, "Column_4", "count", "sum") _G["nested"]["cb"]:set_header(5, "Column_5", "count", "sum") _G["nested"]["cb"]:set_header(6, "Column_6", "count", "sum") _G["nested"]["cb"]:fromstring("1 1 nan nan nan nan nan nan nan nan nan nan nan nan") +_G["rates"] = _G["kvp"]["r"] _G["count"] = 0 -_G["nan"] = 0/0 _G["_VERSION"] = "Lua 5.1" +_G["nan"] = 0/0 diff --git a/src/test/test_lua_sandbox.c b/src/test/test_luasandbox.c similarity index 65% rename from src/test/test_lua_sandbox.c rename to src/test/test_luasandbox.c index 309934f..5edfc98 100644 --- a/src/test/test_lua_sandbox.c +++ b/src/test/test_luasandbox.c @@ -17,50 +17,37 @@ #include "luasandbox/lua.h" #include "luasandbox_output.h" -#ifdef _WIN32 -#define snprintf _snprintf -#endif +#include "mu_test.h" + +char *e = NULL; +const char *written_data = NULL; +size_t written_data_len = 0; -#ifdef _MSC_VER -#define PRIuSIZE "Iu" + +#ifdef _WIN32 +#define MODULE_PATH "path = 'modules\\?.lua';cpath = 'modules\\?.dll'\n" #else -#define PRIuSIZE "zu" +#define MODULE_PATH "path = 'modules/?.lua';cpath = 'modules/?.so'\n" #endif -#define mu_assert(cond, ...) \ -do { \ - if (!(cond)) { \ - int cnt = snprintf(mu_err, MU_ERR_LEN, "line: %d (%s) ", __LINE__, #cond); \ - if (cnt > 0 && cnt < MU_ERR_LEN) { \ - cnt = snprintf(mu_err+cnt, MU_ERR_LEN-cnt, __VA_ARGS__); \ - if (cnt > 0 && cnt < MU_ERR_LEN) { \ - return mu_err; \ - } \ - } \ - mu_err[MU_ERR_LEN - 1] = 0; \ - return mu_err; \ - } \ -} while (0) - -#define mu_run_test(test) \ -do { \ - char *message = test(); \ - mu_tests_run++; \ - if (message) \ - return message; \ -} while (0) - -#define MU_ERR_LEN 1024 -int mu_tests_run = 0; -char mu_err[MU_ERR_LEN] = { 0 }; -char* e = NULL; -const char* written_data = NULL; -size_t written_data_len = 0; + +static const char *test_cfg = + "memory_limit = 0\n" + "instruction_limit = 0\n" + "output_limit = 0\n" + "remove_entries = {\n" + "[''] = {'collectgarbage','coroutine','dofile','load','loadfile'" + ",'loadstring','newproxy','print'},\n" + "os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'}\n" + "}\n" + "disable_modules = {io = 1}\n" + MODULE_PATH +; -int file_exists(const char* fn) +int file_exists(const char *fn) { - FILE* fh; + FILE *fh; fh = fopen(fn, "r"); if (fh) { fclose(fh); @@ -70,10 +57,10 @@ int file_exists(const char* fn) } -char* read_file(const char* fn) +char* read_file(const char *fn) { - char* str = NULL; - FILE* fh; + char *str = NULL; + FILE *fh; fh = fopen(fn, "rb"); mu_assert(fh != NULL, "fopen() failed"); @@ -96,10 +83,10 @@ char* read_file(const char* fn) } -int process(lua_sandbox* lsb, double ts) +int process(lsb_lua_sandbox *lsb, double ts) { - static const char* func_name = "process"; - lua_State* lua = lsb_get_lua(lsb); + static const char *func_name = "process"; + lua_State *lua = lsb_get_lua(lsb); if (!lua) return 1; if (lsb_pcall_setup(lsb, func_name)) return 1; @@ -157,10 +144,10 @@ int process(lua_sandbox* lsb, double ts) } -int report(lua_sandbox* lsb, double tc) +int report(lsb_lua_sandbox *lsb, double tc) { - static const char* func_name = "report"; - lua_State* lua = lsb_get_lua(lsb); + static const char *func_name = "report"; + lua_State *lua = lsb_get_lua(lsb); if (!lua) return 1; if (lsb_pcall_setup(lsb, func_name)) return 1; @@ -184,13 +171,13 @@ int report(lua_sandbox* lsb, double tc) } -int write_output(lua_State* lua) +int write_output(lua_State *lua) { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); + void *luserdata = lua_touserdata(lua, lua_upvalueindex(1)); if (NULL == luserdata) { luaL_error(lua, "write_output() invalid lightuserdata"); } - lua_sandbox* lsb = (lua_sandbox*)luserdata; + lsb_lua_sandbox *lsb = (lsb_lua_sandbox *)luserdata; int n = lua_gettop(lua); lsb_output(lsb, 1, n, 1); @@ -199,27 +186,31 @@ int write_output(lua_State* lua) } -int write_message(lua_State* lua) +static char* test_create_error() { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - luaL_error(lua, "write_message() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; + lsb_lua_sandbox *sb = lsb_create(NULL, NULL, NULL, NULL); + mu_assert(!sb, "lsb_create() null lua_file"); - if (lsb_output_protobuf(lsb, 1, 0) != 0) { - luaL_error(lua, "write_message() could not encode protobuf - %s", - lsb_get_error(lsb)); - } - written_data = lsb_get_output(lsb, &written_data_len); - return 0; + return NULL; } -static char* test_create_error() +static char* test_read_config() { - lua_sandbox* sb = lsb_create(NULL, NULL, "modules", 0, 0, 0); - mu_assert(!sb, "lsb_create() null lua_file"); + const char *cfg = "memory_limit = 65765\n" + "instruction_limit = 1000\n" + "output_limit = 1024\n" + "array = {'foo', 99}\n" + "hash = {foo = 'bar', hash1 = {subfoo = 'subbar'}}\n" + MODULE_PATH; + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/read_config.lua", cfg, NULL); + int result = lsb_init(sb, NULL); + mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_get_error(sb)); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; } @@ -229,41 +220,40 @@ static char* test_init_error() { // null sandbox int result = lsb_init(NULL, NULL); - mu_assert(result == 0, "lsb_init() null sandbox ptr"); + mu_assert(result == 1, "lsb_init() null sandbox ptr"); // load error - lua_sandbox* sb = lsb_create(NULL, "lua/simple1.lua", "modules", 0, - 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple1.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, NULL); mu_assert(result == 2, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_state s = lsb_get_state(sb); mu_assert(s == LSB_TERMINATED, "lsb_get_state() received: %d", s); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // out of memory - sb = lsb_create(NULL, "lua/simple.lua", "modules", 6000, 0, 0); + sb = lsb_create(NULL, "lua/simple.lua", "memory_limit = 6000", NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, NULL); mu_assert(result == 2, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s\n", e); - sb = lsb_create(NULL, "lua/lpeg_date_time.lua", NULL, 0, 0, 0); + sb = lsb_create(NULL, "lua/lpeg_date_time.lua", "path = '';cpath = ''", NULL); mu_assert(sb, "lsb_create() received: NULL"); // disabled external modules result = lsb_init(sb, NULL); mu_assert(result == 2, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - const char* expected = "lua/lpeg_date_time.lua:7: module 'date_time' not found:"; + const char *expected = "lua/lpeg_date_time.lua:7: module 'date_time' not found:"; mu_assert(strcmp(lsb_get_error(sb), expected) == 0, "lsb_get_error() received: %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -272,17 +262,17 @@ static char* test_init_error() static char* test_destroy_error() { - const char* expected = "lsb_preserve_global_data could not open: " - "invaliddir/simple.preserve"; - e = lsb_destroy(NULL, NULL); + const char *expected = "lsb_preserve_global_data could not open: " + "invaliddir/simple.preserve"; + e = lsb_destroy(NULL); mu_assert(!e, "lsb_destroy() received: %s", e); - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, "invaliddir/simple.preserve"); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, "invaliddir/simple.preserve"); + e = lsb_destroy(sb); mu_assert(e, "lsb_destroy() received NULL"); mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); @@ -298,7 +288,7 @@ static char* test_usage_error() size_t u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); mu_assert(u == 0, "NULL sandbox memory usage received: %" PRIuSIZE, u); - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); u = lsb_usage(NULL, LSB_UT_MAX + 1, LSB_US_CURRENT); @@ -312,19 +302,18 @@ static char* test_usage_error() u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); mu_assert(u == 0, "Terminated memory usage received: %" PRIuSIZE, u); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; } - static char* test_misc() { lsb_state s = lsb_get_state(NULL); mu_assert(s == LSB_UNKNOWN, "lsb_get_state() received: %d", s); - const char* le = lsb_get_error(NULL); + const char *le = lsb_get_error(NULL); mu_assert(strlen(le) == 0, "lsb_get_error() received: %s", le); return NULL; @@ -333,11 +322,13 @@ static char* test_misc() static char* test_simple() { - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", - 65765, 1000, 1024); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", + "memory_limit = 65765;" + "instruction_limit = 1000;" + "output_limit = 1024;", NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, "simple.preserve"); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); @@ -378,16 +369,17 @@ static char* test_simple() lsb_state s = lsb_get_state(sb); mu_assert(s == LSB_RUNNING, "lsb_get_state() received: %d", s); - e = lsb_destroy(sb, "simple.preserve"); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; } + static char* test_simple_error() { - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -412,7 +404,7 @@ static char* test_simple_error() mu_assert(result == 1, "process() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -421,77 +413,31 @@ static char* test_simple_error() static char* test_output() { - const char* outputs[] = { + const char *outputs[] = { "1.2 string nil true false" , "" - , "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65" - , "\x10\x80\x94\xeb\xdc\x03\x52\x12\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f" - , "\x10\x80\x94\xeb\xdc\x03\x52\x24\x0a\x06\x63\x6f\x75\x6e\x74\x73\x10\x03\x3a\x18\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x00\x00\x00\x00\x00\x00\x10\x40" - , "\x10\x80\x94\xeb\xdc\x03\x52\x19\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x39\x00\x00\x00\x00\x00\x00\x14\x40" - , "\x10\x80\x94\xeb\xdc\x03\x52\x2b\x0a\x06\x63\x6f\x75\x6e\x74\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\x18\x40\x00\x00\x00\x00\x00\x00\x1c\x40\x00\x00\x00\x00\x00\x00\x20\x40" -#ifdef LUA_JIT - , "\x10\x80\x94\xeb\xdc\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33" -#else - , "\x10\x80\x94\xeb\xdc\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33" -#endif - , "\x10\x80\x94\xeb\xdc\x03\x52\x8d\x01\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x82\x01\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" - , "\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x28\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x40\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" - , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01" - , "\x10\x80\x94\xeb\xdc\x03\x52\x16\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x32\x02\x01\x02" - , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01" - , "\x10\x80\x94\xeb\xdc\x03\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x01\x52\x14\x0a\x05\x63\x6f\x75\x6e\x74\x10\x02\x1a\x07\x69\x6e\x74\x65\x67\x65\x72\x30\x02" - , "\x10\x80\x94\xeb\xdc\x03\x52\x1a\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x06\x64\x6f\x75\x62\x6c\x65\x39\x00\x00\x00\x00\x00\x00\xf0\x3f" - , "\x10\x80\x94\xeb\xdc\x03\x52\x23\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x06\x64\x6f\x75\x62\x6c\x65\x3a\x10\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\xf0\x3f" - , "\x10\x80\x94\xeb\xdc\x03\x52\x17\x0a\x05\x6e\x61\x6d\x65\x73\x10\x01\x1a\x04\x6c\x69\x73\x74\x2a\x02\x73\x31\x2a\x02\x73\x32" - , "\x10\x80\x94\xeb\xdc\x03\x52\x15\x0a\x05\x6e\x61\x6d\x65\x73\x1a\x04\x6c\x69\x73\x74\x22\x02\x73\x31\x22\x02\x73\x32" - , "\x10\x80\x94\xeb\xdc\x03\x52\x11\x0a\x05\x6e\x61\x6d\x65\x73\x10\x01\x2a\x02\x73\x31\x2a\x02\x73\x32" - , "\x10\x80\x94\xeb\xdc\x03\x52\x9f\x60\x0a\x03\x68\x6c\x6c\x10\x01\x1a\x03\x68\x6c\x6c\x2a\x90\x60\x48\x59\x4c\x4c" - , "\x10\x80\x94\xeb\xdc\x03\x52\x93\x01\x0a\x02\x63\x62\x10\x01\x1a\x04\x63\x62\x75\x66\x2a\x84\x01{\"time\":0,\"rows\":2,\"columns\":1,\"seconds_per_row\":60,\"column_info\":[{\"name\":\"Column_1\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\nnan\n" , NULL }; - enum {output_size = 63 * 1024}; - - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 0, 0 - , output_size); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, "circular_buffer.preserve"); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); - lsb_add_function(sb, &write_message, "write_message"); for (int x = 0; outputs[x]; ++x) { result = process(sb, x); mu_assert(!result, "process() test: %d failed: %d %s", x, result, lsb_get_error(sb)); if (outputs[x][0]) { - if (outputs[x][0] == 0x10) { - size_t header = 18; - if (x == 19) { - mu_assert(written_data_len == 12346, "test: %d received: %" PRIuSIZE, x, written_data_len); - written_data_len = header + 28; // just compare the protobuf before the hyperloglog - } - if (memcmp(outputs[x], written_data + header, written_data_len - header) - != 0) { - char hex_data[output_size * 3 + 1]; - size_t z = 0; - for (size_t y = header; y < written_data_len; ++y, z += 3) { - snprintf(hex_data + z, output_size - z, "%02x ", - (unsigned char)written_data[y]); - } - hex_data[z] = 0; - mu_assert(0, "test: %d received: %s", x, hex_data); - } - } else { - mu_assert(strcmp(outputs[x], written_data) == 0, - "test: %d received: %s", x, written_data); - } + mu_assert(strcmp(outputs[x], written_data) == 0, + "test: %d received: %s", x, written_data); } } - e = lsb_destroy(sb, "circular_buffer.preserve"); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -500,51 +446,34 @@ static char* test_output() static char* test_output_errors() { - const char* tests[] = + const char *tests[] = { "process() lua/output_errors.lua:10: bad argument #1 to 'output' (unsupported type)" , "process() lua/output_errors.lua:16: output_limit exceeded" - , "process() lua/output_errors.lua:19: write_message() could not encode protobuf - array has mixed types" - , "process() lua/output_errors.lua:22: write_message() could not encode protobuf - unsupported type: nil" - , "process() lua/output_errors.lua:24: bad argument #1 to 'write_output' (unknown userdata type)" - , "process() lua/output_errors.lua:27: output_limit exceeded" - , "process() lua/output_errors.lua:30: write_message() could not encode protobuf - unsupported array type: table" - , "process() lua/output_errors.lua:36: strbuf output_limit exceeded" - , "process() lua/output_errors.lua:38: write_message() could not encode protobuf - takes a table argument" - , "process() lua/output_errors.lua:41: write_message() could not encode protobuf - invalid string value_type: 2" - , "process() lua/output_errors.lua:44: write_message() could not encode protobuf - invalid string value_type: 3" - , "process() lua/output_errors.lua:47: write_message() could not encode protobuf - invalid string value_type: 4" - , "process() lua/output_errors.lua:50: write_message() could not encode protobuf - invalid numeric value_type: 0" - , "process() lua/output_errors.lua:53: write_message() could not encode protobuf - invalid numeric value_type: 1" - , "process() lua/output_errors.lua:56: write_message() could not encode protobuf - invalid numeric value_type: 4" - , "process() lua/output_errors.lua:59: write_message() could not encode protobuf - invalid boolean value_type: 0" - , "process() lua/output_errors.lua:62: write_message() could not encode protobuf - invalid boolean value_type: 1" - , "process() lua/output_errors.lua:65: write_message() could not encode protobuf - invalid boolean value_type: 2" - , "process() lua/output_errors.lua:68: write_message() could not encode protobuf - invalid boolean value_type: 3" - , "process() lua/output_errors.lua:72: write_message() could not encode protobuf - user data object does not implement lsb_output" - , "process() lua/output_errors.lua:76: write_message() could not encode protobuf - user data object does not implement lsb_output" + , "process() lua/output_errors.lua:18: bad argument #1 to 'write_output' (unknown userdata type)" + , "process() lua/output_errors.lua:21: output_limit exceeded" + , "process() lua/output_errors.lua:27: strbuf output_limit exceeded" , NULL }; for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/output_errors.lua", "modules", - 0, 0, 128); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output_errors.lua", + MODULE_PATH "output_limit = 128", NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); - lsb_add_function(sb, &write_message, "write_message"); result = process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); - const char* le = lsb_get_error(sb); + const char *le = lsb_get_error(sb); mu_assert(le, "test: %d received NULL", i); mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } @@ -554,15 +483,15 @@ static char* test_output_errors() static char* test_cbuf_errors() { - const char* tests[] = + const char *tests[] = { "process() not enough memory" , NULL }; for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer_errors.lua", - "modules", 32767, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer_errors.lua", + MODULE_PATH "memory_limit = 32767", NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -572,11 +501,11 @@ static char* test_cbuf_errors() result = process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); - const char* le = lsb_get_error(sb); + const char *le = lsb_get_error(sb); mu_assert(le, "test: %d received NULL", i); mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } @@ -586,15 +515,15 @@ static char* test_cbuf_errors() static char* test_cbuf_core() { - lua_sandbox* sb = lsb_create(NULL, "../../ep_base/Source/lua_circular_buffer/test.lua", - "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, + "../../ep_base/Source/lua_circular_buffer/test.lua", + test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, ""); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, ""); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -603,7 +532,8 @@ static char* test_cbuf_core() static char* test_cbuf() { - const char* outputs[] = { + const char *output_file = "circular_buffer.preserve"; + const char *outputs[] = { "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\tnan\tnan\nnan\tnan\tnan\nnan\tnan\tnan\n" , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" , "{\"time\":2,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n3\t1\t3\nnan\tnan\tnan\n1\t1\t1\n" @@ -611,11 +541,11 @@ static char* test_cbuf() , NULL }; - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer.lua", "modules", - 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -650,7 +580,7 @@ static char* test_cbuf() mu_assert(strcmp(outputs[3], written_data) == 0, "received: %s", written_data); - e = lsb_destroy(sb, "circular_buffer.preserve"); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -659,7 +589,8 @@ static char* test_cbuf() static char* test_cbuf_delta() { - const char* outputs[] = { + const char *output_file = "circular_buffer_delta.preserve"; + const char *outputs[] = { "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" #ifdef LUA_JIT , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n0\t1\t1\t1\n1\t2\t1\t2\n2\t3\t1\t3\n" @@ -682,11 +613,12 @@ static char* test_cbuf_delta() , NULL }; - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer_delta.lua", - "modules", 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer_delta.lua", + test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -715,7 +647,7 @@ static char* test_cbuf_delta() written_data); } - e = lsb_destroy(sb, "circular_buffer_delta.preserve"); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -724,7 +656,8 @@ static char* test_cbuf_delta() static char* test_cjson() { - lua_sandbox* sb = lsb_create(NULL, "lua/cjson.lua", "modules", 0, 0, 64); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cjson.lua", + MODULE_PATH "output_limit = 64", NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -735,7 +668,7 @@ static char* test_cjson() mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -744,7 +677,7 @@ static char* test_cjson() static char* test_cjson_unlimited() { - lua_sandbox* sb = lsb_create(NULL, "lua/cjson_unlimited.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cjson_unlimited.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -758,7 +691,7 @@ static char* test_cjson_unlimited() mu_assert(written_data_len == 103001, "received %d bytes", (int)written_data_len); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -767,7 +700,7 @@ static char* test_cjson_unlimited() static char* test_errors() { - const char* tests[] = { + const char *tests[] = { #ifdef _WIN32 "process() lua/errors.lua:9: module 'unknown' not found:\n\tno file 'lua\\unknown.lua'\n\tno file 'lua\\unknown.dll'" #else @@ -796,8 +729,18 @@ static char* test_errors() }; for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/errors.lua", "lua", 32767, 1000, - 128); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/errors.lua", + "memory_limit = 32767;" + "instruction_limit = 1000;" + "output_limit = 128;" +#ifdef _WIN32 + "path = 'lua\\?.lua'" + "cpath = 'lua\\?.dll'", +#else + "path = 'lua/?.lua'" + "cpath = 'lua/?.so'", +#endif + NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -807,11 +750,11 @@ static char* test_errors() result = process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); - const char* le = lsb_get_error(sb); + const char *le = lsb_get_error(sb); mu_assert(le, "test: %d received NULL", i); mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } @@ -821,7 +764,7 @@ static char* test_errors() static char* test_lpeg() { - const char* tests[] = { + const char *tests[] = { "lua/lpeg.lua" , "lua/lpeg_clf.lua" , "lua/lpeg_cbufd.lua" @@ -836,7 +779,7 @@ static char* test_lpeg() }; for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, tests[i], "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, tests[i], test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -847,7 +790,7 @@ static char* test_lpeg() mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } @@ -858,7 +801,7 @@ static char* test_lpeg() static char* test_util() { - lua_sandbox* sb = lsb_create(NULL, "lua/util_test.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/util_test.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -869,7 +812,7 @@ static char* test_util() mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -878,23 +821,24 @@ static char* test_util() static char* test_serialize() { - const char* output_file = "serialize.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "modules", - 64000, 1000, 64000); + const char *output_file = "serialize.preserve"; + + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); #ifdef LUA_JIT - char* expected = read_file("output/serialize.data"); + char *expected = read_file("output/serialize.data"); #else - char* expected = read_file("output/serialize.lua51.data"); + char *expected = read_file("output/serialize.lua51.data"); #endif - char* actual = read_file(output_file); + char *actual = read_file(output_file); mu_assert(strcmp(expected, actual) == 0, "serialization mismatch"); free(expected); free(actual); @@ -905,12 +849,12 @@ static char* test_serialize() static char* test_restore() { - const char* output_file = "restore.preserve"; + const char *output_file = "restore.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/restore.lua", "modules", - 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -919,11 +863,11 @@ static char* test_restore() lsb_get_error(sb)); mu_assert(strcmp("101", written_data) == 0, "test: initial load received: %s", written_data); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/restore.lua", "modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -936,11 +880,11 @@ static char* test_restore() written_data); result = report(sb, 2); // change the preservation version mu_assert(result == 0, "report() received: %d", result); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data with a version change - sb = lsb_create(NULL, "lua/restore.lua", "modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -951,7 +895,7 @@ static char* test_restore() lsb_get_error(sb)); mu_assert(strcmp("101", written_data) == 0, "test: reload with version change received: %s", written_data); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -960,18 +904,18 @@ static char* test_restore() static char* test_serialize_failure() { - const char* output_file = "serialize_failure.preserve"; - const char* expected = "serialize_data cannot preserve type 'function'"; + const char *output_file = "serialize_failure.preserve"; + const char *expected = "serialize_data cannot preserve type 'function'"; - lua_sandbox* sb = lsb_create(NULL, - "lua/serialize_failure.lua", "modules", - 32767, 1000, 1024); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, + "lua/serialize_failure.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(e, "lsb_destroy() received: no error"); mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); free(e); @@ -984,15 +928,15 @@ static char* test_serialize_failure() static char* test_bloom_filter_core() { - lua_sandbox* sb = lsb_create(NULL, "../../ep_base/Source/lua_bloom_filter/test.lua", - "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, + "../../ep_base/Source/lua_bloom_filter/test.lua", + test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, ""); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1001,19 +945,19 @@ static char* test_bloom_filter_core() static char* test_bloom_filter() { - const char* output_file = "bloom_filter.preserve"; - const char* tests[] = { + const char *output_file = "bloom_filter.preserve"; + const char *tests[] = { "1" , "2" , "3" , NULL }; - lua_sandbox* sb = lsb_create(NULL, "lua/bloom_filter.lua", "modules", - 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/bloom_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -1037,11 +981,11 @@ static char* test_bloom_filter() mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, written_data); // count should remain the same - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/bloom_filter.lua", "modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/bloom_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); @@ -1070,7 +1014,7 @@ static char* test_bloom_filter() mu_assert(strcmp("1", written_data) == 0, "test: clear received: %s", written_data); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1079,30 +1023,29 @@ static char* test_bloom_filter() static char* test_hyperloglog_core() { - lua_sandbox* sb = lsb_create(NULL, "../../ep_base/Source/lua_hyperloglog/test.lua", - "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, + "../../ep_base/Source/lua_hyperloglog/test.lua", + test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, ""); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; } - static char* test_hyperloglog() { - const char* output_file = "hyperloglog.preserve"; + const char *output_file = "hyperloglog.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "modules", - 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -1124,11 +1067,11 @@ static char* test_hyperloglog() mu_assert(strcmp("100070", written_data) == 0, "test: cache received: %s", written_data); // count should remain the same - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/hyperloglog.lua", "modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); @@ -1158,7 +1101,7 @@ static char* test_hyperloglog() mu_assert(strcmp("0", written_data) == 0, "test: clear received: %s", written_data); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1167,8 +1110,7 @@ static char* test_hyperloglog() static char* test_struct() { - lua_sandbox* sb = lsb_create(NULL, "lua/struct.lua", "modules", - 65765, 1000, 1024); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/struct.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); @@ -1178,7 +1120,7 @@ static char* test_struct() result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1187,32 +1129,14 @@ static char* test_struct() static char* test_sandbox_config() { - lua_sandbox* sb = lsb_create(NULL, "lua/sandbox_config.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sandbox_config.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_decode_message() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/decode_message.lua", "modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_add_function(sb, &lsb_decode_protobuf, "decode_message"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1221,19 +1145,19 @@ static char* test_decode_message() static char* test_cuckoo_filter() { - const char* output_file = "cuckoo_filter.preserve"; - const char* tests[] = { + const char *output_file = "cuckoo_filter.preserve"; + const char *tests[] = { "1" , "2" , "3" , NULL }; - lua_sandbox* sb = lsb_create(NULL, "lua/cuckoo_filter.lua", "modules", - 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cuckoo_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -1257,11 +1181,11 @@ static char* test_cuckoo_filter() mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, written_data); // count should remain the same - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/cuckoo_filter.lua", "modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/cuckoo_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); @@ -1292,7 +1216,7 @@ static char* test_cuckoo_filter() mu_assert(strcmp("0", written_data) == 0, "test: clear received: %s", written_data); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1301,13 +1225,14 @@ static char* test_cuckoo_filter() static char* test_sax() { - const char* output_file = "sax.preserve"; - const char* word = "CDC CDC ABEGH ABEGH"; + const char *output_file = "sax.preserve"; + const char *word = "CDC CDC ABEGH ABEGH"; - lua_sandbox* sb = lsb_create(NULL, "lua/sax.lua", "modules", 0, 0, 0); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sax.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); lsb_add_function(sb, &write_output, "write_output"); @@ -1322,11 +1247,11 @@ static char* test_sax() result = report(sb, 0); mu_assert(result == 0, "report() received: %d", result); mu_assert(strcmp(word, written_data) == 0, "received: %s", written_data); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); // re-load to test the preserved data - sb = lsb_create(NULL, "lua/sax.lua", "modules", 0, 0, 0); + sb = lsb_create(NULL, "lua/sax.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); result = lsb_init(sb, output_file); @@ -1337,7 +1262,7 @@ static char* test_sax() report(sb, 0); mu_assert(strcmp(word, written_data) == 0, "received: %s", written_data); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; @@ -1348,8 +1273,7 @@ static char* benchmark_counter() { int iter = 10000000; - lua_sandbox* sb = lsb_create(NULL, "lua/counter.lua", "modules", 32000, - 10, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1359,9 +1283,9 @@ static char* benchmark_counter() process(sb, 0); } t = clock() - t; - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_counter() %g seconds\n", ((float)t) / CLOCKS_PER_SEC + printf("benchmark_counter() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1371,84 +1295,57 @@ static char* benchmark_counter() static char* benchmark_serialize() { int iter = 1000; - const char* output_file = "serialize.preserve"; + const char *output_file = "serialize.preserve"; clock_t t = clock(); for (int x = 0; x < iter; ++x) { - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "modules", - 64000, 1000, 1024); + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); + int result = lsb_init(sb, output_file); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } t = clock() - t; - printf("benchmark_serialize() %g seconds\n", ((float)t) / CLOCKS_PER_SEC + printf("benchmark_serialize() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; } -static char* benchmark_deserialize() +static char* benchmark_deserialize() // todo need to alter since this now performs the serialization too { int iter = 1000; clock_t t = clock(); for (int x = 0; x < iter; ++x) { - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, "output/serialize.data"); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } t = clock() - t; - printf("benchmark_deserialize() %g seconds\n", ((float)t) / CLOCKS_PER_SEC + printf("benchmark_deserialize() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; } -static char* benchmark_lpeg_decoder() -{ - int iter = 10000; - - lua_sandbox* sb = lsb_create(NULL, "lua/decoder.lua", "modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - lsb_add_function(sb, &write_message, "write_message"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_lpeg_decoder() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - static char* benchmark_lua_types_output() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", - 100000, 1000, 1024 * 63); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1460,9 +1357,9 @@ static char* benchmark_lua_types_output() mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); } t = clock() - t; - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_lua_types_output() %g seconds\n", ((float)t) + printf("benchmark_lua_types_output() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1473,7 +1370,7 @@ static char* benchmark_cbuf_output() { int iter = 10000; - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1485,34 +1382,9 @@ static char* benchmark_cbuf_output() mu_assert(0 == process(sb, 1), "%s", lsb_get_error(sb)); } t = clock() - t; - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cbuf_output() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_message_output() -{ - int iter = 1000000; - - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_message, "write_message"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 7), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_message_output() %g seconds\n", ((float)t) / CLOCKS_PER_SEC + printf("benchmark_cbuf_output() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1523,9 +1395,8 @@ static char* benchmark_cbuf_add() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, - "lua/circular_buffer_add.lua", "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, + "lua/circular_buffer_add.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1540,9 +1411,9 @@ static char* benchmark_cbuf_add() t = clock() - t; mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_cbuf_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cbuf_add() %g seconds\n", ((float)t) / CLOCKS_PER_SEC + printf("benchmark_cbuf_add() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1553,9 +1424,8 @@ static char* benchmark_bloom_filter_add() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, - "lua/bloom_filter_benchmark.lua", "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, + "lua/bloom_filter_benchmark.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1571,9 +1441,9 @@ static char* benchmark_bloom_filter_add() mu_assert(strcmp("999970", written_data) == 0, "received: %s", written_data); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_bloom_filter_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_bloom_filter_add() %g seconds\n", ((float)t) + printf("benchmark_bloom_filter_add() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1584,8 +1454,7 @@ static char* benchmark_hyperloglog_add() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1601,39 +1470,9 @@ static char* benchmark_hyperloglog_add() mu_assert(strcmp("1006268", written_data) == 0, "received: %s", written_data); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_hyperloglog_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_hyperloglog_add() %g seconds\n", ((float)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - -static char* benchmark_decode_message() -{ - int iter = 10000; - - lua_sandbox* sb = lsb_create(NULL, "lua/decode_message_benchmark.lua", "modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - lsb_add_function(sb, &lsb_decode_protobuf, "decode_message"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - report(sb, 0); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_decode_message() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_decode_message() %g seconds\n", ((float)t) + printf("benchmark_hyperloglog_add() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1644,9 +1483,9 @@ static char* benchmark_cuckoo_filter_add() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, - "lua/cuckoo_filter_benchmark.lua", "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, + "lua/cuckoo_filter_benchmark.lua", test_cfg, + NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1662,9 +1501,9 @@ static char* benchmark_cuckoo_filter_add() mu_assert(strcmp("999985", written_data) == 0, "received: %s", written_data); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_cuckoo_filter_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cuckoo_filter_add() %g seconds\n", ((float)t) + printf("benchmark_cuckoo_filter_add() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; @@ -1675,8 +1514,7 @@ static char* benchmark_sax_add() { int iter = 1000000; - lua_sandbox* sb = lsb_create(NULL, "lua/sax_benchmark.lua", "modules", - 0, 0, 0); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sax_benchmark.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); int result = lsb_init(sb, NULL); mu_assert(result == 0, "lsb_init() received: %d %s", result, @@ -1692,18 +1530,18 @@ static char* benchmark_sax_add() mu_assert(strcmp("ABEGH", written_data) == 0, "received: %s", written_data); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_sax_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); + e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_sax_add() %g seconds\n", ((float)t) + printf("benchmark_sax_add() %g seconds\n", ((double)t) / CLOCKS_PER_SEC / iter); return NULL; } - static char* all_tests() { + mu_run_test(test_read_config); mu_run_test(test_create_error); mu_run_test(test_init_error); mu_run_test(test_destroy_error); @@ -1731,21 +1569,17 @@ static char* all_tests() mu_run_test(test_hyperloglog); mu_run_test(test_struct); mu_run_test(test_sandbox_config); - mu_run_test(test_decode_message); mu_run_test(test_cuckoo_filter); mu_run_test(test_sax); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); mu_run_test(benchmark_deserialize); - mu_run_test(benchmark_lpeg_decoder); mu_run_test(benchmark_lua_types_output); mu_run_test(benchmark_cbuf_output); - mu_run_test(benchmark_message_output); mu_run_test(benchmark_cbuf_add); mu_run_test(benchmark_bloom_filter_add); mu_run_test(benchmark_hyperloglog_add); - mu_run_test(benchmark_decode_message); mu_run_test(benchmark_cuckoo_filter_add); mu_run_test(benchmark_sax_add); @@ -1755,7 +1589,7 @@ static char* all_tests() int main() { - char* result = all_tests(); + char *result = all_tests(); if (result) { printf("%s\n", result); } else { @@ -1764,5 +1598,5 @@ int main() printf("Tests run: %d\n", mu_tests_run); free(e); - return result != 0; + return result != NULL; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt new file mode 100644 index 0000000..eab9c23 --- /dev/null +++ b/src/util/CMakeLists.txt @@ -0,0 +1,22 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set(UTIL_SRC +heka_message.c +input_buffer.c +output_buffer.c +protobuf.c +running_stats.c +string.c +string_matcher.c +util.c +) + +add_library(luasandboxutil SHARED ${UTIL_SRC}) +set_target_properties(luasandboxutil PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) + +install(TARGETS luasandboxutil DESTINATION lib COMPONENT core) +add_subdirectory(test) + diff --git a/src/util/heka_message.c b/src/util/heka_message.c new file mode 100644 index 0000000..f46cb16 --- /dev/null +++ b/src/util/heka_message.c @@ -0,0 +1,538 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight Heka message implementation @file */ + +#include "util/heka_message.h" + +#include +#include + +#include "../luasandbox_impl.h" +#include "luasandbox_output.h" +#include "util/protobuf.h" + + +static size_t decode_header(char *buf, size_t len, size_t max_message_size) +{ + if (*buf != 0x08) { + return 0; + } + + char *p = buf; + if (p && p < buf + len - 1) { + long long vi; + if (lsb_pb_read_varint(p + 1, buf + len, &vi)) { + if (vi > 0 && vi <= (long long)max_message_size) { + return vi; + } + } + } + return 0; +} + + +static const char* +read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) +{ + if (wiretype != 2) { + return NULL; + } + + long long vi; + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || p + vi > e) { + return NULL; + } + s->s = p; + s->len = vi; + return p + vi; +} + + +static bool +read_string_value(const char *p, const char *e, int ai, lsb_read_value *val) +{ + int acnt = 0; + int tag = 0; + int wiretype = 0; + while (p && p < e) { + val->type = LSB_READ_NIL; + p = lsb_pb_read_key(p, &tag, &wiretype); + p = read_string(wiretype, p, e, &val->u.s); + if (p) { + if (ai == acnt++) { + val->type = LSB_READ_STRING; + return true; + } + } + } + return false; +} + + +static bool +read_integer_value(const char *p, const char *e, int ai, lsb_read_value *val) +{ + int acnt = 0; + long long ll = 0; + while (p && p < e) { + p = lsb_pb_read_varint(p, e, &ll); + if (p) { + if (ai == acnt++) { + val->type = LSB_READ_NUMERIC; + val->u.d = (double)ll; + return true; + } + } + } + return false; +} + + +static bool +read_double_value(const char *p, const char *e, int ai, lsb_read_value *val) +{ + if (p + (sizeof(double) * (ai + 1)) > e) { + return false; + } + val->type = LSB_READ_NUMERIC; + p += sizeof(double) * ai; + memcpy(&val->u.d, p, sizeof(double)); + return true; +} + + +static const char* +process_varint(int wiretype, const char *p, const char *e, long long *val) +{ + if (wiretype != 0) { + return NULL; + } + p = lsb_pb_read_varint(p, e, val); + return p ? p : NULL; +} + + +static const char* +process_fields(lsb_heka_field *f, const char *p, const char *e) +{ + int tag = 0; + int wiretype = 0; + long long vi = 0; + + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || p + vi > e) { + return NULL; + } + e = p + vi; // only process to the end of the current field record + + do { + p = lsb_pb_read_key(p, &tag, &wiretype); + + switch (tag) { + case LSB_PB_NAME: + p = read_string(wiretype, p, e, &f->name); + break; + + case LSB_PB_VALUE_TYPE: + p = process_varint(wiretype, p, e, &vi); + if (p) { + f->value_type = (int)vi; + } + break; + + case LSB_PB_REPRESENTATION: + p = read_string(wiretype, p, e, &f->representation); + break; + + // don't bother with the value(s) until we actually need them + // since this stream is created by Hindsight + // - tags are guaranteed to be properly ordered (values at the end) + // - there won't be repeated tags for packed values + case LSB_PB_VALUE_STRING: + case LSB_PB_VALUE_BYTES: + if (wiretype != 2) { + p = NULL; + break; + } + f->value.s = p - 1; + f->value.len = e - f->value.s; + p = e; + break; + + case LSB_PB_VALUE_INTEGER: + case LSB_PB_VALUE_BOOL: + if (wiretype != 0 && wiretype != 2) { + p = NULL; + break; + } + // fall thru + case LSB_PB_VALUE_DOUBLE: + if (tag == 7 && wiretype != 1 && wiretype != 2) { + p = NULL; + break; + } + if (wiretype == 2) { + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || p + vi > e) { + p = NULL; + break; + } + } + f->value.s = p; + f->value.len = e - f->value.s; + p = e; + break; + + default: + p = NULL; // don't allow unknown tags + break; + } + } while (p && p < e); + + return p && f->name.s ? p : NULL; +} + + +bool lsb_decode_heka_message(lsb_heka_message *m, + const char *buf, + size_t len, + lsb_logger logger) +{ + if (!buf || len == 0) { + if (logger) { + logger(__FUNCTION__, 4, "invalid buffer"); + } + return false; + } + + const char *cp = buf; // current position + const char *lp = buf; // last position + const char *ep = buf + len; // end position + int wiretype = 0; + int tag = 0; + long long val = 0; + bool timestamp = false; + + lsb_clear_heka_message(m); + + do { + cp = lsb_pb_read_key(cp, &tag, &wiretype); + + switch (tag) { + case LSB_PB_UUID: + cp = read_string(wiretype, cp, ep, &m->uuid); + if (m->uuid.len != LSB_UUID_SIZE) cp = NULL; + break; + + case LSB_PB_TIMESTAMP: + cp = process_varint(wiretype, cp, ep, &m->timestamp); + if (cp) timestamp = true; + break; + + case LSB_PB_TYPE: + cp = read_string(wiretype, cp, ep, &m->type); + break; + + case LSB_PB_LOGGER: + cp = read_string(wiretype, cp, ep, &m->logger); + break; + + case LSB_PB_SEVERITY: + cp = process_varint(wiretype, cp, ep, &val); + if (cp) m->severity = (int)val; + break; + + case LSB_PB_PAYLOAD: + cp = read_string(wiretype, cp, ep, &m->payload); + break; + + case LSB_PB_ENV_VERSION: + cp = read_string(wiretype, cp, ep, &m->env_version); + break; + + case LSB_PB_PID: + cp = process_varint(wiretype, cp, ep, &val); + if (cp) m->pid = (int)val; + break; + + case LSB_PB_HOSTNAME: + cp = read_string(wiretype, cp, ep, &m->hostname); + break; + + case LSB_PB_FIELDS: + if (wiretype != 2) { + cp = NULL; + break; + } + if (m->fields_len == m->fields_size) { + int step = 8; + m->fields_size += step; + lsb_heka_field *tmp = realloc(m->fields, + m->fields_size * sizeof(lsb_heka_field)); + if (!tmp) { + if (logger) logger(__FUNCTION__, 0, "fields reallocation failed"); + return false; + } + m->fields = tmp; + memset(&m->fields[m->fields_len], 0, step * sizeof(lsb_heka_field)); + } + cp = process_fields(&m->fields[m->fields_len], cp, ep); + ++m->fields_len; + break; + + default: + cp = NULL; + break; + } + if (cp) lp = cp; + } while (cp && cp < ep); + + if (!cp) { + if (logger) { + logger(__FUNCTION__, 4, "tag:%d wiretype:%d position:%td", tag, + wiretype, lp - buf); + } + return false; + } + + if (!m->uuid.s) { + if (logger) logger(__FUNCTION__, 4, "missing " LSB_UUID); + return false; + } + + if (!timestamp) { + if (logger) logger(__FUNCTION__, 4, "missing " LSB_TIMESTAMP); + return false; + } + + m->raw.s = buf; + m->raw.len = len; + return true; +} + + +bool lsb_find_heka_message(lsb_heka_message *m, + lsb_input_buffer *ib, + bool decode, + size_t *discarded_bytes, + lsb_logger logger) +{ + *discarded_bytes = 0; + if (ib->readpos == ib->scanpos) { + return false; // empty buffer + } + + char *p = memchr(&ib->buf[ib->scanpos], 0x1e, ib->readpos - ib->scanpos); + if (p) { + if (p != ib->buf + ib->scanpos) { + // partial buffer skipped before locating a possible header + *discarded_bytes += p - ib->buf - ib->scanpos; + } + ib->scanpos = p - ib->buf; + + if (ib->readpos - ib->scanpos < 2) { + return false; // header length is not buf + } + + size_t hlen = ib->buf[ib->scanpos + 1]; + size_t hend = ib->scanpos + hlen + 3; + if (hend > ib->readpos) { + return false; // header is not in buf + } + if (ib->buf[hend - 1] != 0x1f) { + // invalid header length + ++ib->scanpos; + ++*discarded_bytes; + size_t db; + bool b = lsb_find_heka_message(m, ib, decode, &db, logger); + *discarded_bytes += db; + return b; + } + + if (!ib->msglen) { + ib->msglen = decode_header(&ib->buf[ib->scanpos + 2], hlen, + ib->maxsize - LSB_MAX_HDR_SIZE); + } + + if (ib->msglen) { + size_t mend = hend + ib->msglen; + if (mend > ib->readpos) { + return false; // message is not in buf + } + + if (decode) { + if (lsb_decode_heka_message(m, &ib->buf[hend], ib->msglen, logger)) { + ib->scanpos = mend; + ib->msglen = 0; + return true; + } else { + // message decode failure + ++ib->scanpos; + ++*discarded_bytes; + ib->msglen = 0; + size_t db; + bool b = lsb_find_heka_message(m, ib, decode, &db, logger); + *discarded_bytes += db; + return b; + } + } else { + // allow a framed message is non Heka protobuf format + lsb_clear_heka_message(m); + m->raw.s = &ib->buf[hend]; + m->raw.len = ib->msglen; + ib->scanpos = mend; + ib->msglen = 0; + return true; + } + } else { + // header decode failure + ++ib->scanpos; + ++*discarded_bytes; + size_t db; + bool b = lsb_find_heka_message(m, ib, decode, &db, logger); + *discarded_bytes += db; + return b; + } + } else { + // full buffer skipped since no header was located + *discarded_bytes += ib->readpos - ib->scanpos; + ib->scanpos = ib->readpos = 0; + } + return false; +} + + +int lsb_init_heka_message(lsb_heka_message *m, int num_fields) +{ + if (num_fields < 1) return 1; + + m->fields = malloc(num_fields * sizeof(lsb_heka_field)); + if (!m->fields) return 2; + + m->fields_size = num_fields; + lsb_clear_heka_message(m); + return 0; +} + + +void lsb_clear_heka_message(lsb_heka_message *m) +{ + lsb_init_const_string(&m->raw); + lsb_init_const_string(&m->uuid); + lsb_init_const_string(&m->type); + lsb_init_const_string(&m->logger); + lsb_init_const_string(&m->payload); + lsb_init_const_string(&m->env_version); + lsb_init_const_string(&m->hostname); + + if (m->fields) memset(m->fields, 0, m->fields_size * sizeof(lsb_heka_field)); + m->timestamp = 0; + m->severity = 7; + m->pid = 0; + m->fields_len = 0; +} + + +void lsb_free_heka_message(lsb_heka_message *m) +{ + free(m->fields); + m->fields = NULL; + m->fields_size = 0; + lsb_clear_heka_message(m); +} + + +bool lsb_read_heka_field(lsb_heka_message *m, + lsb_const_string *name, + int fi, + int ai, + lsb_read_value *val) +{ + int fcnt = 0; + const char *p, *e; + val->type = LSB_READ_NIL; + + for (int i = 0; i < m->fields_len; ++i) { + if (name->len == m->fields[i].name.len + && strncmp(name->s, m->fields[i].name.s, m->fields[i].name.len) == 0) { + if (fi == fcnt++) { + p = m->fields[i].value.s; + e = p + m->fields[i].value.len; + switch (m->fields[i].value_type) { + case LSB_PB_STRING: + case LSB_PB_BYTES: + return read_string_value(p, e, ai, val); + case LSB_PB_INTEGER: + return read_integer_value(p, e, ai, val); + case LSB_PB_BOOL: + if (read_integer_value(p, e, ai, val)) { + val->type = LSB_READ_BOOL; + return true; + } + return false; + case LSB_PB_DOUBLE: + return read_double_value(p, e, ai, val); + default: + return false; + } + } + } + } + return false; +} + + +int lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) +{ + static const size_t needed = 18; + ob->pos = 0; // writing a uuid will always clear the buffer since it is the + // start of a new message + int result = lsb_expand_output_buffer(ob, needed); + if (result) return result; + + ob->buf[ob->pos++] = 2 | (LSB_PB_UUID << 3); // write key + ob->buf[ob->pos++] = LSB_UUID_SIZE; // write length + if (uuid && len == LSB_UUID_SIZE) { + memcpy(ob->buf + ob->pos, uuid, LSB_UUID_SIZE); + ob->pos += LSB_UUID_SIZE; + } else if (uuid && len == LSB_UUID_STR_SIZE) { + int cnt = sscanf(uuid, "%02hhx%02hhx%02hhx%02hhx" + "-%02hhx%02hhx" + "-%02hhx%02hhx" + "-%02hhx%02hhx" + "-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx", + (unsigned char *)ob->buf + ob->pos, + (unsigned char *)ob->buf + ob->pos + 1, + (unsigned char *)ob->buf + ob->pos + 2, + (unsigned char *)ob->buf + ob->pos + 3, + (unsigned char *)ob->buf + ob->pos + 4, + (unsigned char *)ob->buf + ob->pos + 5, + (unsigned char *)ob->buf + ob->pos + 6, + (unsigned char *)ob->buf + ob->pos + 7, + (unsigned char *)ob->buf + ob->pos + 8, + (unsigned char *)ob->buf + ob->pos + 9, + (unsigned char *)ob->buf + ob->pos + 10, + (unsigned char *)ob->buf + ob->pos + 11, + (unsigned char *)ob->buf + ob->pos + 12, + (unsigned char *)ob->buf + ob->pos + 13, + (unsigned char *)ob->buf + ob->pos + 14, + (unsigned char *)ob->buf + ob->pos + 15); + if (cnt == LSB_UUID_SIZE) { + ob->pos += cnt; + } + } + + if (ob->pos == 2) { // only the header has been written + for (int x = 0; x < LSB_UUID_SIZE; ++x) { + ob->buf[ob->pos++] = rand() % 256; + } + ob->buf[8] = (ob->buf[8] & 0x0F) | 0x40; + ob->buf[10] = (ob->buf[10] & 0x0F) | 0xA0; + } + return 0; +} diff --git a/src/util/input_buffer.c b/src/util/input_buffer.c new file mode 100644 index 0000000..bdf8b3f --- /dev/null +++ b/src/util/input_buffer.c @@ -0,0 +1,72 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream input buffer implementation @file */ + +#include "util/input_buffer.h" + +#include +#include +#include +#include + +#include "util/util.h" +#include "util/heka_message.h" + +int lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size) +{ + b->buf = NULL; + if (max_message_size == 0) return 1; + max_message_size += LSB_MAX_HDR_SIZE; + b->size = max_message_size < BUFSIZ ? max_message_size : BUFSIZ; + b->maxsize = max_message_size; + b->readpos = 0; + b->scanpos = 0; + b->msglen = 0; + b->buf = malloc(b->size); + return b->buf ? 0 : 2; +} + + +void lsb_free_input_buffer(lsb_input_buffer *b) +{ + free(b->buf); + b->buf = NULL; + b->size = 0; + b->readpos = 0; + b->scanpos = 0; + b->msglen = 0; +} + + +int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) +{ + if (b->scanpos != 0) { // shift the data to the beginning of the buffer + if (b->scanpos == b->readpos) { + b->scanpos = b->readpos = 0; + } else { + memmove(b->buf, b->buf + b->scanpos, b->readpos - b->scanpos); + b->readpos = b->readpos - b->scanpos; + b->scanpos = 0; + } + } + + if (b->readpos + len > b->size) { + size_t newsize = b->readpos + len; + if (newsize > b->maxsize) return 1; + + newsize = lsb_lp2(newsize); + if (newsize > b->maxsize) newsize = b->maxsize;; + char *tmp = realloc(b->buf, newsize); + if (tmp) { + b->buf = tmp; + b->size = newsize; + } else { + return 2; + } + } + return 0; +} diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c new file mode 100644 index 0000000..5dbc217 --- /dev/null +++ b/src/util/output_buffer.c @@ -0,0 +1,223 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream output buffer implementation @file */ + +#include "util/output_buffer.h" + +#include +#include +#include +#include + +#include "util/util.h" + + +int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) +{ + if (max_message_size && max_message_size < LSB_OUTPUT_SIZE) { + b->size = max_message_size; + } else { + b->size = LSB_OUTPUT_SIZE; + } + b->maxsize = max_message_size; + b->pos = 0; + b->buf = malloc(b->size); + return b->buf ? 0 : 2; +} + + +void lsb_free_output_buffer(lsb_output_buffer *b) +{ + free(b->buf); + b->buf = NULL; + b->size = 0; + b->pos = 0; +} + + +int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) +{ + if (needed <= b->size - b->pos) { + return 0; + } + + if (b->maxsize && needed + b->pos > b->maxsize) { + return 1; + } + + size_t newsize = lsb_lp2(b->pos + needed); + if (b->maxsize && newsize > b->maxsize) { + newsize = b->maxsize; + } + + void *ptr = realloc(b->buf, newsize); + if (!ptr) return 2; + + b->buf = ptr; + b->size = newsize; + return 0; +} + + +int lsb_outputc(lsb_output_buffer *b, char ch) +{ + int result = lsb_expand_output_buffer(b, 2); + if (result) return result; + + b->buf[b->pos++] = ch; + b->buf[b->pos] = 0; + return 0; +} + + +int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) +{ + va_list args; + int remaining = 0; + char *ptr = NULL, *old_ptr = NULL; + do { + ptr = b->buf + b->pos; + remaining = (int)(b->size - b->pos); + va_start(args, fmt); + int needed = vsnprintf(ptr, remaining, fmt, args); + va_end(args); + if (needed == -1) { + // Windows and Unix have different return values for this function + // -1 on Unix is a format error + // -1 on Windows means the buffer is too small and the required len + // is not returned + needed = remaining; + } + if (needed >= remaining) { + if (b->maxsize + && (b->size >= b->maxsize + || b->pos + needed >= b->maxsize)) { + return 1; // exceeded max + } + size_t newsize = b->size * 2; + while ((size_t)needed >= newsize - b->pos) { + newsize *= 2; + } + if (b->maxsize && newsize > b->maxsize) { + newsize = b->maxsize; + } + void *p = malloc(newsize); + if (p != NULL) { + memcpy(p, b->buf, b->pos); + old_ptr = b->buf; + b->buf = p; + b->size = newsize; + } else { + return 2; // malloc failed + } + } else { + b->pos += needed; + break; + } + } while (1); + + free(old_ptr); + return 0; +} + + +int lsb_outputs(lsb_output_buffer *b, const char *str, size_t len) +{ + int result = lsb_expand_output_buffer(b, len + 1); + if (result) return result; + + memcpy(b->buf + b->pos, str, len); + b->pos += len; + b->buf[b->pos] = 0; + return 0; +} + + +int lsb_outputd(lsb_output_buffer *b, double d) +{ + if (isnan(d)) { + return lsb_outputs(b, "nan", 3); + } + if (d == INFINITY) { + return lsb_outputs(b, "inf", 3); + } + if (d == -INFINITY) { + return lsb_outputs(b, "-inf", 4); + } + return lsb_outputfd(b, d); +} + + +int lsb_outputfd(lsb_output_buffer *b, double d) +{ + if (d < INT_MIN || d > INT_MAX) { + return lsb_outputf(b, "%0.17g", d); + } + + const int precision = 8; + const unsigned magnitude = 100000000; + char buffer[20]; + char *p = buffer; + int negative = 0; + + if (d < 0) { + negative = 1; + d = -d; + } + + int number = (int)d; + double tmp = (d - number) * magnitude; + unsigned fraction = (unsigned)tmp; + double diff = tmp - fraction; + + if (diff > 0.5) { + ++fraction; + if (fraction >= magnitude) { + fraction = 0; + ++number; + } + } else if (diff == 0.5 && ((fraction == 0) || (fraction & 1))) { + // bankers rounding + ++fraction; + } + + // decimal fraction + if (fraction != 0) { + int nodigits = 1; + char c = 0; + for (int x = 0; x < precision; ++x) { + c = fraction % 10; + if (!(c == 0 && nodigits)) { + *p++ = c + '0'; + nodigits = 0; + } + fraction /= 10; + } + *p++ = '.'; + } + + // number + do { + *p++ = (number % 10) + '0'; + number /= 10; + } while (number > 0); + + int result = lsb_expand_output_buffer(b, (p - buffer) + negative); + if (result) return result; + + if (negative) { + b->buf[b->pos++] = '-'; + } + + do { + --p; + b->buf[b->pos++] = *p; + } while (p != buffer); + + b->buf[b->pos] = 0; + return 0; +} diff --git a/src/util/protobuf.c b/src/util/protobuf.c new file mode 100644 index 0000000..cbad299 --- /dev/null +++ b/src/util/protobuf.c @@ -0,0 +1,130 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Generic protobuf utility functions @file */ + +#include "util/protobuf.h" + +#include +#include +#include + +#define check_rv(fn) { int rv = fn; if (rv) return rv; }; + +const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype) +{ + *wiretype = 7 & (unsigned char)*p; + *tag = (unsigned char)*p >> 3; + return ++p; +} + + +int lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, + unsigned char wiretype) +{ + check_rv(lsb_expand_output_buffer(ob, 1)); + ob->buf[ob->pos++] = wiretype | (tag << 3); + return 0; +} + + +const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi) +{ + *vi = 0; + int i, shift = 0; + for (i = 0; p != e && i < LSB_MAX_VARINT_BYTES; ++i, ++p) { + *vi |= ((unsigned long long)*p & 0x7f) << shift; + shift += 7; + if ((*p & 0x80) == 0) break; + } + if (i == LSB_MAX_VARINT_BYTES || p == e) { + return NULL; + } + return ++p; +} + + +int lsb_pb_output_varint(char *buf, unsigned long long i) +{ + int pos = 0; + if (i == 0) { + buf[pos++] = 0; + return pos; + } + + while (i) { + buf[pos++] = (i & 0x7F) | 0x80; + i >>= 7; + } + buf[pos - 1] &= 0x7F; // end the varint + return pos; +} + + +int lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i) +{ + check_rv(lsb_expand_output_buffer(ob, LSB_MAX_VARINT_BYTES)); + ob->pos += lsb_pb_output_varint(ob->buf + ob->pos, i); + return 0; +} + + +int lsb_pb_write_bool(lsb_output_buffer *ob, int i) +{ + check_rv(lsb_expand_output_buffer(ob, 1)); + if (i) { + ob->buf[ob->pos++] = 1; + } else { + ob->buf[ob->pos++] = 0; + } + return 0; +} + + +int lsb_pb_write_double(lsb_output_buffer *ob, double i) +{ + static const size_t needed = sizeof(double); + check_rv(lsb_expand_output_buffer(ob, needed)); + // todo add big endian support if necessary + memcpy(&ob->buf[ob->pos], &i, needed); + ob->pos += needed; + return 0; +} + + +int +lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len) +{ + check_rv(lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH)); + check_rv(lsb_pb_write_varint(ob, len)); + check_rv(lsb_expand_output_buffer(ob, len)); + memcpy(&ob->buf[ob->pos], s, len); + ob->pos += len; + return 0; +} + + +int lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos) +{ + if (len_pos >= ob->pos) return 1; + + size_t len = ob->pos - len_pos - 1; + if (len < 128) { + ob->buf[len_pos] = (char)len; + return 0; + } + size_t l = len, cnt = 0; + while (l) { + l >>= 7; + ++cnt; // compute the number of bytes needed for the varint length + } + size_t needed = cnt - 1; + check_rv(lsb_expand_output_buffer(ob, needed)); + ob->pos += needed; + memmove(&ob->buf[len_pos + cnt], &ob->buf[len_pos + 1], len); + lsb_pb_output_varint(ob->buf + len_pos, len); + return 0; +} diff --git a/src/util/running_stats.c b/src/util/running_stats.c new file mode 100644 index 0000000..7482f21 --- /dev/null +++ b/src/util/running_stats.c @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Running stats implementation @file */ + +#include "util/running_stats.h" + +#include + +void lsb_init_running_stats(lsb_running_stats *s) +{ + s->count = 0.0; + s->mean = 0.0; + s->sum = 0.0; +} + + +void lsb_update_running_stats(lsb_running_stats *s, double d) +{ + if (!isfinite(d)) return; + + double old_mean = s->mean; + double old_sum = s->sum; + + if (++s->count == 1) { + s->mean = d; + } else { + s->mean = old_mean + (d - old_mean) / s->count; + s->sum = old_sum + (d - old_mean) * (d - s->mean); + } +} + + +double lsb_sd_running_stats(lsb_running_stats *s) +{ + if (s->count < 2) return 0.0; + return sqrt(s->sum / (s->count - 1)); +} + diff --git a/src/util/string.c b/src/util/string.c new file mode 100644 index 0000000..415b876 --- /dev/null +++ b/src/util/string.c @@ -0,0 +1,15 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** String functions @file */ + +#include "util/string.h" + +void lsb_init_const_string(lsb_const_string *s) +{ + s->s = NULL; + s->len = 0; +} diff --git a/src/util/string_matcher.c b/src/util/string_matcher.c new file mode 100644 index 0000000..eeac454 --- /dev/null +++ b/src/util/string_matcher.c @@ -0,0 +1,275 @@ +/* +** Modified Lua lstrlib.c for the Lua sandbox message matcher pattern-matching +* +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ + +#include "util/string_matcher.h" + +#include + +/* macro to `unsign' a character */ +#define uchar(c) ((unsigned char)(c)) + +typedef struct MatchState { + const char *src_init; /* init of source string */ + const char *src_end; /* end (`\0') of source string */ +} MatchState; + + +#define L_ESC '%' +#define SPECIALS "^$*+?.([%-" + + +static const char* classend(const char *p) +{ + switch (*p++) { + case L_ESC: + { + if (*p == '\0') return NULL; // error pattern ends with a % + return p + 1; + } + case '[': + { + if (*p == '^') p++; + do { /* look for a `]' */ + if (*p == '\0') return NULL; // error missing closing ] + if (*(p++) == L_ESC && *p != '\0') p++; /* skip escapes (e.g. `%]') */ + }while (*p != ']'); + return p + 1; + } + default: + { + return p; + } + } +} + + +static int match_class(int c, int cl) +{ + int res; + switch (tolower(cl)) { + case 'a' : + res = isalpha(c); break; + case 'c' : + res = iscntrl(c); break; + case 'd' : + res = isdigit(c); break; + case 'l' : + res = islower(c); break; + case 'p' : + res = ispunct(c); break; + case 's' : + res = isspace(c); break; + case 'u' : + res = isupper(c); break; + case 'w' : + res = isalnum(c); break; + case 'x' : + res = isxdigit(c); break; + case 'z' : + res = (c == 0); break; + default: + return (cl == c); + } + return (islower(cl) ? res : !res); +} + + +static int matchbracketclass(int c, const char *p, const char *ec) +{ + int sig = 1; + if (*(p + 1) == '^') { + sig = 0; + p++; /* skip the `^' */ + } + while (++p < ec) { + if (*p == L_ESC) { + p++; + if (match_class(c, uchar(*p))) return sig; + } else if ((*(p + 1) == '-') && (p + 2 < ec)) { + p += 2; + if (uchar(*(p - 2)) <= c && c <= uchar(*p)) return sig; + } else if (uchar(*p) == c) return sig; + } + return !sig; +} + + +static int singlematch(int c, const char *p, const char *ep) +{ + switch (*p) { + case '.': + return 1; /* matches any char */ + case L_ESC: + return match_class(c, uchar(*(p + 1))); + case '[': + return matchbracketclass(c, p, ep - 1); + default: + return (uchar(*p) == c); + } +} + + +static const char* match(MatchState *ms, const char *s, const char *p); + + +static const char* matchbalance(MatchState *ms, const char *s, + const char *p) +{ + if (*p == 0 || *(p + 1) == 0) return NULL; // ubalanced pattern; + if (*s != *p) return NULL; + else { + int b = *p; + int e = *(p + 1); + int cont = 1; + while (++s < ms->src_end) { + if (*s == e) { + if (--cont == 0) return s + 1; + } else if (*s == b) cont++; + } + } + return NULL; /* string ends out of balance */ +} + + +static const char* max_expand(MatchState *ms, const char *s, + const char *p, const char *ep) +{ + ptrdiff_t i = 0; /* counts maximum expand for item */ + while ((s + i) < ms->src_end && singlematch(uchar(*(s + i)), p, ep)) i++; + /* keeps trying to match with the maximum repetitions */ + while (i >= 0) { + const char *res = match(ms, (s + i), ep + 1); + if (res) return res; + i--; /* else didn't match; reduce 1 repetition to try again */ + } + return NULL; +} + + +static const char* min_expand(MatchState *ms, const char *s, + const char *p, const char *ep) +{ + for (;;) { + const char *res = match(ms, s, ep + 1); + if (res != NULL) return res; + else if (s < ms->src_end && singlematch(uchar(*s), p, ep)) s++; /* try with one more repetition */ + else return NULL; + } +} + + +static const char* match(MatchState *ms, const char *s, const char *p) +{ +init: /* using goto's to optimize tail recursion */ + switch (*p) { + case L_ESC: + { + switch (*(p + 1)) { + case 'b': + { /* balanced string? */ + s = matchbalance(ms, s, p + 2); + if (s == NULL) return NULL; + p += 4; goto init; /* else return match(ms, s, p+4); */ + } + case 'f': + { /* frontier? */ + const char *ep; char previous; + p += 2; + if (*p != '[') return NULL; // missing [ after %f + ep = classend(p); /* points to what is next */ + if (ep == NULL) return NULL; + previous = (s == ms->src_init) ? '\0' : *(s - 1); + if (matchbracketclass(uchar(previous), p, ep - 1) || + !matchbracketclass(uchar(*s), p, ep - 1)) return NULL; + p = ep; goto init; /* else return match(ms, s, ep); */ + } + default: + { + goto dflt; /* case default */ + } + } + } + case '\0': + { /* end of pattern */ + return s; /* match succeeded */ + } + case '$': + { + if (*(p + 1) == '\0') /* is the `$' the last char in pattern? */ + return (s == ms->src_end) ? s : NULL; /* check end of string */ + else goto dflt; + } + default: + dflt: + { /* it is a pattern item */ + const char *ep = classend(p); /* points to what is next */ + if (ep == NULL) return NULL; + + int m = s < ms->src_end && singlematch(uchar(*s), p, ep); + switch (*ep) { + case '?': + { /* optional */ + const char *res; + if (m && ((res = match(ms, s + 1, ep + 1)) != NULL)) return res; + p = ep + 1; goto init; /* else return match(ms, s, ep+1); */ + } + case '*': + { /* 0 or more repetitions */ + return max_expand(ms, s, p, ep); + } + case '+': + { /* 1 or more repetitions */ + return (m ? max_expand(ms, s + 1, p, ep) : NULL); + } + case '-': + { /* 0 or more repetitions (minimum) */ + return min_expand(ms, s, p, ep); + } + default: + { + if (!m) return NULL; + s++; p = ep; goto init; /* else return match(ms, s+1, ep); */ + } + } + } + } +} + + +bool lsb_string_match(const char *s, size_t len, const char *p) +{ + MatchState ms; + int anchor = (*p == '^') ? (p++, 1) : 0; + const char *s1 = s; + ms.src_init = s; + ms.src_end = s + len; + do { + const char *res; + if ((res = match(&ms, s1, p)) != NULL) { + return true; + } + } while (s1++ < ms.src_end && !anchor); + return false; +} diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt new file mode 100644 index 0000000..f41496d --- /dev/null +++ b/src/util/test/CMakeLists.txt @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +add_executable(test_input_buffer test_input_buffer.c) +target_link_libraries(test_input_buffer luasandboxutil) +add_test(NAME test_input_buffer COMMAND test_input_buffer) + +add_executable(test_output_buffer test_output_buffer.c) +target_link_libraries(test_output_buffer luasandboxutil) +add_test(NAME test_output_buffer COMMAND test_output_buffer) + +add_executable(test_protobuf test_protobuf.c) +target_link_libraries(test_protobuf luasandboxutil) +add_test(NAME test_protobuf COMMAND test_protobuf) + +add_executable(test_string_matcher test_string_matcher.c) +target_link_libraries(test_string_matcher luasandboxutil) +add_test(NAME test_string_matcher COMMAND test_string_matcher) + +add_executable(test_running_stats test_running_stats.c) +target_link_libraries(test_running_stats luasandboxutil) +add_test(NAME test_running_stats COMMAND test_running_stats) + +add_executable(test_util test_util.c) +target_link_libraries(test_util luasandboxutil) +add_test(NAME test_util COMMAND test_util) + +add_executable(test_heka_message test_heka_message.c) +target_link_libraries(test_heka_message luasandboxutil) +add_test(NAME test_heka_message COMMAND test_heka_message) diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c new file mode 100644 index 0000000..158828b --- /dev/null +++ b/src/util/test/test_heka_message.c @@ -0,0 +1,334 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief heka_message unit tests @file */ + +#include +#include + +#include + +#include "../../test/mu_test.h" +#include "util/heka_message.h" + +#define TEST_UUID "\x0a\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +#define TEST_NS "\x10\x01" + +// {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} +static char pb[] = "\x0a\x10\x73\x1e\x36\x84\xec\x25\x42\x76\xa4\x01\x79\x6f\x17\xdd\x20\x63\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; + + +struct log_message { + const char *component; + int severity; + char msg[1024]; +}; + +static struct log_message log = { 0 }; + +void logger(const char *component, int severity, const char *fmt, ...) +{ + log.component = component; + log.severity = severity; + va_list args; + va_start(args, fmt); + vsnprintf(log.msg, sizeof log.msg, fmt, args); + va_end(args); +} + +static char* test_stub() +{ + return NULL; +} + + +static char* test_init() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 10), "failed"); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_init_failure() +{ + lsb_heka_message m; + mu_assert(lsb_init_heka_message(&m, 0), "suceeded"); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_decode() +{ + +#define add_string(str) {.s = str, .len = sizeof str - 1}, + lsb_const_string tests[] = { + add_string(TEST_UUID TEST_NS) // required fields + add_string(TEST_UUID TEST_NS "\x1a\x04" "Type") + add_string(TEST_UUID TEST_NS "\x22\x06" "Logger") + add_string(TEST_UUID TEST_NS "\x28\x07") // Severity + add_string(TEST_UUID TEST_NS "\x32\x07" "Payload") + add_string(TEST_UUID TEST_NS "\x3a\x0a" "EnvVersion") + add_string(TEST_UUID TEST_NS "\x40\x11") // Pid + add_string(TEST_UUID TEST_NS "\x4a\x08" "Hostname") + add_string(TEST_UUID TEST_NS "\x52\x11\x0a\x03" "foo\x10\x00\x1a\x03" "rep\x22\x03" "bar") // string + add_string(TEST_UUID TEST_NS "\x52\x11\x0a\x03" "foo\x10\x01\x1a\x03" "rep\x2a\x03" "bar") // bytes + add_string(TEST_UUID TEST_NS "\x52\x0e\x0a\x03" "foo\x10\x02\x1a\x03" "rep\x30\x11") // integer + add_string(TEST_UUID TEST_NS "\x52\x15\x0a\x03" "foo\x10\x03\x1a\x03" "rep\x39\x00\x00\x00\x00\x00\x00\x00\x00") // double + add_string(TEST_UUID TEST_NS "\x52\x0e\x0a\x03" "foo\x10\x04\x1a\x03" "rep\x40\x01") // bool + }; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed"); + for (unsigned i = 0; i < sizeof tests / sizeof(lsb_const_string); ++i){ + bool ok = lsb_decode_heka_message(&m, tests[i].s, tests[i].len, logger); + mu_assert(ok, "test: %d failed err: %s", i, log.msg); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_decode_failure() +{ + struct decode_failure { + const char *s; + const char *e; + }; + + struct decode_failure tests[] = { + { TEST_NS, "missing Uuid" } // required test + , { TEST_UUID, "missing Timestamp" } // require test + , { "\x0a\x01\xff", "tag:1 wiretype:2 position:0" } // invalid UUID length + , { "\xf0", "tag:30 wiretype:0 position:0" } // unknown message tag + , { "\x0b", "tag:1 wiretype:3 position:0" } // uuid invalid wiretype + , { "\x11", "tag:2 wiretype:1 position:0" } // timestamp invalid wiretype + , { "\x1b", "tag:3 wiretype:3 position:0" } // type invalid wiretype + , { "\x23", "tag:4 wiretype:3 position:0" } // logger invalid wiretype + , { "\x2b", "tag:5 wiretype:3 position:0" } // severity invalid wiretype + , { "\x33", "tag:6 wiretype:3 position:0" } // payload invalid wiretype + , { "\x3b", "tag:7 wiretype:3 position:0" } // env_version invalid wiretype + , { "\x43", "tag:8 wiretype:3 position:0" } // pid invalid wiretype + , { "\x4b", "tag:9 wiretype:3 position:0" } // hostname invalid wiretype + , { "\x53", "tag:10 wiretype:3 position:0" } // fields invalid wiretype + , { "\x52\x10", "tag:10 wiretype:2 position:0" } // invalid field length + , { "\x52\x01\x0b", "tag:10 wiretype:2 position:0" } // invalid name wiretype + , { "\x52\x01\x13", "tag:10 wiretype:2 position:0" } // invalid value_type wiretype + , { "\x52\x01\x1b", "tag:10 wiretype:2 position:0" } // invalid representation wiretype + , { "\x52\x01\x23", "tag:10 wiretype:2 position:0" } // invalid string wiretype + , { "\x52\x01\x2b", "tag:10 wiretype:2 position:0" } // invalid bytes wiretype + , { "\x52\x01\x33", "tag:10 wiretype:2 position:0" } // invalid integer wiretype + , { "\x52\x01\x3b", "tag:10 wiretype:2 position:0" } // invalid bool wiretype + , { "\x52\x01\x43", "tag:10 wiretype:2 position:0" } // invalid double wiretype + , { "\x52\x01\x4b", "tag:10 wiretype:2 position:0" } // unknown field tag + , { "\x52\xc\x10\x00\x1a\x03" "rep\x22\x03" "bar", "tag:10 wiretype:2 position:0" } // no name + , { "\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "tag:2 wiretype:0 position:0" } // invalid varint + , { "\x0a\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "tag:1 wiretype:2 position:0" } // invalid varint length + }; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 10), "failed"); + for (unsigned i = 0; i < sizeof(tests) / sizeof(struct decode_failure); ++i) { + bool ok = lsb_decode_heka_message(&m, tests[i].s, strlen(tests[i].s), logger); + mu_assert(!ok, "test: %u no error generated", i); + mu_assert(!strcmp(log.msg, tests[i].e), "test: %u expected: %s received: %s", + i, tests[i].e, log.msg); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_find_message() +{ + struct find_message { + lsb_const_string s; + bool b; + size_t d; + }; + +#define add_input(str, result, discard) {.s = (lsb_const_string){.s = str, .len = sizeof str - 1}, .b = result, .d = discard}, + struct find_message tests[] = { + add_input("\x1e\x02\x08\x14\x1f" TEST_UUID TEST_NS, true, 0) // full message + add_input("\x1e", false, 0) + add_input("\x02", false, 0) + add_input("\x08", false, 0) + add_input("\x14", false, 0) + add_input("\x1f", false, 0) + add_input(TEST_UUID, false, 0) + add_input(TEST_NS "\x1e\x02\x08\x14\x1f", true, 0) // completion of an incremental message and the start of another + add_input(TEST_UUID TEST_NS, true, 0) + add_input(TEST_UUID TEST_NS, false, 20) // no framing + add_input("\x1e\x02\x08\x15\x1f" TEST_UUID TEST_NS "\x00", false, 26) // message decode failure + add_input("\x1e\x02\x08\x00\x1f", false, 5) // header decoder failure + add_input("\x1e\x02\x08\x14\x01\x1f", false, 6) // invalid header length + add_input("\x1e\x02\x09\x14\x1f", false, 5) // invalid protobuf header incorrect tag + add_input("\x1e\x02\x08\x65\x1f", false, 5) // invalid header message too long + }; + + lsb_heka_message m; + lsb_input_buffer ib; + size_t db; + mu_assert(!lsb_init_heka_message(&m, 1), "failed"); + mu_assert(!lsb_init_input_buffer(&ib, 100), "failed"); + for (unsigned i = 0; i < sizeof(tests) / sizeof(struct find_message); ++i) { + mu_assert(!lsb_expand_input_buffer(&ib, tests[i].s.len), "buffer exhausted"); + memcpy(ib.buf + ib.readpos, tests[i].s.s, tests[i].s.len); + ib.readpos += tests[i].s.len; + bool b = lsb_find_heka_message(&m, &ib, true, &db, NULL); + mu_assert(tests[i].b == b, "test: %u failed", i); + mu_assert(tests[i].d == db, "test: %u failed expected: %" PRIuSIZE " received: %" PRIuSIZE, i, tests[i].d, db); + } + lsb_free_input_buffer(&ib); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_read_heka_field() +{ + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, NULL), "decode failed"); + + lsb_read_value v; + lsb_const_string cs; + cs.s = "string"; + cs.len = 6; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "standalone"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "string", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + cs.s = "strings"; + cs.len = 7; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "item 0"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "s1", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 1, &v), "item 1"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "s2", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 2, &v), "item 2"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "s3", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 3, &v) == false, + "no item 3"); + mu_assert(v.type == LSB_READ_NIL, "%d", v.type); + + cs.s = "number"; + cs.len = 6; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "standalone"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + + cs.s = "numbers"; + cs.len = 7; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "item 0"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 1, &v), "item 1"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 2, "invalid value: %g", v.u.d); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 2, &v), "item 2"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 3, "invalid value: %g", v.u.d); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 3, &v) == false, + "no item 3"); + mu_assert(v.type == LSB_READ_NIL, "%d", v.type); + + cs.s = "bool"; + cs.len = 4; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "standalone"); + mu_assert(v.type == LSB_READ_BOOL, "%d", v.type); + mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_write_heka_uuid() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_UUID_SIZE + 2); + + const char header[2] = "\x0a\x10"; + const char bin_uuid[LSB_UUID_SIZE] = { 0 }; + mu_assert_rv(0, lsb_write_heka_uuid(&ob, bin_uuid, LSB_UUID_SIZE)); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); + mu_assert(memcmp(ob.buf + 2, bin_uuid, LSB_UUID_SIZE) == 0, "invalid"); + + const char str_uuid[LSB_UUID_STR_SIZE] = "00000000-0000-0000-0000-" + "000000000000"; + mu_assert_rv(0, lsb_write_heka_uuid(&ob, str_uuid, LSB_UUID_STR_SIZE)); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); + mu_assert(memcmp(ob.buf + 2, bin_uuid, LSB_UUID_SIZE) == 0, "invalid"); + + const char err_uuid[LSB_UUID_STR_SIZE] = "00000000+0000-0000-0000-" + "000000000000"; + mu_assert_rv(0, lsb_write_heka_uuid(&ob, err_uuid, LSB_UUID_STR_SIZE)); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); + mu_assert(ob.buf[8] & 0x40, "invalid format should create a type 4 uuid"); + + mu_assert_rv(0, lsb_write_heka_uuid(&ob, NULL, 0)); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(ob.buf[8] & 0x40, "null string should create a type 4 uuid"); + lsb_free_output_buffer(&ob); + + mu_assert_rv(0, lsb_write_heka_uuid(&ob, bin_uuid, 10)); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(ob.buf[8] & 0x40, "unexpected length should create a type 4 uuid"); + lsb_free_output_buffer(&ob); + + lsb_output_buffer sob; + lsb_init_output_buffer(&sob, LSB_UUID_SIZE); + mu_assert_rv(1, lsb_write_heka_uuid(&sob, NULL, 0)); + mu_assert(sob.pos == 0, "received: %" PRIuSIZE, sob.pos); + lsb_free_output_buffer(&sob); + + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init); + mu_run_test(test_init_failure); + mu_run_test(test_decode); + mu_run_test(test_decode_failure); + mu_run_test(test_find_message); + mu_run_test(test_read_heka_field); + mu_run_test(test_write_heka_uuid); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c new file mode 100644 index 0000000..0b4e4d7 --- /dev/null +++ b/src/util/test/test_input_buffer.c @@ -0,0 +1,148 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include +#include + +#include "../../test/mu_test.h" +#include "util/input_buffer.h" +#include "util/heka_message.h" + +static char* test_stub() +{ + return NULL; +} + +static char* test_init_small_buf() +{ + size_t size = 1024; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert(b.size == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_init_large_buf() +{ + size_t size = 1024 * 1024; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert(b.size == BUFSIZ, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_init_zero_buf() +{ + lsb_input_buffer b; + mu_assert(lsb_init_input_buffer(&b, 0), "init succeeded"); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_expand_buf() +{ + size_t size = 1024 * 1024; + size_t rsize = 1024 * 16; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert(b.size == BUFSIZ, "received: %" PRIuSIZE, b.size); + + mu_assert(!lsb_expand_input_buffer(&b, 1024 * 9), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + + mu_assert(!lsb_expand_input_buffer(&b, 1024), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_expand_failure() +{ + size_t size = 1024; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert_rv(1, lsb_expand_input_buffer(&b, size + LSB_MAX_HDR_SIZE + 1)); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_shift_empty() +{ + size_t size = 16; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + b.readpos = 10; + b.scanpos = 10; + mu_assert(!lsb_expand_input_buffer(&b, 1), "expand failed"); + mu_assert(b.scanpos == 0, "received: %" PRIuSIZE, b.scanpos); + mu_assert(b.readpos == 0, "received: %" PRIuSIZE, b.readpos); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_shift_partial() +{ + size_t size = 16; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + memset(b.buf, '#', size); + b.readpos = 10; + b.scanpos = 9; + b.buf[b.scanpos] = 'A'; + mu_assert(!lsb_expand_input_buffer(&b, 1), "expand failed"); + mu_assert(b.scanpos == 0, "received: %" PRIuSIZE, b.scanpos); + mu_assert(b.readpos == 1, "received: %" PRIuSIZE, b.readpos); + mu_assert(b.buf[0] == 'A', "received: %c", b.buf[0]); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init_small_buf); + mu_run_test(test_init_large_buf); + mu_run_test(test_init_zero_buf); + mu_run_test(test_expand_buf); + mu_run_test(test_expand_failure); + mu_run_test(test_shift_empty); + mu_run_test(test_shift_partial); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c new file mode 100644 index 0000000..243823d --- /dev/null +++ b/src/util/test/test_output_buffer.c @@ -0,0 +1,180 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_output_buffer unit tests @file */ + +#include +#include + +#include "../../test/mu_test.h" +#include "util/output_buffer.h" +#include "util/heka_message.h" + +static char* test_stub() +{ + return NULL; +} + +static char* test_init_small_buf() +{ + size_t size = 512; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(b.size == size, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_init_large_buf() +{ + size_t size = 1024 * 1024; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size , "received: %" PRIuSIZE, b.size); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_init_zero_buf() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == 0 , "received: %" PRIuSIZE, b.size); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_expand_buf() +{ + size_t size = 1024 * 1024; + size_t rsize = 1024 * 16; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + + mu_assert(!lsb_expand_output_buffer(&b, 1024 * 9), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); + + mu_assert(!lsb_expand_output_buffer(&b, 1024), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); + + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_expand_failure() +{ + size_t size = 1024; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(lsb_expand_output_buffer(&b, size + 1), "expand succeeded"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputc() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputc(&b, 'a'); + mu_assert(strcmp("a", b.buf) == 0, "received: %s", b.buf); + lsb_outputc(&b, 'b'); + mu_assert(strcmp("ab", b.buf) == 0, "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputf() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputf(&b, "%s", "foo"); + mu_assert(strcmp("foo", b.buf) == 0, "received: %s", b.buf); + lsb_outputf(&b, " %s", "bar"); + mu_assert(strcmp("foo bar", b.buf) == 0, "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputs() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputs(&b, "foo", 3); + mu_assert(strcmp("foo", b.buf) == 0, "received: %s", b.buf); + lsb_outputf(&b, " bar", 4); + mu_assert(strcmp("foo bar", b.buf) == 0, "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputd() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputd(&b, 10.1); + mu_assert(strcmp("10.1", b.buf) == 0, "received: %s", b.buf); + double d = INT_MAX; + lsb_outputd(&b, d + 1); + mu_assert(strcmp("10.12147483648", b.buf) == 0, "received: %s", b.buf); + d = INT_MIN; + lsb_outputd(&b, d - 1); + mu_assert(strcmp("10.12147483648-2147483649", b.buf) == 0, + "received: %s", b.buf); + lsb_outputd(&b, 0.0/0.0); + mu_assert(strcmp("10.12147483648-2147483649nan", b.buf) == 0, + "received: %s", b.buf); + lsb_outputd(&b, 1.0/0.0); + mu_assert(strcmp("10.12147483648-2147483649naninf", b.buf) == 0, + "received: %s", b.buf); + lsb_outputd(&b, -1.0/0.0); + mu_assert(strcmp("10.12147483648-2147483649naninf-inf", b.buf) == 0, + "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init_small_buf); + mu_run_test(test_init_large_buf); + mu_run_test(test_init_zero_buf); + mu_run_test(test_expand_buf); + mu_run_test(test_expand_failure); + mu_run_test(test_outputc); + mu_run_test(test_outputf); + mu_run_test(test_outputs); + mu_run_test(test_outputd); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c new file mode 100644 index 0000000..aefc837 --- /dev/null +++ b/src/util/test/test_protobuf.c @@ -0,0 +1,230 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief util protobuf unit tests @file */ + +#include +#include + +#include "util/heka_message.h" +#include "util/protobuf.h" + +#include "../../test/mu_test.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_lsb_pb_read_key() +{ + int tag, wt; + const char input[] = "\x0b"; + const char *p = lsb_pb_read_key(input, &tag, &wt); + mu_assert(p = input + 1, "received: %p", p); + mu_assert(tag == 1, "received: %d", tag); + mu_assert(wt == 3, "received: %d", wt); + return NULL; +} + + +static char* test_lsb_pb_write_key() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); + + mu_assert_rv(0, lsb_pb_write_key(&ob, 1, 3)); + mu_assert(memcmp(ob.buf, "\x0b", 1) == 0, "received: %02hhx", ob.buf[0]); + + ob.pos = ob.maxsize; + mu_assert_rv(1, lsb_pb_write_key(&ob, 1, 3)); + + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_varint() +{ + struct test_data { + long long v; + const char *s; + }; + + struct test_data tests[] = { + { 0, "\x00" }, + { 16, "\x10" }, + { 300, "\xac\x02" }, + { -1, "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" }, + { 5000000000LL, "\x80\xe4\x97\xd0\x12" } + }; + + long long vi; + const char *p; + for (unsigned i = 0; i < sizeof tests / sizeof tests[0]; ++i){ + long long v = tests[i].v; + const char *s = tests[i].s; + size_t len = strlen(s); + len = len ? len : 1; + p = lsb_pb_read_varint(s, s + len, &vi); + mu_assert(p = s + len, "received: %p", p); + mu_assert(v == vi, "expected: %lld received: %lld", v, vi); + + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); + mu_assert_rv(0, lsb_pb_write_varint(&ob, v)); + if (strncmp(s, ob.buf, len) != 0) { + char expected[LSB_MAX_VARINT_BYTES + 1] = { 0 }; + char received[LSB_MAX_VARINT_BYTES + 1] = { 0 }; + for (unsigned i = 0; i < len; ++i) { + sprintf(expected + i * 2, "%02hhx", s[i]); + sprintf(received + i * 2, "%02hhx", ob.buf[i]); + } + mu_assert(false, "expected: %s received: %s", expected, received); + } + lsb_free_output_buffer(&ob); + } + + const char tl[] = "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + mu_assert(!lsb_pb_read_varint(tl, tl + sizeof(tl) - 1, &vi), + "parsed invalid varint (too long)"); + + const char ts[] = "\xff"; + mu_assert(!lsb_pb_read_varint(ts, ts + sizeof(ts) - 1, &vi), + "parsed invalid varint (too short)"); + + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); + ob.pos = ob.maxsize; + mu_assert_rv(1, lsb_pb_write_varint(&ob, 1)); + lsb_free_output_buffer(&ob); + + return NULL; +} + + +static char* test_lsb_pb_write_bool() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, 1); + mu_assert_rv(0, lsb_pb_write_bool(&ob, 7)); + mu_assert(ob.buf[0] == 1, "received: %02hhx", ob.buf[0]); + mu_assert(lsb_pb_write_bool(&ob, 7), "buffer should be full"); + + ob.pos = 0; + mu_assert_rv(0, lsb_pb_write_bool(&ob, 0)); + mu_assert(ob.buf[0] == 0, "received: %02hhx", ob.buf[0]); + + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_lsb_pb_write_double() +{ + double d = 7.13; + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, sizeof d); + mu_assert_rv(0, lsb_pb_write_double(&ob, d)); + mu_assert(memcmp(ob.buf, &d, sizeof d) == 0, "received: %g", + *((double *)ob.buf)); + mu_assert_rv(1, lsb_pb_write_double(&ob, d)); + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_lsb_pb_write_string() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, 64); + const char foo[] = "foo"; + const size_t len = sizeof foo - 1; + + // failure writing the key + ob.pos = ob.maxsize; + mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); + + // failure writing the len + ob.pos = ob.maxsize - 1; + mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); + + // failure writing the string + ob.pos = ob.maxsize - 3; + mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); + + ob.pos = 0; + mu_assert_rv(0, lsb_pb_write_string(&ob, 1, foo, len)); + mu_assert(ob.pos = len + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp("\x0a\x03" "foo", ob.buf, 5) == 0, "received: " + "%02hhx%02hhx%02hhx%02hhx2%hhx", ob.buf[0], ob.buf[1], ob.buf[2], + ob.buf[3], ob.buf[4]); + + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_lsb_pb_update_field_length() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, 1024); + memset(ob.buf, 0, 1024); + ob.buf[0] = 'a'; + ob.buf[2] = 'b'; + ob.pos = 3; + mu_assert_rv(0, lsb_pb_update_field_length(&ob, 1)); + mu_assert(ob.buf[1] == 1, "received: %d", ob.buf[1]); + mu_assert(ob.buf[2] == 'b', "received: %02hhx", ob.buf[2]); + mu_assert(ob.pos == 3, "received: %" PRIuSIZE, ob.pos); + + // buffer full + ob.pos = 1024; + mu_assert_rv(1, lsb_pb_update_field_length(&ob, 1)); + + // position is out of bounds + ob.pos = 512; + mu_assert_rv(1, lsb_pb_update_field_length(&ob, 512)); + + ob.buf[301] = 'x'; + ob.pos = 302; + mu_assert_rv(0, lsb_pb_update_field_length(&ob, 1)); + mu_assert((unsigned char)ob.buf[1] == 0xac, "received: %02hhx", ob.buf[1]); + mu_assert(ob.buf[2] == 0x02, "received: %02hhx", ob.buf[2]); + mu_assert(ob.buf[3] == 'b', "received: %02hhx", ob.buf[3]); + mu_assert(ob.buf[302] == 'x', "received: %02hhx", ob.buf[302]); + mu_assert(ob.pos == 303, "received: %" PRIuSIZE, ob.pos); + + lsb_free_output_buffer(&ob); + return NULL; +} + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_lsb_pb_read_key); + mu_run_test(test_lsb_pb_write_key); + mu_run_test(test_varint); + mu_run_test(test_lsb_pb_write_bool); + mu_run_test(test_lsb_pb_write_double); + mu_run_test(test_lsb_pb_write_string); + mu_run_test(test_lsb_pb_update_field_length); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_running_stats.c b/src/util/test/test_running_stats.c new file mode 100644 index 0000000..4ea5597 --- /dev/null +++ b/src/util/test/test_running_stats.c @@ -0,0 +1,84 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include + +#include "../../test/mu_test.h" +#include "util/running_stats.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_init() +{ + lsb_running_stats stats; + lsb_init_running_stats(&stats); + mu_assert(stats.count == 0, "received: %g", stats.count); + mu_assert(stats.mean == 0, "received: %g", stats.mean); + mu_assert(stats.sum == 0, "received: %g", stats.sum); + return NULL; +} + + +static char* test_calculation() +{ + lsb_running_stats stats; + lsb_init_running_stats(&stats); + double sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 0, "received: %g", sd); + lsb_update_running_stats(&stats, 1.0); + lsb_update_running_stats(&stats, 2.0); + lsb_update_running_stats(&stats, 3.0); + mu_assert(stats.count == 3, "received: %g", stats.count); + mu_assert(stats.mean == 2, "received: %g", stats.mean); + sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 1.0, "received: %g", sd); + return NULL; +} + + +static char* test_nan_inf() +{ + lsb_running_stats stats; + lsb_init_running_stats(&stats); + double sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 0, "received: %g", sd); + lsb_update_running_stats(&stats, INFINITY); + lsb_update_running_stats(&stats, NAN); + lsb_update_running_stats(&stats, -INFINITY); + mu_assert(stats.count == 0, "received: %g", stats.count); + sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 0, "received: %g", sd); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init); + mu_run_test(test_calculation); + mu_run_test(test_nan_inf); + return NULL; +} + + +int main() +{ + char* result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c new file mode 100644 index 0000000..db4440c --- /dev/null +++ b/src/util/test/test_string_matcher.c @@ -0,0 +1,142 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include + +#include "../../test/mu_test.h" +#include "util/string_matcher.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_success() +{ + char *tests[] = { + " " , "." + , "foobar", "foo" + , "foobar", "bar$" + , "a" , "%a" + , "1" , "%d" + , "a" , "%l" + , "," , "%p" + , "\t" , "%s" + , "A" , "%u" + , "a" , "%w" + , "f" , "%x" + , "%" , "%%" + , "a" , "[a-c]" + , "d" , "[^a-c]" + , "abc" , "^%a+$" + , "(" , "(" + , "|foo|" , "%b||" + , "?" , "%?" + , "+" , "%+" + , "-" , "%-" + , "*" , "%*" + , "%" , "%%" + , "." , "%." + , "^" , "%^" + , "$" , "%$" + , "[" , "%[" + , "]" , "%]" + , "]" , "]" + , "(" , "%(" + , ")" , "%)" + , ")" , ")" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(lsb_string_match(tests[i], strlen(tests[i]), tests[i + 1]), + "no match test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + mu_assert(lsb_string_match("\0", 1, "%z"), "NULL match"); + return NULL; +} + + +static char* test_failure() +{ + char *tests[] = { + "" , "." + , "_foobar", "^foo" + , "foobar_", "bar$" + , "1" , "%a" + , "a" , "%d" + , "A" , "%l" + , "a" , "%p" + , "." , "%s" + , "a" , "%u" + , "." , "%w" + , "p" , "%x" + , "a" , "%%" + , "d" , "[a-c]" + , "a" , "[^a-c]" + , "a." , "^%a+$" + , "foo()" , "bar()" + , "|foo" , "%b||" + , "aa" , "(%a)%1*" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(!lsb_string_match(tests[i], strlen(tests[i]), tests[i + 1]), + "match test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + return NULL; +} + + +static char* test_invalid() +{ + char *tests[] = { + "" , "[f" + , "" , "%b|" + , "" , "?" + , "" , "+" + , "" , "-" + , "" , "*" + , "" , "%" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(!lsb_string_match(tests[i], strlen(tests[i]), tests[i + 1]), + "invalid test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_success); + mu_run_test(test_failure); + mu_run_test(test_invalid); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c new file mode 100644 index 0000000..b46ee56 --- /dev/null +++ b/src/util/test/test_util.c @@ -0,0 +1,71 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_util unit tests @file */ + +#include + +#include "../../test/mu_test.h" +#include "util/util.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_lsb_lp2() +{ + size_t lp2; + + lp2 = lsb_lp2(0); + mu_assert(lp2 == 0, "received: %" PRIuSIZE, lp2); + + lp2 = lsb_lp2(1); + mu_assert(lp2 == 1, "received: %" PRIuSIZE, lp2); + + lp2 = lsb_lp2(1000); + mu_assert(lp2 == 1024, "received: %" PRIuSIZE, lp2); + return NULL; +} + + +static char* test_lsb_timespec_delta() +{ + double d; + d = lsb_timespec_delta( + &(struct timespec){ .tv_sec = 10, .tv_nsec = 600000000L }, + &(struct timespec){ .tv_sec = 11, .tv_nsec = 800000000L }); + mu_assert(d == 1.2, "received: %g", d); + + d = lsb_timespec_delta( + &(struct timespec){ .tv_sec = 10, .tv_nsec = 600000000L }, + &(struct timespec){ .tv_sec = 12, .tv_nsec = 300000000L }); + mu_assert(d == 1.7, "received: %g", d); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_lsb_lp2); + mu_run_test(test_lsb_timespec_delta); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/util.c b/src/util/util.c new file mode 100644 index 0000000..4f0dba5 --- /dev/null +++ b/src/util/util.c @@ -0,0 +1,34 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** General purpose utility functions @file */ + +#include "util/util.h" + +size_t lsb_lp2(unsigned long long x) +{ + if (x == 0) return 0; + x = x - 1; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >> 16); + x = x | (x >> 32); + return (size_t)(x + 1); +} + + +double lsb_timespec_delta(const struct timespec* s, const struct timespec* e) +{ + double delta; + if (e->tv_nsec - s->tv_nsec < 0) { + delta = e->tv_sec - s->tv_sec - (e->tv_nsec - s->tv_nsec) / -1e9; + } else { + delta = e->tv_sec - s->tv_sec + (e->tv_nsec - s->tv_nsec) / 1e9; + } + return delta; +} From fd650f0e782bdbcdd82cb743dd4a3f89f7e4b9f0 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 21 Jan 2016 20:03:01 -0800 Subject: [PATCH 093/235] Fix up the header installation/packaging - Add an API to access the lua file name in the Heka sandbox - Restructure the include directories to ease installation - Advance the version number and update the cmake requirement --- CMakeLists.txt | 6 +++--- .../heka}/message_matcher.h | 2 +- .../heka}/sandbox.h | 11 ++++++++++- include/{ => luasandbox}/util/heka_message.h | 8 ++++---- include/{ => luasandbox}/util/input_buffer.h | 0 include/{ => luasandbox}/util/output_buffer.h | 0 include/{ => luasandbox}/util/protobuf.h | 2 +- include/{ => luasandbox}/util/running_stats.h | 0 include/{ => luasandbox}/util/string.h | 0 .../{ => luasandbox}/util/string_matcher.h | 0 include/{ => luasandbox}/util/util.h | 0 include/luasandbox_output.h | 2 +- src/CMakeLists.txt | 5 +---- src/heka_sandbox/message.c | 6 +++--- src/heka_sandbox/message_impl.h | 2 +- src/heka_sandbox/message_matcher.c | 6 ++++-- src/heka_sandbox/sandbox.c | 19 +++++++++++++++---- src/heka_sandbox/sandbox_impl.h | 6 +++--- src/heka_sandbox/stream_reader.c | 2 +- src/heka_sandbox/stream_reader_impl.h | 4 ++-- src/heka_sandbox/test/test_message_matcher.c | 4 ++-- src/heka_sandbox/test/test_sandbox.c | 8 ++++++-- src/luasandbox.c | 2 +- src/luasandbox_impl.h | 2 +- src/util/heka_message.c | 4 ++-- src/util/input_buffer.c | 6 +++--- src/util/output_buffer.c | 4 ++-- src/util/protobuf.c | 2 +- src/util/running_stats.c | 2 +- src/util/string.c | 2 +- src/util/string_matcher.c | 2 +- src/util/test/test_heka_message.c | 2 +- src/util/test/test_input_buffer.c | 4 ++-- src/util/test/test_output_buffer.c | 4 ++-- src/util/test/test_protobuf.c | 4 ++-- src/util/test/test_running_stats.c | 2 +- src/util/test/test_string_matcher.c | 2 +- src/util/test/test_util.c | 2 +- src/util/util.c | 2 +- 39 files changed, 82 insertions(+), 59 deletions(-) rename include/{heka_sandbox => luasandbox/heka}/message_matcher.h (98%) rename include/{heka_sandbox => luasandbox/heka}/sandbox.h (97%) rename include/{ => luasandbox}/util/heka_message.h (97%) rename include/{ => luasandbox}/util/input_buffer.h (100%) rename include/{ => luasandbox}/util/output_buffer.h (100%) rename include/{ => luasandbox}/util/protobuf.h (98%) rename include/{ => luasandbox}/util/running_stats.h (100%) rename include/{ => luasandbox}/util/string.h (100%) rename include/{ => luasandbox}/util/string_matcher.h (100%) rename include/{ => luasandbox}/util/util.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 84167ee..289a27d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,12 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 10) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_MINOR 11) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/include/heka_sandbox/message_matcher.h b/include/luasandbox/heka/message_matcher.h similarity index 98% rename from include/heka_sandbox/message_matcher.h rename to include/luasandbox/heka/message_matcher.h index 942e1ee..777b7ae 100644 --- a/include/heka_sandbox/message_matcher.h +++ b/include/luasandbox/heka/message_matcher.h @@ -11,7 +11,7 @@ #include -#include "util/heka_message.h" +#include "luasandbox/util/heka_message.h" typedef struct lsb_message_matcher lsb_message_matcher; typedef struct lsb_message_match_builder lsb_message_match_builder; diff --git a/include/heka_sandbox/sandbox.h b/include/luasandbox/heka/sandbox.h similarity index 97% rename from include/heka_sandbox/sandbox.h rename to include/luasandbox/heka/sandbox.h index 2ac6e9b..2bb76e3 100644 --- a/include/heka_sandbox/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -15,7 +15,7 @@ #include "luasandbox.h" #include "luasandbox/lua.h" #include "luasandbox/lauxlib.h" -#include "util/heka_message.h" +#include "luasandbox/util/heka_message.h" #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" @@ -235,6 +235,15 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown); */ LSB_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); +/** + * Returns the filename of the Lua source. + * + * @param hsb Heka sandbox + * + * @return const char* filename. + */ +LSB_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); + // todo need access to the sandbox statistics. For simplicity it will most // likely return a Heka protobuf string with all the info diff --git a/include/util/heka_message.h b/include/luasandbox/util/heka_message.h similarity index 97% rename from include/util/heka_message.h rename to include/luasandbox/util/heka_message.h index 7fbe098..255aef3 100644 --- a/include/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -13,10 +13,10 @@ #include #include "luasandbox.h" -#include "util/input_buffer.h" -#include "util/output_buffer.h" -#include "util/string.h" -#include "util/util.h" +#include "luasandbox/util/input_buffer.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox/util/string.h" +#include "luasandbox/util/util.h" #define LSB_UUID_SIZE 16 #define LSB_UUID_STR_SIZE 36 diff --git a/include/util/input_buffer.h b/include/luasandbox/util/input_buffer.h similarity index 100% rename from include/util/input_buffer.h rename to include/luasandbox/util/input_buffer.h diff --git a/include/util/output_buffer.h b/include/luasandbox/util/output_buffer.h similarity index 100% rename from include/util/output_buffer.h rename to include/luasandbox/util/output_buffer.h diff --git a/include/util/protobuf.h b/include/luasandbox/util/protobuf.h similarity index 98% rename from include/util/protobuf.h rename to include/luasandbox/util/protobuf.h index 48f36f2..e7619ea 100644 --- a/include/util/protobuf.h +++ b/include/luasandbox/util/protobuf.h @@ -14,7 +14,7 @@ #include #include -#include "util/output_buffer.h" +#include "luasandbox/util/output_buffer.h" #define LSB_MAX_VARINT_BYTES 10 diff --git a/include/util/running_stats.h b/include/luasandbox/util/running_stats.h similarity index 100% rename from include/util/running_stats.h rename to include/luasandbox/util/running_stats.h diff --git a/include/util/string.h b/include/luasandbox/util/string.h similarity index 100% rename from include/util/string.h rename to include/luasandbox/util/string.h diff --git a/include/util/string_matcher.h b/include/luasandbox/util/string_matcher.h similarity index 100% rename from include/util/string_matcher.h rename to include/luasandbox/util/string_matcher.h diff --git a/include/util/util.h b/include/luasandbox/util/util.h similarity index 100% rename from include/util/util.h rename to include/luasandbox/util/util.h diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index 9ce473c..85a4400 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -13,7 +13,7 @@ #include "luasandbox.h" #include "luasandbox/lua.h" -#include "util/output_buffer.h" +#include "luasandbox/util/output_buffer.h" #ifdef __cplusplus extern "C" { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2b64b9e..aa6ae7c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,10 +11,7 @@ luasandbox_serialize.c set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) -add_custom_target(copy_includes -COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/luasandbox.h ${EP_BASE}/include -COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/luasandbox_output.h ${EP_BASE}/include -COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_SOURCE_DIR}/include/luasandbox_serialize.h ${EP_BASE}/include) +add_custom_target(copy_includes COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/include) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) add_dependencies(luasandbox copy_includes) diff --git a/src/heka_sandbox/message.c b/src/heka_sandbox/message.c index 04abe17..0fd36b2 100644 --- a/src/heka_sandbox/message.c +++ b/src/heka_sandbox/message.c @@ -19,9 +19,9 @@ #include "luasandbox/lauxlib.h" #include "luasandbox_output.h" #include "sandbox_impl.h" -#include "util/heka_message.h" -#include "util/output_buffer.h" -#include "util/protobuf.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox/util/protobuf.h" /** * Adds missing headers specified in the configuration to the message output. diff --git a/src/heka_sandbox/message_impl.h b/src/heka_sandbox/message_impl.h index 6998fa3..156792e 100644 --- a/src/heka_sandbox/message_impl.h +++ b/src/heka_sandbox/message_impl.h @@ -12,7 +12,7 @@ #include #include "luasandbox.h" -#include "util/heka_message.h" +#include "luasandbox/util/heka_message.h" // these functions are intentionally not exported diff --git a/src/heka_sandbox/message_matcher.c b/src/heka_sandbox/message_matcher.c index 682b423..3f4b132 100644 --- a/src/heka_sandbox/message_matcher.c +++ b/src/heka_sandbox/message_matcher.c @@ -6,7 +6,7 @@ /** @brief Hindsight/Heka message matcher implementation @file */ -#include "heka_sandbox/message_matcher.h" +#include "luasandbox/heka/message_matcher.h" #include #include @@ -15,7 +15,7 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" #include "luasandbox/lualib.h" -#include "util/string_matcher.h" +#include "luasandbox/util/string_matcher.h" static const char *grammar = "local l = require 'lpeg'\n" @@ -566,6 +566,8 @@ lsb_create_message_matcher(const lsb_message_match_builder *mmb, void lsb_destroy_message_matcher(lsb_message_matcher *mm) { + if (!mm) return; + for (int i = 0; i < mm->nodes_size; ++i) { free(mm->nodes[i].variable); switch (mm->nodes[i].value_type) { diff --git a/src/heka_sandbox/sandbox.c b/src/heka_sandbox/sandbox.c index 7db094b..2087e64 100644 --- a/src/heka_sandbox/sandbox.c +++ b/src/heka_sandbox/sandbox.c @@ -6,20 +6,20 @@ /** @brief Heka sandbox implementation @file */ -#include "heka_sandbox/sandbox.h" +#include "luasandbox/heka/sandbox.h" #include #include #include -#include "heka_sandbox/message_matcher.h" +#include "luasandbox/heka/message_matcher.h" #include "luasandbox.h" #include "luasandbox_output.h" #include "message_impl.h" #include "sandbox_impl.h" #include "stream_reader_impl.h" -#include "util/protobuf.h" -#include "util/running_stats.h" +#include "luasandbox/util/protobuf.h" +#include "luasandbox/util/running_stats.h" #ifdef _WIN32 #include @@ -119,6 +119,8 @@ static int inject_message_input(lua_State *lua) return luaL_error(lua, "%s() failed: rejected by the callback", im_func_name); } + ++hsb->stats.im_cnt; + hsb->stats.im_bytes += output.len; return 0; } @@ -147,6 +149,8 @@ static int inject_message_analysis(lua_State *lua) return luaL_error(lua, "%s() failed: rejected by the callback", im_func_name); } + ++hsb->stats.im_cnt; + hsb->stats.im_bytes += output_len; return 0; } @@ -733,3 +737,10 @@ const char* lsb_heka_get_error(lsb_heka_sandbox *hsb) if (!hsb) return ""; return lsb_get_error(hsb->lsb); } + + +const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb) +{ + if (!hsb) return NULL; + return lsb_get_lua_file(hsb->lsb); +} diff --git a/src/heka_sandbox/sandbox_impl.h b/src/heka_sandbox/sandbox_impl.h index 23abebf..22317d1 100644 --- a/src/heka_sandbox/sandbox_impl.h +++ b/src/heka_sandbox/sandbox_impl.h @@ -10,9 +10,9 @@ #define luasandbox_heka_sandbox_sandbox_impl_h_ #include "luasandbox.h" -#include "heka_sandbox/sandbox.h" -#include "util/heka_message.h" -#include "util/running_stats.h" +#include "luasandbox/heka/sandbox.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/running_stats.h" struct heka_stats { long long im_cnt; diff --git a/src/heka_sandbox/stream_reader.c b/src/heka_sandbox/stream_reader.c index 40de720..e0735d8 100644 --- a/src/heka_sandbox/stream_reader.c +++ b/src/heka_sandbox/stream_reader.c @@ -11,7 +11,7 @@ #include #include -#include "heka_sandbox/sandbox.h" +#include "luasandbox/heka/sandbox.h" #include "message_impl.h" #include "luasandbox/lauxlib.h" diff --git a/src/heka_sandbox/stream_reader_impl.h b/src/heka_sandbox/stream_reader_impl.h index 2ed37c9..10df60f 100644 --- a/src/heka_sandbox/stream_reader_impl.h +++ b/src/heka_sandbox/stream_reader_impl.h @@ -11,8 +11,8 @@ #include "luasandbox/lua.h" -#include "util/input_buffer.h" -#include "util/heka_message.h" +#include "luasandbox/util/input_buffer.h" +#include "luasandbox/util/heka_message.h" extern const char *mozsvc_heka_stream_reader; extern const char *mozsvc_heka_stream_reader_table; diff --git a/src/heka_sandbox/test/test_message_matcher.c b/src/heka_sandbox/test/test_message_matcher.c index 493ef7e..46d2daf 100644 --- a/src/heka_sandbox/test/test_message_matcher.c +++ b/src/heka_sandbox/test/test_message_matcher.c @@ -12,8 +12,8 @@ #include #include -#include "heka_sandbox/message_matcher.h" -#include "util/heka_message.h" +#include "luasandbox/heka/message_matcher.h" +#include "luasandbox/util/heka_message.h" #include "test.h" diff --git a/src/heka_sandbox/test/test_sandbox.c b/src/heka_sandbox/test/test_sandbox.c index afa81e4..3c18611 100644 --- a/src/heka_sandbox/test/test_sandbox.c +++ b/src/heka_sandbox/test/test_sandbox.c @@ -14,7 +14,7 @@ #include #include -#include "heka_sandbox/sandbox.h" +#include "luasandbox/heka/sandbox.h" // {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; @@ -193,10 +193,14 @@ static int ucp(void *parent, void *sequence_id) static char* test_create_input_sandbox() { + static const char *lua_file = "lua/input.lua"; lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); + const char* lfn = lsb_heka_get_lua_file(hsb); + mu_assert(strcmp(lua_file, lfn) == 0, "expected %s received %s", lua_file, + lfn); lsb_heka_destroy_sandbox(hsb); hsb = lsb_heka_create_input(NULL, "notfourd.lua", NULL, NULL, NULL, diff --git a/src/luasandbox.c b/src/luasandbox.c index 7ae2c03..dd8fa12 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -19,7 +19,7 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" #include "luasandbox/lualib.h" -#include "util/output_buffer.h" +#include "luasandbox/util/output_buffer.h" #include "luasandbox_impl.h" #include "luasandbox_serialize.h" diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index 084ac8f..a005529 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -11,7 +11,7 @@ #include "luasandbox.h" #include "luasandbox/lua.h" -#include "util/output_buffer.h" +#include "luasandbox/util/output_buffer.h" #ifdef _WIN32 #define snprintf _snprintf diff --git a/src/util/heka_message.c b/src/util/heka_message.c index f46cb16..9e90ae7 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -6,14 +6,14 @@ /** @brief Hindsight Heka message implementation @file */ -#include "util/heka_message.h" +#include "luasandbox/util/heka_message.h" #include #include #include "../luasandbox_impl.h" #include "luasandbox_output.h" -#include "util/protobuf.h" +#include "luasandbox/util/protobuf.h" static size_t decode_header(char *buf, size_t len, size_t max_message_size) diff --git a/src/util/input_buffer.c b/src/util/input_buffer.c index bdf8b3f..e9176fa 100644 --- a/src/util/input_buffer.c +++ b/src/util/input_buffer.c @@ -6,15 +6,15 @@ /** Data stream input buffer implementation @file */ -#include "util/input_buffer.h" +#include "luasandbox/util/input_buffer.h" #include #include #include #include -#include "util/util.h" -#include "util/heka_message.h" +#include "luasandbox/util/util.h" +#include "luasandbox/util/heka_message.h" int lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size) { diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index 5dbc217..bb28ee0 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -6,14 +6,14 @@ /** Data stream output buffer implementation @file */ -#include "util/output_buffer.h" +#include "luasandbox/util/output_buffer.h" #include #include #include #include -#include "util/util.h" +#include "luasandbox/util/util.h" int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) diff --git a/src/util/protobuf.c b/src/util/protobuf.c index cbad299..51922db 100644 --- a/src/util/protobuf.c +++ b/src/util/protobuf.c @@ -6,7 +6,7 @@ /** Generic protobuf utility functions @file */ -#include "util/protobuf.h" +#include "luasandbox/util/protobuf.h" #include #include diff --git a/src/util/running_stats.c b/src/util/running_stats.c index 7482f21..c3fe00d 100644 --- a/src/util/running_stats.c +++ b/src/util/running_stats.c @@ -6,7 +6,7 @@ /** @brief Running stats implementation @file */ -#include "util/running_stats.h" +#include "luasandbox/util/running_stats.h" #include diff --git a/src/util/string.c b/src/util/string.c index 415b876..848e74c 100644 --- a/src/util/string.c +++ b/src/util/string.c @@ -6,7 +6,7 @@ /** String functions @file */ -#include "util/string.h" +#include "luasandbox/util/string.h" void lsb_init_const_string(lsb_const_string *s) { diff --git a/src/util/string_matcher.c b/src/util/string_matcher.c index eeac454..ef44d83 100644 --- a/src/util/string_matcher.c +++ b/src/util/string_matcher.c @@ -23,7 +23,7 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ -#include "util/string_matcher.h" +#include "luasandbox/util/string_matcher.h" #include diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 158828b..5c73566 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -12,7 +12,7 @@ #include #include "../../test/mu_test.h" -#include "util/heka_message.h" +#include "luasandbox/util/heka_message.h" #define TEST_UUID "\x0a\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" #define TEST_NS "\x10\x01" diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c index 0b4e4d7..b5d9d45 100644 --- a/src/util/test/test_input_buffer.c +++ b/src/util/test/test_input_buffer.c @@ -10,8 +10,8 @@ #include #include "../../test/mu_test.h" -#include "util/input_buffer.h" -#include "util/heka_message.h" +#include "luasandbox/util/input_buffer.h" +#include "luasandbox/util/heka_message.h" static char* test_stub() { diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index 243823d..a33f0f0 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -10,8 +10,8 @@ #include #include "../../test/mu_test.h" -#include "util/output_buffer.h" -#include "util/heka_message.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox/util/heka_message.h" static char* test_stub() { diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c index aefc837..c8b8f7d 100644 --- a/src/util/test/test_protobuf.c +++ b/src/util/test/test_protobuf.c @@ -9,8 +9,8 @@ #include #include -#include "util/heka_message.h" -#include "util/protobuf.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/protobuf.h" #include "../../test/mu_test.h" diff --git a/src/util/test/test_running_stats.c b/src/util/test/test_running_stats.c index 4ea5597..22a59c7 100644 --- a/src/util/test/test_running_stats.c +++ b/src/util/test/test_running_stats.c @@ -9,7 +9,7 @@ #include #include "../../test/mu_test.h" -#include "util/running_stats.h" +#include "luasandbox/util/running_stats.h" static char* test_stub() { diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c index db4440c..6b74bf0 100644 --- a/src/util/test/test_string_matcher.c +++ b/src/util/test/test_string_matcher.c @@ -9,7 +9,7 @@ #include #include "../../test/mu_test.h" -#include "util/string_matcher.h" +#include "luasandbox/util/string_matcher.h" static char* test_stub() { diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index b46ee56..8a4a17d 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -9,7 +9,7 @@ #include #include "../../test/mu_test.h" -#include "util/util.h" +#include "luasandbox/util/util.h" static char* test_stub() { diff --git a/src/util/util.c b/src/util/util.c index 4f0dba5..6c2e9e3 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -6,7 +6,7 @@ /** General purpose utility functions @file */ -#include "util/util.h" +#include "luasandbox/util/util.h" size_t lsb_lp2(unsigned long long x) { From e8a34d80b5d20dacd9303cb2b2b49c04da728d9a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 22 Jan 2016 14:57:15 -0800 Subject: [PATCH 094/235] Expose stop and terminate APIs - Increment stat collectors and add unit tests --- CMakeLists.txt | 2 +- include/luasandbox.h | 11 +++++ include/luasandbox/heka/sandbox.h | 36 +++++++++++++- include/luasandbox/util/heka_message.h | 3 +- src/heka_sandbox/message.c | 16 +++--- src/heka_sandbox/sandbox.c | 25 ++++++++-- src/heka_sandbox/sandbox_impl.h | 8 +-- src/heka_sandbox/test/lua/output.lua | 13 +++-- src/heka_sandbox/test/test_sandbox.c | 67 ++++++++++++++++++++------ src/luasandbox.c | 15 ++++++ 10 files changed, 159 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 289a27d..fd9253a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 11) +set(CPACK_PACKAGE_VERSION_MINOR 12) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") diff --git a/include/luasandbox.h b/include/luasandbox.h index 60fff83..3c9e739 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -123,6 +123,17 @@ lsb_create(void *parent, const char *lua_file, const char *cfg, */ LSB_EXPORT int lsb_init(lsb_lua_sandbox *lsb, const char *state_file); +/** + * Aborts the running sandbox from a different thread of execution. A "shutting + * down" termination message is generated. + * + * @param lsb sandbox to abort + * + * @return + * + */ +LSB_EXPORT void lsb_stop_sandbox(lsb_lua_sandbox *lsb); + /** * Frees the memory associated with the sandbox. * diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 2bb76e3..5eb6e86 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -19,6 +19,15 @@ #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" +enum lsb_heka_pm_rv { + LSB_HEKA_PM_SENT = 0, + LSB_HEKA_PM_FAIL = -1, + LSB_HEKA_PM_SKIP = -2, + LSB_HEKA_PM_RETRY = -3, + LSB_HEKA_PM_BATCH = -4, + LSB_HEKA_PM_ASYNC = -5 +}; + typedef struct lsb_heka_sandbox lsb_heka_sandbox; #ifdef __cplusplus @@ -200,16 +209,39 @@ int lsb_heka_process_message_output(lsb_heka_sandbox *hsb, void *sequence_id, bool profile); +/** + * Aborts the running sandbox from a different thread of execution. A "shutting + * down" termination message is generated. Used to abort long runnning sandboxes + * such as an input sandbox. + * + * @param hsb Heka sandbox to abort + * + * @return + * + */ +LSB_EXPORT void +lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb); + +/** + * Terminates the sandbox as if it had a fatal error (not thread safe). + * + * @param hsb Heka sandbox to terminate + * @param err Reason for termination + */ +LSB_EXPORT void +lsb_heka_terminate_sandbox(lsb_heka_sandbox *lsb, const char *err); + /** * Frees all memory associated with the sandbox; hsb cannont be used after this * point and the host should set it to NULL. * * @param hsb Heka sandbox to free * - * @return + * @return NULL on success, pointer to an error message on failure that MUST BE + * FREED by the caller. * */ -LSB_EXPORT void +LSB_EXPORT char * lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); /** diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index 255aef3..7b8cdac 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -20,7 +20,8 @@ #define LSB_UUID_SIZE 16 #define LSB_UUID_STR_SIZE 36 -#define LSB_MAX_HDR_SIZE (255 + 3) +#define LSB_HDR_FRAME_SIZE 3 +#define LSB_MAX_HDR_SIZE (255 + LSB_HDR_FRAME_SIZE) #define LSB_UUID "Uuid" #define LSB_TIMESTAMP "Timestamp" diff --git a/src/heka_sandbox/message.c b/src/heka_sandbox/message.c index 0fd36b2..39b24f4 100644 --- a/src/heka_sandbox/message.c +++ b/src/heka_sandbox/message.c @@ -833,29 +833,31 @@ int heka_encode_message(lua_State *lua) if (strlen(err) == 0) err = "exceeded output_limit"; return luaL_error(lua, "encode_message() failed: %s", err); } - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = lsb->output.pos; - if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] - > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; - } size_t len = 0; const char *output = lsb_get_output(lsb, &len); + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len; if (framed) { char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f int hlen = lsb_pb_output_varint(header + 3, len) + 1; + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len + hlen + LSB_HDR_FRAME_SIZE; header[1] = (char)hlen; header[hlen + 2] = '\x1f'; luaL_Buffer b; luaL_buffinit(lua, &b); - luaL_addlstring(&b, (char *)header, hlen + 3); + luaL_addlstring(&b, header, hlen + LSB_HDR_FRAME_SIZE); luaL_addlstring(&b, output, len); luaL_pushresult(&b); } else { lua_pushlstring(lua, output, len); } + + if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] + > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + } return 1; } diff --git a/src/heka_sandbox/sandbox.c b/src/heka_sandbox/sandbox.c index 2087e64..3f3c7d1 100644 --- a/src/heka_sandbox/sandbox.c +++ b/src/heka_sandbox/sandbox.c @@ -475,6 +475,11 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, } lsb_terminate(hsb->lsb, err); return 1; + } else if (status == LSB_HEKA_PM_FAIL) { + ++hsb->stats.pm_cnt; + ++hsb->stats.pm_failures; + } else if (hsb->type != 'o' || status != LSB_HEKA_PM_RETRY) { + ++hsb->stats.pm_cnt; } return status; } @@ -655,14 +660,27 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, } -void lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) +void lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb) { - if (!hsb) return; + lsb_stop_sandbox(hsb->lsb); +} + + +void lsb_heka_terminate_sandbox(lsb_heka_sandbox *hsb, const char *err) +{ + lsb_terminate(hsb->lsb, err); +} + + +char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) +{ + if (!hsb) return NULL; + char *msg = lsb_destroy(hsb->lsb); - lsb_destroy(hsb->lsb); free(hsb->hostname); free(hsb->name); free(hsb); + return msg; } @@ -688,7 +706,6 @@ int lsb_heka_process_message_output(lsb_heka_sandbox *hsb, nargs = 1; lua_pushlightuserdata(lua, sequence_id); } - return process_message(hsb, msg, lua, nargs, profile); } diff --git a/src/heka_sandbox/sandbox_impl.h b/src/heka_sandbox/sandbox_impl.h index 22317d1..267e2ac 100644 --- a/src/heka_sandbox/sandbox_impl.h +++ b/src/heka_sandbox/sandbox_impl.h @@ -15,11 +15,11 @@ #include "luasandbox/util/running_stats.h" struct heka_stats { - long long im_cnt; - long long im_bytes; + unsigned long long im_cnt; + unsigned long long im_bytes; - long long pm_cnt; - long long pm_failures; + unsigned long long pm_cnt; + unsigned long long pm_failures; lsb_running_stats pm; lsb_running_stats te; diff --git a/src/heka_sandbox/test/lua/output.lua b/src/heka_sandbox/test/lua/output.lua index edb5eba..9e884f1 100644 --- a/src/heka_sandbox/test/lua/output.lua +++ b/src/heka_sandbox/test/lua/output.lua @@ -2,16 +2,23 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +local sid + function process_message(sequence_id) if not sequence_id then update_checkpoint() return 0 else - update_checkpoint(sequence_id, 7) - return -5 + if not sid then + update_checkpoint(sequence_id, 7) + sid = sequence_id + return -5 + else + return -3 -- retry + end end + return 0 end function timer_event(ns) end - diff --git a/src/heka_sandbox/test/test_sandbox.c b/src/heka_sandbox/test/test_sandbox.c index 3c18611..6f702b7 100644 --- a/src/heka_sandbox/test/test_sandbox.c +++ b/src/heka_sandbox/test/test_sandbox.c @@ -15,6 +15,7 @@ #include #include "luasandbox/heka/sandbox.h" +#include "../sandbox_impl.h" // {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; @@ -201,7 +202,7 @@ static char* test_create_input_sandbox() const char* lfn = lsb_heka_get_lua_file(hsb); mu_assert(strcmp(lua_file, lfn) == 0, "expected %s received %s", lua_file, lfn); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); hsb = lsb_heka_create_input(NULL, "notfourd.lua", NULL, NULL, NULL, iim); @@ -212,7 +213,16 @@ static char* test_create_input_sandbox() hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, NULL, NULL); mu_assert(!hsb, "lsb_heka_create_input succeeded"); - lsb_heka_destroy_sandbox(hsb); // test NULL + e = lsb_heka_destroy_sandbox(hsb); // test NULL + + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, + iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + lsb_heka_terminate_sandbox(hsb, "boom"); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp("boom", err) == 0, "received: %s", err); + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "received %s", e); return NULL; } @@ -223,7 +233,7 @@ static char* test_create_analysis_sandbox() hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, aim); mu_assert(hsb, "lsb_heka_create_analysis failed"); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); hsb = lsb_heka_create_analysis(NULL, "notfound.lua", NULL, NULL, NULL, aim); mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); @@ -269,7 +279,7 @@ static char* test_timer_event() mu_assert(1 == lsb_heka_timer_event(hsb, 2, false), "err: %s", lsb_heka_get_error(hsb)); mu_assert(hsb, "lsb_heka_create_analysis failed"); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); hsb = lsb_heka_create_analysis(NULL, "lua/input.lua", NULL, NULL, NULL, aim); mu_assert(hsb, "lsb_heka_create_analysis succeeded"); @@ -277,7 +287,7 @@ static char* test_timer_event() const char *err = lsb_heka_get_error(hsb); mu_assert(strcmp("timer_event() function was not found", err) == 0, "received: %s", err); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -314,7 +324,10 @@ static char* test_pm_input() results[i].rv, rv); } - lsb_heka_destroy_sandbox(hsb); + mu_assert(5 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); + mu_assert(1 == hsb->stats.pm_failures, "expected %llu", + hsb->stats.pm_failures); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } @@ -332,7 +345,10 @@ static char* test_pm_analysis() const char *err = lsb_heka_get_error(hsb); const char *eerr = ""; mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - lsb_heka_destroy_sandbox(hsb); + mu_assert(1 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); + mu_assert(0 == hsb->stats.pm_failures, "expected %llu", + hsb->stats.pm_failures); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } @@ -349,7 +365,7 @@ static char* test_pm_no_return() const char *err = lsb_heka_get_error(hsb); const char *eerr = "process_message() must return a numeric status code"; mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } @@ -367,11 +383,23 @@ static char* test_pm_output() const char *err = lsb_heka_get_error(hsb); const char *eerr = ""; mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + mu_assert(0 == hsb->stats.pm_failures, "expected %llu", + hsb->stats.pm_failures); mu_assert_rv(-5, lsb_heka_process_message_output(hsb, &m, (void *)99, false)); mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - lsb_heka_destroy_sandbox(hsb); + mu_assert(2 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); + mu_assert(7 == hsb->stats.pm_failures, "expected %llu", + hsb->stats.pm_failures); + + mu_assert_rv(-3, lsb_heka_process_message_output(hsb, &m, (void *)100, false)); + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + mu_assert(2 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); + mu_assert(7 == hsb->stats.pm_failures, "expected %llu", + hsb->stats.pm_failures); + + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } @@ -383,8 +411,12 @@ static char* test_im_input() hsb = lsb_heka_create_input(NULL, "lua/iim.lua", NULL, "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); + + mu_assert(6 == hsb->stats.im_cnt, "expected %llu", hsb->stats.im_cnt); + mu_assert(303 == hsb->stats.im_bytes, "expected %llu", + hsb->stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -395,8 +427,11 @@ static char* test_im_analysis() hsb = lsb_heka_create_analysis(NULL, "lua/aim.lua", NULL, "Hostname = 'foo.com';Logger = 'aim'", dlog, aim); + mu_assert(3 == hsb->stats.im_cnt, "expected %llu", hsb->stats.im_cnt); + mu_assert(232 == hsb->stats.im_bytes, "expected %llu", + hsb->stats.im_bytes); mu_assert(hsb, "lsb_heka_create_analysis failed"); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -407,7 +442,9 @@ static char* test_encode_message() hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, "Hostname = 'sh';Logger = 'sl'", dlog, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); - lsb_heka_destroy_sandbox(hsb); + size_t cur_out = lsb_usage(hsb->lsb, LSB_UT_OUTPUT, LSB_US_CURRENT); + mu_assert(33 == cur_out, "received %" PRIuSIZE, cur_out); + e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -420,7 +457,7 @@ static char* test_decode_message() hsb = lsb_heka_create_output(NULL, "lua/decode_message.lua", NULL, NULL, dlog, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } @@ -439,7 +476,7 @@ static char* test_read_message() int rv = lsb_heka_process_message_analysis(hsb, &m, false); mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, lsb_heka_get_error(hsb)); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } @@ -465,7 +502,7 @@ static char* benchmark_decode_message() / CLOCKS_PER_SEC / iter); mu_assert(hsb, "lsb_heka_create_output failed"); - lsb_heka_destroy_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; } diff --git a/src/luasandbox.c b/src/luasandbox.c index dd8fa12..33ba926 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -547,6 +547,21 @@ int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) } +static void stop_hook(lua_State* lua, lua_Debug* ar) +{ + (void)ar; /* unused arg. */ + lua_sethook(lua, NULL, 0, 0); + luaL_error(lua, "shutting down"); +} + + +void lsb_stop_sandbox(lsb_lua_sandbox* lsb) +{ + lua_State* lua = lsb_get_lua(lsb); + lua_sethook(lua, stop_hook, LUA_MASKCALL | LUA_MASKRET | LUA_MASKCOUNT, 1); +} + + char* lsb_destroy(lsb_lua_sandbox *lsb) { char *err = NULL; From 02411e231c7893a9c76ff1debd21a9da3ed3afc8 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 25 Jan 2016 16:27:01 -0800 Subject: [PATCH 095/235] New APIs and bug fixes - add lsb_read_file and lsb_clear_output_buffer - fix the remove_entries table builder - pull in Lua updates to separate the base and coroutine libs --- cmake/externals.cmake | 2 +- include/luasandbox/util/output_buffer.h | 10 +++++++- include/luasandbox/util/util.h | 11 +++++++++ src/heka_sandbox/message.c | 4 +-- src/heka_sandbox/sandbox.c | 11 ++++++--- src/luasandbox.c | 5 ++-- src/luasandbox_output.c | 6 ++--- src/luasandbox_serialize.c | 6 ++--- src/test/CMakeLists.txt | 2 +- src/test/test_luasandbox.c | 33 +++---------------------- src/util/heka_message.c | 5 ++-- src/util/output_buffer.c | 30 ++++++++++++++++------ src/util/test/test_protobuf.c | 4 ++- src/util/util.c | 30 ++++++++++++++++++++++ 14 files changed, 103 insertions(+), 56 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index e6c9fad..453a119 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -65,7 +65,7 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG cd455beaacf009838c2a493a6be98438a584b8b7 + GIT_TAG e068a11788a23ea1542ecbd54c9f806a77780db2 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/include/luasandbox/util/output_buffer.h b/include/luasandbox/util/output_buffer.h index 83cceb5..2a3b13e 100644 --- a/include/luasandbox/util/output_buffer.h +++ b/include/luasandbox/util/output_buffer.h @@ -16,10 +16,11 @@ #define LSB_OUTPUT_SIZE 1024 struct lsb_output_buffer { + char *buf; size_t maxsize; size_t size; size_t pos; - char *buf; + int err; }; #ifdef __cplusplus @@ -45,6 +46,13 @@ int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); */ LSB_EXPORT void lsb_free_output_buffer(lsb_output_buffer *b); +/** + * Resets the position and error state of the buffer + * + * @param b Output buffer + */ +LSB_EXPORT void lsb_clear_output_buffer(lsb_output_buffer *b); + /** * Resize the output buffer when more space is needed. * diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index b9da612..5e90dbe 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -40,6 +40,17 @@ LSB_EXPORT size_t lsb_lp2(unsigned long long x); LSB_EXPORT double lsb_timespec_delta(const struct timespec* s, const struct timespec* e); + +/** + * Read a file into a string + * + * @param fn Filename to read + * + * @return char* NULL on failure otherwise a pointer to the file contents (must + * be freed by the caller). + */ +LSB_EXPORT char* lsb_read_file(const char *fn); + #ifdef __cplusplus } #endif diff --git a/src/heka_sandbox/message.c b/src/heka_sandbox/message.c index 39b24f4..0e77bb6 100644 --- a/src/heka_sandbox/message.c +++ b/src/heka_sandbox/message.c @@ -827,7 +827,7 @@ int heka_encode_message(lua_State *lua) lsb_heka_sandbox *hsb = lsb_get_parent(lsb); set_missing_headers(lua, 1, hsb); - lsb->output.pos = 0; + lsb_clear_output_buffer(&lsb->output); if (heka_encode_message_table(lsb, 1)) { const char *err = lsb_get_error(lsb); if (strlen(err) == 0) err = "exceeded output_limit"; @@ -866,7 +866,7 @@ int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) { int result = 0; lsb_output_buffer *ob = &lsb->output; - ob->pos = 0; + lsb_clear_output_buffer(ob); // use existing or create a type 4 uuid lua_getfield(lsb->lua, idx, LSB_UUID); diff --git a/src/heka_sandbox/sandbox.c b/src/heka_sandbox/sandbox.c index 3f3c7d1..b1af4d9 100644 --- a/src/heka_sandbox/sandbox.c +++ b/src/heka_sandbox/sandbox.c @@ -255,8 +255,8 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) }; static const char *analysis[] = { - NULL, "", "collectgarbage", "coroutine", "dofile", "load", "loadfile", - "loadstring", "newproxy", "print", + NULL, "", "collectgarbage", "dofile", "load", "loadfile", "loadstring", + "newproxy", "print", NULL, "os", "getenv", "execute", "exit", "remove", "rename", "setlocale", "tmpname", NULL, "string", "dump" @@ -278,6 +278,8 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) lua_newtable(lua); lua_pushboolean(lua, true); lua_setfield(lua, -2, "io"); + lua_pushboolean(lua, true); + lua_setfield(lua, -2, "coroutine"); lua_setfield(lua, 1, "disable_modules"); break; } @@ -288,11 +290,14 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) lua_pushstring(lua, list[i]); lua_rawseti(lua, -2, j++); } else { + if (lua_gettop(lua) == 3) lua_pop(lua, 1); // remove the previous list lua_newtable(lua); - lua_setfield(lua, -2, list[++i]); + lua_pushvalue(lua, -1); + lua_setfield(lua, -3, list[++i]); j = 1; } } + lua_pop(lua, 1); // remove the last list lua_setfield(lua, 1, "remove_entries"); lua_getfield(lua, 1, LSB_HEKA_MAX_MESSAGE_SIZE); diff --git a/src/luasandbox.c b/src/luasandbox.c index 33ba926..6eb331b 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -27,7 +27,8 @@ static jmp_buf g_jbuf; static const luaL_Reg preload_module_list[] = { - { "", luaopen_base }, + { LUA_BASELIBNAME, luaopen_base }, + { LUA_COLIBNAME, luaopen_coroutine }, { LUA_TABLIBNAME, luaopen_table }, { LUA_IOLIBNAME, luaopen_io }, { LUA_OSLIBNAME, luaopen_os }, @@ -495,7 +496,7 @@ int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) lsb_terminate(lsb, NULL); return 1; } - lua_pushstring(lsb->lua, ""); + lua_pushstring(lsb->lua, LUA_BASELIBNAME); if (lua_pcall(lsb->lua, 1, 0, 0)) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_init %s", lua_tostring(lsb->lua, -1)); diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c index 181551c..4016704 100644 --- a/src/luasandbox_output.c +++ b/src/luasandbox_output.c @@ -42,7 +42,7 @@ lua_CFunction lsb_get_output_function(lua_State *lua, int index) void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) { if (!append) { - lsb->output.pos = 0; + lsb_clear_output_buffer(&lsb->output); } int result = 0; @@ -112,8 +112,8 @@ const char* lsb_get_output(lsb_lua_sandbox *lsb, size_t *len) if (len) { *len = lsb->output.pos; } - if (lsb->output.pos == 0) return ""; + if (lsb->output.err || lsb->output.pos == 0) return ""; - lsb->output.pos = 0; + lsb_clear_output_buffer(&lsb->output); return lsb->output.buf; } diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 5ecaa3d..480c95d 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -231,7 +231,7 @@ serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) static int serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *output) { - output->pos = 0; + lsb_clear_output_buffer(output); switch (lua_type(lsb->lua, index)) { case LUA_TNUMBER: if (lsb_serialize_double(output, lua_tonumber(lsb->lua, index))) { @@ -333,7 +333,7 @@ serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) data->keys.pos += 1; lua_pushlightuserdata(lsb->lua, data->keys.buf + pos); lua_pushlightuserdata(lsb->lua, &lsb->output); - lsb->output.pos = 0; + lsb_clear_output_buffer(&lsb->output); result = fp(lsb->lua); lua_pop(lsb->lua, 2); // remove the key and the output if (result == 0) { @@ -465,7 +465,7 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; #endif lsb->output.maxsize = max_output_size; - lsb->output.pos = 0; + lsb_clear_output_buffer(lsb->output); if (lsb->output.size > cur_output_size) { void* ptr = realloc(lsb->output.data, cur_output_size); if (!ptr) return 1; diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 3160e97..d5c174d 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. add_executable(test_luasandbox test_luasandbox.c) -target_link_libraries(test_luasandbox luasandbox) +target_link_libraries(test_luasandbox luasandbox luasandboxutil) add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) if(WIN32) diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 5edfc98..d71c060 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -16,6 +16,7 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" #include "luasandbox_output.h" +#include "luasandbox/util/util.h" #include "mu_test.h" @@ -57,32 +58,6 @@ int file_exists(const char *fn) } -char* read_file(const char *fn) -{ - char *str = NULL; - FILE *fh; - fh = fopen(fn, "rb"); - mu_assert(fh != NULL, "fopen() failed"); - - int result = fseek(fh, 0, SEEK_END); - mu_assert(result == 0, "fseek(SEEK_END) failed %d", result); - - long pos = ftell(fh); - mu_assert(pos != -1, "ftell() failed %s", strerror(errno)); - rewind(fh); - - str = malloc(pos + 1); - mu_assert(str != NULL, "malloc failed"); - - size_t b = fread(str, 1, pos, fh); - mu_assert((long)b == pos, "fread() failed"); - str[pos] = 0; - - fclose(fh); - return str; -} - - int process(lsb_lua_sandbox *lsb, double ts) { static const char *func_name = "process"; @@ -834,11 +809,11 @@ static char* test_serialize() mu_assert(!e, "lsb_destroy() received: %s", e); #ifdef LUA_JIT - char *expected = read_file("output/serialize.data"); + char *expected = lsb_read_file("output/serialize.data"); #else - char *expected = read_file("output/serialize.lua51.data"); + char *expected = lsb_read_file("output/serialize.lua51.data"); #endif - char *actual = read_file(output_file); + char *actual = lsb_read_file(output_file); mu_assert(strcmp(expected, actual) == 0, "serialization mismatch"); free(expected); free(actual); diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 9e90ae7..2f0933b 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -13,6 +13,7 @@ #include "../luasandbox_impl.h" #include "luasandbox_output.h" +#include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" @@ -490,8 +491,8 @@ bool lsb_read_heka_field(lsb_heka_message *m, int lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) { static const size_t needed = 18; - ob->pos = 0; // writing a uuid will always clear the buffer since it is the - // start of a new message + lsb_clear_output_buffer(ob); // writing a uuid will always clear the buffer + //since it is the start of a new message int result = lsb_expand_output_buffer(ob, needed); if (result) return result; diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index bb28ee0..725a791 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -26,7 +26,8 @@ int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) b->maxsize = max_message_size; b->pos = 0; b->buf = malloc(b->size); - return b->buf ? 0 : 2; + b->err = b->buf ? 0 : 2; + return b->err; } @@ -35,18 +36,26 @@ void lsb_free_output_buffer(lsb_output_buffer *b) free(b->buf); b->buf = NULL; b->size = 0; + lsb_clear_output_buffer(b); +} + + +void lsb_clear_output_buffer(lsb_output_buffer *b) +{ b->pos = 0; + b->err = 0; } int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) { - if (needed <= b->size - b->pos) { - return 0; - } + if (b->err) return b->err; + + if (needed <= b->size - b->pos) return 0; if (b->maxsize && needed + b->pos > b->maxsize) { - return 1; + b->err = 1; + return b->err; } size_t newsize = lsb_lp2(b->pos + needed); @@ -55,7 +64,10 @@ int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) } void *ptr = realloc(b->buf, newsize); - if (!ptr) return 2; + if (!ptr) { + b->err = 2; + return b->err; + } b->buf = ptr; b->size = newsize; @@ -96,7 +108,8 @@ int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) if (b->maxsize && (b->size >= b->maxsize || b->pos + needed >= b->maxsize)) { - return 1; // exceeded max + b->err = 1; + return b->err; // exceeded max } size_t newsize = b->size * 2; while ((size_t)needed >= newsize - b->pos) { @@ -112,7 +125,8 @@ int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) b->buf = p; b->size = newsize; } else { - return 2; // malloc failed + b->err = 2; + return b->err; // malloc failed } } else { b->pos += needed; diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c index c8b8f7d..b89fb4f 100644 --- a/src/util/test/test_protobuf.c +++ b/src/util/test/test_protobuf.c @@ -115,6 +115,7 @@ static char* test_lsb_pb_write_bool() mu_assert(ob.buf[0] == 1, "received: %02hhx", ob.buf[0]); mu_assert(lsb_pb_write_bool(&ob, 7), "buffer should be full"); + ob.err = 0; ob.pos = 0; mu_assert_rv(0, lsb_pb_write_bool(&ob, 0)); mu_assert(ob.buf[0] == 0, "received: %02hhx", ob.buf[0]); @@ -157,7 +158,7 @@ static char* test_lsb_pb_write_string() ob.pos = ob.maxsize - 3; mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); - ob.pos = 0; + lsb_clear_output_buffer(&ob); mu_assert_rv(0, lsb_pb_write_string(&ob, 1, foo, len)); mu_assert(ob.pos = len + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(memcmp("\x0a\x03" "foo", ob.buf, 5) == 0, "received: " @@ -190,6 +191,7 @@ static char* test_lsb_pb_update_field_length() ob.pos = 512; mu_assert_rv(1, lsb_pb_update_field_length(&ob, 512)); + ob.err = 0; ob.buf[301] = 'x'; ob.pos = 302; mu_assert_rv(0, lsb_pb_update_field_length(&ob, 1)); diff --git a/src/util/util.c b/src/util/util.c index 6c2e9e3..f1bf8c8 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -6,6 +6,10 @@ /** General purpose utility functions @file */ + +#include +#include + #include "luasandbox/util/util.h" size_t lsb_lp2(unsigned long long x) @@ -32,3 +36,29 @@ double lsb_timespec_delta(const struct timespec* s, const struct timespec* e) } return delta; } + + +char* lsb_read_file(const char *fn) +{ + char *str = NULL; + FILE *fh = fopen(fn, "rb"); + if (!fh) return str; + + if (fseek(fh, 0, SEEK_END)) goto cleanup; + long pos = ftell(fh); + if (pos == -1) goto cleanup; + rewind(fh); + + str = malloc(pos + 1); + if (!str) goto cleanup; + + size_t b = fread(str, 1, pos, fh); + if ((long)b == pos) { + str[pos] = 0; + } + +cleanup: + fclose(fh); + + return str; +} From c13b863882473572a0e9ffb005ecbc021ff62a13 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 26 Jan 2016 14:58:35 -0800 Subject: [PATCH 096/235] Add access to the collected Heka sandbox stats - add lsb_heka_get_stats and lsb_heka_is_running API - rename lsb_heka_process_message_* API to lsb_heka_pm_* --- include/luasandbox/heka/sandbox.h | 43 +++++++++++-- src/heka_sandbox/sandbox.c | 39 ++++++++++-- src/heka_sandbox/test/test_sandbox.c | 95 ++++++++++++++++++---------- src/luasandbox.c | 3 +- src/test/test_luasandbox.c | 5 +- 5 files changed, 138 insertions(+), 47 deletions(-) diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 5eb6e86..0ece047 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -30,6 +30,21 @@ enum lsb_heka_pm_rv { typedef struct lsb_heka_sandbox lsb_heka_sandbox; +typedef struct lsb_heka_stats { + unsigned long long mem_cur; + unsigned long long mem_max; + unsigned long long ins_max; + unsigned long long out_max; + unsigned long long im_cnt; + unsigned long long im_bytes; + unsigned long long pm_cnt; + unsigned long long pm_failures; + double pm_avg; + double pm_sd; + double te_avg; + double te_sd; +} lsb_heka_stats; + #ifdef __cplusplus extern "C" { @@ -117,7 +132,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, * */ LSB_EXPORT -int lsb_heka_process_message_input(lsb_heka_sandbox *hsb, +int lsb_heka_pm_input(lsb_heka_sandbox *hsb, lsb_heka_message *msg, double cp_numeric, const char *cp_string, @@ -158,7 +173,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, * */ LSB_EXPORT -int lsb_heka_process_message_analysis(lsb_heka_sandbox *hsb, +int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, lsb_heka_message *msg, bool profile); @@ -204,7 +219,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, * */ LSB_EXPORT -int lsb_heka_process_message_output(lsb_heka_sandbox *hsb, +int lsb_heka_pm_output(lsb_heka_sandbox *hsb, lsb_heka_message *msg, void *sequence_id, bool profile); @@ -241,7 +256,7 @@ lsb_heka_terminate_sandbox(lsb_heka_sandbox *lsb, const char *err); * FREED by the caller. * */ -LSB_EXPORT char * +LSB_EXPORT char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); /** @@ -276,8 +291,24 @@ LSB_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); */ LSB_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); -// todo need access to the sandbox statistics. For simplicity it will most -// likely return a Heka protobuf string with all the info +/** + * Retrieve the sandbox profiling/monitoring statistics. This call accesses + * internal data and is not thread safe. + * + * @param hsb Heka sandbox + * + * @return lsb_heka_stats A copy of the stats structure + */ +LSB_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); + +/** + * Queries the state of the sandbox. + * + * @param hsb Heka sandbox + * + * @return True if the sandbox has not been terminated + */ +LSB_EXPORT bool lsb_heka_is_running(lsb_heka_sandbox *hsb); #ifdef __cplusplus } diff --git a/src/heka_sandbox/sandbox.c b/src/heka_sandbox/sandbox.c index b1af4d9..dcc56f9 100644 --- a/src/heka_sandbox/sandbox.c +++ b/src/heka_sandbox/sandbox.c @@ -427,8 +427,8 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, return 1; } if (profile) { - clock_gettime(CLOCK_MONOTONIC, &ts); // todo fix on Mac & Win - lsb_update_running_stats(&hsb->stats.pm, lsb_timespec_delta(&ts1, &ts)); + clock_gettime(CLOCK_MONOTONIC, &ts1); // todo fix on Mac & Win + lsb_update_running_stats(&hsb->stats.pm, lsb_timespec_delta(&ts, &ts1)); } hsb->msg = NULL; @@ -490,7 +490,7 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, } -int lsb_heka_process_message_input(lsb_heka_sandbox *hsb, +int lsb_heka_pm_input(lsb_heka_sandbox *hsb, lsb_heka_message *msg, double cp_numeric, const char *cp_string, @@ -583,7 +583,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, } -int lsb_heka_process_message_analysis(lsb_heka_sandbox *hsb, +int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, lsb_heka_message *msg, bool profile) { @@ -689,7 +689,7 @@ char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) } -int lsb_heka_process_message_output(lsb_heka_sandbox *hsb, +int lsb_heka_pm_output(lsb_heka_sandbox *hsb, lsb_heka_message *msg, void *sequence_id, bool profile) @@ -766,3 +766,32 @@ const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb) if (!hsb) return NULL; return lsb_get_lua_file(hsb->lsb); } + + +lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb) +{ + if (!hsb) return (struct lsb_heka_stats){0}; + + return (struct lsb_heka_stats){ + .mem_cur = lsb_usage(hsb->lsb, LSB_UT_MEMORY, LSB_US_CURRENT), + .mem_max = lsb_usage(hsb->lsb, LSB_UT_MEMORY, LSB_US_MAXIMUM), + .out_max = lsb_usage(hsb->lsb, LSB_UT_OUTPUT, LSB_US_MAXIMUM), + .ins_max = lsb_usage(hsb->lsb, LSB_UT_INSTRUCTION, LSB_US_MAXIMUM), + .im_cnt = hsb->stats.im_cnt, + .im_bytes = hsb->stats.im_bytes, + .pm_cnt = hsb->stats.pm_cnt, + .pm_failures = hsb->stats.pm_failures, + .pm_avg = hsb->stats.pm.mean, + .pm_sd = lsb_sd_running_stats(&hsb->stats.pm), + .te_avg = hsb->stats.te.mean, + .te_sd = lsb_sd_running_stats(&hsb->stats.te) + }; +} + + +bool lsb_heka_is_running(lsb_heka_sandbox *hsb) +{ + if (!hsb) return false; + if (lsb_get_state(hsb->lsb) == LSB_RUNNING) return true; + return false; +} diff --git a/src/heka_sandbox/test/test_sandbox.c b/src/heka_sandbox/test/test_sandbox.c index 6f702b7..a8f7078 100644 --- a/src/heka_sandbox/test/test_sandbox.c +++ b/src/heka_sandbox/test/test_sandbox.c @@ -15,7 +15,6 @@ #include #include "luasandbox/heka/sandbox.h" -#include "../sandbox_impl.h" // {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; @@ -272,13 +271,38 @@ static char* test_timer_event() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(0 < stats.mem_cur, "received %llu", stats.mem_cur); + mu_assert(0 < stats.mem_max, "received %llu", stats.mem_max); + mu_assert(0 == stats.out_max, "received %llu", stats.out_max); + mu_assert(0 < stats.ins_max, "received %llu", stats.ins_max); + mu_assert(0 == stats.pm_cnt, "received %llu", stats.pm_cnt); + mu_assert(0 == stats.pm_failures, "received %llu", stats.pm_failures); + mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); + mu_assert(0 == stats.te_avg, "received %g", stats.te_avg); + mu_assert(0 == stats.te_sd, "received %g", stats.te_sd); + mu_assert(true == lsb_heka_is_running(hsb), "not running"); + mu_assert(0 == lsb_heka_timer_event(hsb, 0, false), "err: %s", lsb_heka_get_error(hsb)); mu_assert(0 == lsb_heka_timer_event(hsb, 1, true), "err: %s", lsb_heka_get_error(hsb)); mu_assert(1 == lsb_heka_timer_event(hsb, 2, false), "err: %s", lsb_heka_get_error(hsb)); - mu_assert(hsb, "lsb_heka_create_analysis failed"); + mu_assert(false == lsb_heka_is_running(hsb), "running"); + + stats = lsb_heka_get_stats(hsb); + mu_assert(0 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(0 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(0 == stats.pm_cnt, "received %llu", stats.pm_cnt); + mu_assert(0 == stats.pm_failures, "received %llu", stats.pm_failures); + mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); + mu_assert(0 < stats.te_avg, "received %g", stats.te_avg); + mu_assert(0 < stats.te_sd, "received %g", stats.te_sd); + e = lsb_heka_destroy_sandbox(hsb); hsb = lsb_heka_create_analysis(NULL, "lua/input.lua", NULL, NULL, NULL, aim); @@ -315,8 +339,8 @@ static char* test_pm_input() hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ - int rv = lsb_heka_process_message_input(hsb, &m, results[i].ncp, - results[i].scp, false); + int rv = lsb_heka_pm_input(hsb, &m, results[i].ncp, + results[i].scp, true); const char *err = lsb_heka_get_error(hsb); mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", results[i].err, err); @@ -324,9 +348,11 @@ static char* test_pm_input() results[i].rv, rv); } - mu_assert(5 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); - mu_assert(1 == hsb->stats.pm_failures, "expected %llu", - hsb->stats.pm_failures); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(5 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(1 == stats.pm_failures, "expected %llu", stats.pm_failures); + mu_assert(0 < stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 < stats.pm_sd, "received %g", stats.pm_sd); e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; @@ -341,13 +367,15 @@ static char* test_pm_analysis() hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, aim); mu_assert(hsb, "lsb_heka_create_analysis failed"); - mu_assert_rv(0, lsb_heka_process_message_analysis(hsb, &m, false)); + mu_assert_rv(0, lsb_heka_pm_analysis(hsb, &m, false)); const char *err = lsb_heka_get_error(hsb); const char *eerr = ""; mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - mu_assert(1 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); - mu_assert(0 == hsb->stats.pm_failures, "expected %llu", - hsb->stats.pm_failures); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(1 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(0 == stats.pm_failures, "expected %llu", stats.pm_failures); + mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; @@ -361,7 +389,7 @@ static char* test_pm_no_return() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, dlog, aim); - mu_assert_rv(1, lsb_heka_process_message_analysis(hsb, &m, false)); + mu_assert_rv(1, lsb_heka_pm_analysis(hsb, &m, false)); const char *err = lsb_heka_get_error(hsb); const char *eerr = "process_message() must return a numeric status code"; mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); @@ -379,25 +407,26 @@ static char* test_pm_output() hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); - mu_assert_rv(0, lsb_heka_process_message_output(hsb, &m, NULL, false)); + mu_assert_rv(0, lsb_heka_pm_output(hsb, &m, NULL, false)); const char *err = lsb_heka_get_error(hsb); const char *eerr = ""; mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - mu_assert(0 == hsb->stats.pm_failures, "expected %llu", - hsb->stats.pm_failures); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(0 == stats.pm_failures, "expected %llu", stats.pm_failures); - mu_assert_rv(-5, lsb_heka_process_message_output(hsb, &m, (void *)99, false)); + mu_assert_rv(-5, lsb_heka_pm_output(hsb, &m, (void *)99, false)); mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - mu_assert(2 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); - mu_assert(7 == hsb->stats.pm_failures, "expected %llu", - hsb->stats.pm_failures); + stats = lsb_heka_get_stats(hsb); + mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); - mu_assert_rv(-3, lsb_heka_process_message_output(hsb, &m, (void *)100, false)); + mu_assert_rv(-3, lsb_heka_pm_output(hsb, &m, (void *)100, + false)); mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); - mu_assert(2 == hsb->stats.pm_cnt, "expected %llu", hsb->stats.pm_cnt); - mu_assert(7 == hsb->stats.pm_failures, "expected %llu", - hsb->stats.pm_failures); + stats = lsb_heka_get_stats(hsb); + mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); @@ -412,9 +441,9 @@ static char* test_im_input() "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); - mu_assert(6 == hsb->stats.im_cnt, "expected %llu", hsb->stats.im_cnt); - mu_assert(303 == hsb->stats.im_bytes, "expected %llu", - hsb->stats.im_bytes); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(6 == stats.im_cnt, "expected %llu", stats.im_cnt); + mu_assert(303 == stats.im_bytes, "expected %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -427,9 +456,9 @@ static char* test_im_analysis() hsb = lsb_heka_create_analysis(NULL, "lua/aim.lua", NULL, "Hostname = 'foo.com';Logger = 'aim'", dlog, aim); - mu_assert(3 == hsb->stats.im_cnt, "expected %llu", hsb->stats.im_cnt); - mu_assert(232 == hsb->stats.im_bytes, "expected %llu", - hsb->stats.im_bytes); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(3 == stats.im_cnt, "expected %llu", stats.im_cnt); + mu_assert(232 == stats.im_bytes, "expected %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_analysis failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -442,8 +471,8 @@ static char* test_encode_message() hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, "Hostname = 'sh';Logger = 'sl'", dlog, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); - size_t cur_out = lsb_usage(hsb->lsb, LSB_UT_OUTPUT, LSB_US_CURRENT); - mu_assert(33 == cur_out, "received %" PRIuSIZE, cur_out); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(33 == stats.out_max, "received %llu", stats.out_max); e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -473,7 +502,7 @@ static char* test_read_message() hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, dlog, aim); mu_assert(hsb, "lsb_heka_create_analysist failed"); - int rv = lsb_heka_process_message_analysis(hsb, &m, false); + int rv = lsb_heka_pm_analysis(hsb, &m, false); mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, lsb_heka_get_error(hsb)); e = lsb_heka_destroy_sandbox(hsb); @@ -494,7 +523,7 @@ static char* benchmark_decode_message() clock_t t = clock(); for (int x = 0; x < iter; ++x) { - mu_assert(0 == lsb_heka_process_message_output(hsb, &m, NULL, false), "%s", + mu_assert(0 == lsb_heka_pm_output(hsb, &m, NULL, false), "%s", lsb_heka_get_error(hsb)); } t = clock() - t; diff --git a/src/luasandbox.c b/src/luasandbox.c index 6eb331b..a73a15e 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -529,8 +529,7 @@ int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) return 2; } else { lua_gc(lsb->lua, LUA_GCCOLLECT, 0); - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = - (unsigned)instruction_usage(lsb); + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = instruction_usage(lsb); if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index d71c060..9152f62 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -17,6 +17,7 @@ #include "luasandbox/lua.h" #include "luasandbox_output.h" #include "luasandbox/util/util.h" +#include "../luasandbox_impl.h" #include "mu_test.h" @@ -1292,7 +1293,7 @@ static char* benchmark_serialize() } -static char* benchmark_deserialize() // todo need to alter since this now performs the serialization too +static char* benchmark_deserialize() { int iter = 1000; @@ -1304,6 +1305,8 @@ static char* benchmark_deserialize() // todo need to alter since this now perfor int result = lsb_init(sb, "output/serialize.data"); mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); + free(sb->state_file); + sb->state_file = NULL; e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } From ff34176e7d06939b0922c022dd715dca907c3e6e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 27 Jan 2016 14:45:40 -0800 Subject: [PATCH 097/235] Update docs - generate a cmake config - cleanup some minor todo's --- CMakeLists.txt | 21 +++++++++++- cmake/luasandboxConfig.cmake.in | 14 ++++++++ .../{analysis_plugins.md => analysis.md} | 2 +- docs/heka_sandbox/index.md | 34 +++++++++++++++++-- .../{input_plugins.md => input.md} | 10 +++--- docs/heka_sandbox/message.md | 12 +++---- docs/heka_sandbox/message_matcher.md | 2 +- .../{output_plugins.md => output.md} | 4 +-- docs/heka_sandbox/stream_reader.md | 4 +-- src/heka_sandbox/message_matcher.c | 14 -------- src/test/test_luasandbox.c | 8 ++--- src/util/util.c | 1 + 12 files changed, 87 insertions(+), 39 deletions(-) create mode 100644 cmake/luasandboxConfig.cmake.in rename docs/heka_sandbox/{analysis_plugins.md => analysis.md} (99%) rename docs/heka_sandbox/{input_plugins.md => input.md} (94%) rename docs/heka_sandbox/{output_plugins.md => output.md} (99%) diff --git a/CMakeLists.txt b/CMakeLists.txt index fd9253a..7a810fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,9 +31,28 @@ set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(mozsvc) include(externals) +set(LIB_INSTALL_DIR lib/) +set(INCLUDE_INSTALL_DIR include/) + include_directories("${EP_BASE}/include") include_directories("${CMAKE_SOURCE_DIR}/include") -install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${LIB_INSTALL_DIR}${PROJECT_NAME}/modules COMPONENT core) install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION share/doc/${PROJECT_NAME} COMPONENT core) +if(WIN32 AND NOT CYGWIN) + set(INSTALL_CMAKE_DIR cmake) +else() + set(INSTALL_CMAKE_DIR ${LIB_INSTALL_DIR}${CMAKE_PROJECT_NAME}/cmake) +endif() + +include(CMakePackageConfigHelpers) +configure_package_config_file(cmake/luasandboxConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake + INSTALL_DESTINATION ${INSTALL_CMAKE_DIR} + PATH_VARS INCLUDE_INSTALL_DIR LIB_INSTALL_DIR) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfigVersion.cmake + VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} + COMPATIBILITY AnyNewerVersion ) # todo change to SameMajorVersion after 1.0 +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfigVersion.cmake + DESTINATION ${INSTALL_CMAKE_DIR} COMPONENT core) + add_subdirectory(src) diff --git a/cmake/luasandboxConfig.cmake.in b/cmake/luasandboxConfig.cmake.in new file mode 100644 index 0000000..fcde622 --- /dev/null +++ b/cmake/luasandboxConfig.cmake.in @@ -0,0 +1,14 @@ +set(LUASANDBOX_VERSION x.y.z) + +@PACKAGE_INIT@ + +set_and_check(LUASANDBOX_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@") +set_and_check(LUASANDBOX_LIB_DIR "@PACKAGE_LIB_INSTALL_DIR@") + +FIND_LIBRARY(LUASANDBOX_LUASB_LIBRARY NAMES luasb PATHS ${PACKAGE_LIB_INSTALL_DIR}) +FIND_LIBRARY(LUASANDBOX_UTIL_LIBRARY NAMES luasandboxutil PATHS ${PACKAGE_LIB_INSTALL_DIR}) +FIND_LIBRARY(LUASANDBOX_LIBRARY NAMES luasandbox PATHS ${PACKAGE_LIB_INSTALL_DIR}) +FIND_LIBRARY(LUASANDBOX_HEKA_LIBRARY NAMES luasandboxheka PATHS ${PACKAGE_LIB_INSTALL_DIR}) +FIND_PATH(LUASANDBOX_MODULES NAMES modules PATHS ${PACKAGE_LIB_INSTALL_DIR}luasandbox) + +set(LUASANDBOX_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY} ${LUASANDBOX_LUASB_LIBRARY}) diff --git a/docs/heka_sandbox/analysis_plugins.md b/docs/heka_sandbox/analysis.md similarity index 99% rename from docs/heka_sandbox/analysis_plugins.md rename to docs/heka_sandbox/analysis.md index ca9a808..a609146 100644 --- a/docs/heka_sandbox/analysis_plugins.md +++ b/docs/heka_sandbox/analysis.md @@ -1,4 +1,4 @@ -## Analysis Plugins +## Analysis Sandbox ### Required Lua Functions (called by the host) diff --git a/docs/heka_sandbox/index.md b/docs/heka_sandbox/index.md index 96a0ba3..2651da2 100644 --- a/docs/heka_sandbox/index.md +++ b/docs/heka_sandbox/index.md @@ -12,7 +12,35 @@ of that infrastructure has been created to replace it called ## Table of Contents * [Message Matcher](message_matcher.md) -* [Input Plugins](input_plugins.md) +* [Input Sandbox](input.md) * [Heka Stream Reader](heka_stream_reader.md) -* [Analysis Plugins](analysis_plugins.md) -* [Output Plugins](output_plugins.md) +* [Analysis Sandbox](analysis.md) +* [Output Sandbox](output.md) + +## Sandbox API Changes from the Go Heka Sandbox + +There are a few intentional changes between tho original Heka sandbox and this version. + +### Changes +1. `read_message` now has a `framed` parameter to retrive the raw message with stream framing. +#### Input Sandbox +1. `inject_message` accepts a numeric or string checkpoint as the second argument +1. `process_message` receives the checkpoint value as the first argument (if it was provided by `inject_message`). +#### Output Sandbox +1. `process_message` receives a sequence_id as the first argument (if it was provided by `update_checkpoint`). + Extended return codes have been added to support skipping, retrying, batching, and asynchronous output. +#### Analysis/Output Sandbox +1. `timer_event` has a second parameter `shutdown` that is set to true when the sandbox is exiting. + +### Additions +#### Input Sandbox +1. A [Heka Stream Reader](heka_stream_reader.md) Lua module was added. +#### Output Sandbox +1. [update_checkpoint](output.md#update_checkpoint) was added for batch and asynchronous processing. + +### Removals +1. The `write_message` API was removed; messages are immutable and this API broke that rule. +1. The `read_next_field` API was removed; instead the raw message should be decoded and the Lua table iterated. + +### Notes +1. The `read_config` API in unchanged but now has access to the entire sandbox configuration. diff --git a/docs/heka_sandbox/input_plugins.md b/docs/heka_sandbox/input.md similarity index 94% rename from docs/heka_sandbox/input_plugins.md rename to docs/heka_sandbox/input.md index 7424deb..272540d 100644 --- a/docs/heka_sandbox/input_plugins.md +++ b/docs/heka_sandbox/input.md @@ -1,8 +1,8 @@ -## Input Plugins +## Input Sandbox ### Recommendations Since he sandbox does not run in isolation there are some expectations of how -the host infrastructure behaves. The current recommendation are based on the +the host infrastructure behaves. The current recommendation are based on the Hindsight reference implementation. ### Required Lua Functions (called by the host) @@ -12,8 +12,8 @@ Hindsight reference implementation. Entry point for message creation. *Arguments* -* offset (nil number, string) - value of the last offset value passed into - `inject_message` +* checkpoint (nil number, string) - value of the last checkpoint value passed + into `inject_message` *Return* * status_code (number) @@ -49,7 +49,7 @@ Sends a Heka protocol buffer message into the host. *Arguments* * msg ([Heka message table](message.md), [Heka stream reader](stream_reader.md) or Heka protobuf string) -* offset (optional: string, number) - checkpoint offset to be returned in the `process_message` call +* checkpoint (optional: number, string) - checkpoint to be returned in the `process_message` call *Return* * none - throws an error on invalid input diff --git a/docs/heka_sandbox/message.md b/docs/heka_sandbox/message.md index bb6d4f7..e6d651b 100644 --- a/docs/heka_sandbox/message.md +++ b/docs/heka_sandbox/message.md @@ -4,14 +4,14 @@ ```lua { -Uuid = "data", -- ignored if not 16 byte raw binary UUID -Logger = "nginx", -- ignored by analysis plugins (the plugin name is used instead) -Hostname = "example.com", -- ignored by analysis plugins (the Hindsight hostname name is used instead) -Timestamp = 1e9, -Type = "TEST", +Uuid = "data", -- auto generated if not a 16 byte raw binary UUID or a 36 character human readable UUID +Logger = "nginx", -- ignored by analysis sandboxes (the `Logger` configuration is used instead) +Hostname = "example.com", -- ignored by analysis sandboxes (the `Hostname` configuration is used instead) +Timestamp = 1e9, -- auto generated if not a number +Type = "TEST", Payload = "Test Payload", EnvVersion = "0.8", -Pid = 1234, -- ignored by analysis plugins (the Hindsight PID is used instead) +Pid = 1234, Severity = 6, Fields = { http_status = 200, -- encoded as a double diff --git a/docs/heka_sandbox/message_matcher.md b/docs/heka_sandbox/message_matcher.md index 1bcfe60..14fe727 100644 --- a/docs/heka_sandbox/message_matcher.md +++ b/docs/heka_sandbox/message_matcher.md @@ -1,6 +1,6 @@ ### Message Matcher Syntax -The message matcher allows plugins to select which messages they want to consume +The message matcher allows sandboxes to select which messages they want to consume (see [Heka Message Structure](message.md)) #### Examples diff --git a/docs/heka_sandbox/output_plugins.md b/docs/heka_sandbox/output.md similarity index 99% rename from docs/heka_sandbox/output_plugins.md rename to docs/heka_sandbox/output.md index 3ae5d75..8644847 100644 --- a/docs/heka_sandbox/output_plugins.md +++ b/docs/heka_sandbox/output.md @@ -1,4 +1,4 @@ -## Output Plugins +## Output Sandbox ### Required Lua Functions (called by the host) @@ -52,7 +52,7 @@ Provides access to the sandbox configuration variables. #### read_message -Provides access to the Heka message data. See [read_message](analysis_plugins.md#read_message) for details. +Provides access to the Heka message data. See [read_message](analysis.md#read_message) for details. #### decode_message diff --git a/docs/heka_sandbox/stream_reader.md b/docs/heka_sandbox/stream_reader.md index cf5e583..0edfdfb 100644 --- a/docs/heka_sandbox/stream_reader.md +++ b/docs/heka_sandbox/stream_reader.md @@ -1,7 +1,7 @@ ## Heka Stream Reader Module Enables parsing of a framed Heka protobuf stream in a Lua sandbox. See: -[Example of a Heka protobuf reader](input_plugins.md#example-of-a-heka-protobuf-stdin-reader) +[Example of a Heka protobuf reader](input.md#example-of-a-heka-protobuf-stdin-reader) ### API @@ -60,4 +60,4 @@ Provides access to the Heka message data within the reader object. local ts = hsr:read_message("Timestamp") ``` -See [read_message](analysis_plugins.md#read_message) for details. +See [read_message](analysis.md#read_message) for details. diff --git a/src/heka_sandbox/message_matcher.c b/src/heka_sandbox/message_matcher.c index 3f4b132..0f4008a 100644 --- a/src/heka_sandbox/message_matcher.c +++ b/src/heka_sandbox/message_matcher.c @@ -129,7 +129,6 @@ typedef enum { TYPE_STRING, TYPE_NUMERIC, TYPE_BOOLEAN, - TYPE_STRING_MATCH } match_type; @@ -268,7 +267,6 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) switch (mn->value_type) { case TYPE_STRING: - case TYPE_STRING_MATCH: if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) && val.type == LSB_READ_STRING) { return string_test(mn, &val.u.s); @@ -353,10 +351,8 @@ static void load_expression_node(lua_State *L, match_node *mn) } else if (strcmp(tmp, "<") == 0) { mn->op = OP_LT; } else if (strcmp(tmp, "=~") == 0) { - mn->value_type = TYPE_STRING_MATCH; mn->op = OP_RE; } else if (strcmp(tmp, "!~") == 0) { - mn->value_type = TYPE_STRING_MATCH; mn->op = OP_NRE; } else if (strcmp(tmp, "TRUE") == 0) { mn->op = OP_TRUE; @@ -497,26 +493,20 @@ lsb_create_message_matcher(const lsb_message_match_builder *mmb, { lua_getglobal(mmb->parser, "parse"); if (!lua_isfunction(mmb->parser, -1)) { - // todo logger fprintf(stderr, "message_matcher error: %s", - // lua_tostring(mmb->parser, -1)); return NULL; } lua_pushstring(mmb->parser, exp); if (lua_pcall(mmb->parser, 1, 2, 0)) { - // todo logger fprintf(stderr, "message_matcher error: %s", - // lua_tostring(mmb->parser, -1)); return NULL; } if (lua_type(mmb->parser, 1) != LUA_TTABLE) { - // todo logger fprintf(stderr, "parse failed"); return NULL; } int size = lua_tointeger(mmb->parser, 2); lsb_message_matcher *mm = calloc(sizeof(lsb_message_matcher) + (sizeof(match_node) * size), 1); if (!mm) { - // todo logger fprintf(stderr, "calloc failed"); return NULL; } mm->nodes_size = size; @@ -533,8 +523,6 @@ lsb_create_message_matcher(const lsb_message_match_builder *mmb, break; default: free(mm); - //todo logeer fprintf(stderr, - //"message_matcher error: invalid table returned"); return NULL; } lua_pop(mmb->parser, 1); @@ -545,7 +533,6 @@ lsb_create_message_matcher(const lsb_message_match_builder *mmb, match_node **stack = calloc(sizeof(match_node *) * size, 1); if (!stack) { free(mm); - // todo logger fprintf(stderr, "message_matcher stack allocation failed"); return NULL; } @@ -572,7 +559,6 @@ void lsb_destroy_message_matcher(lsb_message_matcher *mm) free(mm->nodes[i].variable); switch (mm->nodes[i].value_type) { case TYPE_STRING: - case TYPE_STRING_MATCH: free(mm->nodes[i].value.s); break; default: diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 9152f62..7c8d9e1 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -38,11 +38,11 @@ static const char *test_cfg = "instruction_limit = 0\n" "output_limit = 0\n" "remove_entries = {\n" - "[''] = {'collectgarbage','coroutine','dofile','load','loadfile'" - ",'loadstring','newproxy','print'},\n" + "[''] = {'collectgarbage','dofile','load','loadfile','loadstring'," + "'newproxy','print'},\n" "os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'}\n" "}\n" - "disable_modules = {io = 1}\n" + "disable_modules = {io = 1, coroutine = 1}\n" MODULE_PATH ; @@ -1306,7 +1306,7 @@ static char* benchmark_deserialize() mu_assert(result == 0, "lsb_init() received: %d %s", result, lsb_get_error(sb)); free(sb->state_file); - sb->state_file = NULL; + sb->state_file = NULL; // poke the internals to prevent serialization e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } diff --git a/src/util/util.c b/src/util/util.c index f1bf8c8..134d53d 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -8,6 +8,7 @@ #include +#include #include #include "luasandbox/util/util.h" From d4e5c07c1f204ba59c8932c863bad430196e34aa Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 28 Jan 2016 14:14:52 -0800 Subject: [PATCH 098/235] Fix up the Mac build - remove lsb_timespec_delta API - add lsb_get_time API --- CMakeLists.txt | 6 +++++ cmake/externals.cmake | 12 ++++----- include/luasandbox/util/util.h | 21 +++++---------- src/heka_sandbox/sandbox.c | 18 ++++++------- src/heka_sandbox/test/CMakeLists.txt | 14 ++++++++-- src/test/CMakeLists.txt | 21 +++++---------- src/test/mu_test.h | 2 +- src/util/test/test_heka_message.c | 2 +- src/util/test/test_input_buffer.c | 2 +- src/util/test/test_util.c | 37 +++++++++++++++++--------- src/util/util.c | 39 ++++++++++++++++++---------- 11 files changed, 101 insertions(+), 73 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a810fb..0b172ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -31,6 +31,12 @@ set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(mozsvc) include(externals) +include(CheckFunctionExists) +check_function_exists(clock_gettime HAVE_CLOCK_GETTIME) +if(HAVE_CLOCK_GETTIME) + add_definitions(-DHAVE_CLOCK_GETTIME) +endif() + set(LIB_INSTALL_DIR lib/) set(INCLUDE_INSTALL_DIR include/) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 453a119..50aa3bf 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -65,7 +65,7 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG e068a11788a23ea1542ecbd54c9f806a77780db2 + GIT_TAG 354ec76addd9f161a973d80ddf81c809552fb127 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -115,7 +115,7 @@ add_dependencies(lua_socket ${LUA_PROJECT}) externalproject_add( lua_bloom_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG c22db3f486a5debc269c7f20ff161f30852a4deb + GIT_TAG 255af01bc127fccbd11fc5e57218a5fcc8659c69 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -123,7 +123,7 @@ externalproject_add( externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG b70d98f33f1b7abefab5da1747f4922c6969b634 + GIT_TAG 85fc04e61e650ed65a7ba12ea02f8d307a5f1eab CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -131,7 +131,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 8d86410b27563f678ba0b88661b7975a7506be96 + GIT_TAG 4111ca2d22f47ed451b5e87dd192d53e9965800f CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -139,7 +139,7 @@ externalproject_add( externalproject_add( lua_cuckoo_filter GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG eab57dde3d7f8e35e6c42ce5fb4bf606c4064e1a + GIT_TAG bc7f406b3997fc9432bb5d1db96cad902bd3283a CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -147,7 +147,7 @@ externalproject_add( externalproject_add( lua_sax GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG 028e84132f48431bb01a945392b3a212e1a53a47 + GIT_TAG bd2426c93d0d55977ca7b21354105463b0470a49 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 5e90dbe..7b0ea31 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -27,20 +27,6 @@ extern "C" { */ LSB_EXPORT size_t lsb_lp2(unsigned long long x); - -/** - * Computes the duration between two timespec structures. - * - * @param s Start time - * @param e End time - * - * @return double Time difference in fractional seconds - * - */ -LSB_EXPORT double lsb_timespec_delta(const struct timespec* s, - const struct timespec* e); - - /** * Read a file into a string * @@ -51,6 +37,13 @@ LSB_EXPORT double lsb_timespec_delta(const struct timespec* s, */ LSB_EXPORT char* lsb_read_file(const char *fn); +/** + * Retrieves the highest resolution time available converted to nanoseconds + * + * @return unsigned long long + */ +LSB_EXPORT unsigned long long lsb_get_time(); + #ifdef __cplusplus } #endif diff --git a/src/heka_sandbox/sandbox.c b/src/heka_sandbox/sandbox.c index dcc56f9..6d44fea 100644 --- a/src/heka_sandbox/sandbox.c +++ b/src/heka_sandbox/sandbox.c @@ -410,11 +410,11 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, lua_State *lua, int nargs, bool profile) { - struct timespec ts, ts1; + unsigned long long start, end; hsb->msg = msg; if (profile) { - clock_gettime(CLOCK_MONOTONIC, &ts); // todo fix on Mac & Win + start = lsb_get_time(); } if (lua_pcall(lua, nargs, 2, 0) != 0) { char err[LSB_ERROR_SIZE]; @@ -427,8 +427,8 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, return 1; } if (profile) { - clock_gettime(CLOCK_MONOTONIC, &ts1); // todo fix on Mac & Win - lsb_update_running_stats(&hsb->stats.pm, lsb_timespec_delta(&ts, &ts1)); + end = lsb_get_time(); + lsb_update_running_stats(&hsb->stats.pm, end - start); } hsb->msg = NULL; @@ -734,8 +734,8 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) lua_pushnumber(lua, t * 1e9); lua_pushboolean(lua, shutdown); - struct timespec ts, ts1; - clock_gettime(CLOCK_MONOTONIC, &ts); // todo fix on Mac & Win + unsigned long long start, end; + start = lsb_get_time(); if (lua_pcall(lua, 2, 0, 0) != 0) { char err[LSB_ERROR_SIZE]; size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, @@ -746,8 +746,8 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) lsb_terminate(hsb->lsb, err); return 1; } - clock_gettime(CLOCK_MONOTONIC, &ts1); // todo fix on Mac & Win - lsb_update_running_stats(&hsb->stats.te, lsb_timespec_delta(&ts, &ts1)); + end = lsb_get_time(); + lsb_update_running_stats(&hsb->stats.te, end - start); lsb_pcall_teardown(hsb->lsb); lua_gc(lua, LUA_GCCOLLECT, 0); return 0; @@ -770,7 +770,7 @@ const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb) lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb) { - if (!hsb) return (struct lsb_heka_stats){0}; + if (!hsb) return (struct lsb_heka_stats){0,0,0,0,0,0,0,0,0,0,0,0}; return (struct lsb_heka_stats){ .mem_cur = lsb_usage(hsb->lsb, LSB_UT_MEMORY, LSB_US_CURRENT), diff --git a/src/heka_sandbox/test/CMakeLists.txt b/src/heka_sandbox/test/CMakeLists.txt index baaa6a1..1bbc17f 100644 --- a/src/heka_sandbox/test/CMakeLists.txt +++ b/src/heka_sandbox/test/CMakeLists.txt @@ -8,9 +8,19 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) add_executable(test_heka_sandbox test_sandbox.c) -target_link_libraries(test_heka_sandbox luasandboxheka luasandboxutil luasandbox luasb) +target_link_libraries(test_heka_sandbox luasandboxheka luasandbox) add_test(NAME test_heka_sandbox COMMAND test_heka_sandbox) add_executable(test_message_matcher test_message_matcher.c) -target_link_libraries(test_message_matcher luasandboxheka luasandboxutil luasb) +target_link_libraries(test_message_matcher luasandboxheka luasandboxutil) add_test(NAME test_message_matcher COMMAND test_message_matcher) + +if(WIN32) + set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/heka_sandbox") + set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_message_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) +elseif(APPLE) + set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib:${CMAKE_BINARY_DIR}/src:${CMAKE_BINARY_DIR}/src/util:${CMAKE_BINARY_DIR}/src/heka_sandbox") + set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) + set_tests_properties(test_message_matcher PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) +endif() diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index d5c174d..aa2b4a3 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -4,23 +4,16 @@ add_executable(test_luasandbox test_luasandbox.c) target_link_libraries(test_luasandbox luasandbox luasandboxutil) +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util") add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) -if(WIN32) - add_test(NAME test_move_lua_lib COMMAND cmake -E copy - ${CMAKE_BINARY_DIR}/ep_base/bin/${CMAKE_SHARED_MODULE_PREFIX}luasb${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) - add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy - ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_MODULE_PREFIX}luasandbox${CMAKE_SHARED_MODULE_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}) -elseif(APPLE) - add_test(NAME test_move_lua_lib COMMAND cmake -E copy_directory - ${CMAKE_BINARY_DIR}/ep_base/lib ${CMAKE_CURRENT_BINARY_DIR}/../lib) - add_test(NAME test_move_sandbox_lib COMMAND cmake -E copy - ${CMAKE_BINARY_DIR}/src/${CMAKE_SHARED_LIBRARY_PREFIX}luasandbox${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_CURRENT_BINARY_DIR}/../lib) -endif() - add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_luasandbox COMMAND test_luasandbox) -if(APPLE) - set_tests_properties(test_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=../lib) + +if(WIN32) + set_tests_properties(test_luasandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) +elseif(APPLE) + set_tests_properties(test_luasandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() + diff --git a/src/test/mu_test.h b/src/test/mu_test.h index c338ac8..206651b 100644 --- a/src/test/mu_test.h +++ b/src/test/mu_test.h @@ -63,4 +63,4 @@ do { \ int mu_tests_run = 0; char mu_err[MU_ERR_LEN] = { 0 }; -#endif \ No newline at end of file +#endif diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 5c73566..7e990c9 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -27,7 +27,7 @@ struct log_message { char msg[1024]; }; -static struct log_message log = { 0 }; +static struct log_message log = { .component = NULL, .severity = 0, .msg = {0} }; void logger(const char *component, int severity, const char *fmt, ...) { diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c index b5d9d45..db3f98c 100644 --- a/src/util/test/test_input_buffer.c +++ b/src/util/test/test_input_buffer.c @@ -20,7 +20,7 @@ static char* test_stub() static char* test_init_small_buf() { - size_t size = 1024; + size_t size = 300; lsb_input_buffer b; mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); mu_assert(b.size == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index 8a4a17d..c6d9dd8 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -33,18 +33,30 @@ static char* test_lsb_lp2() } -static char* test_lsb_timespec_delta() +static char* benchmark_lsb_get_time() { - double d; - d = lsb_timespec_delta( - &(struct timespec){ .tv_sec = 10, .tv_nsec = 600000000L }, - &(struct timespec){ .tv_sec = 11, .tv_nsec = 800000000L }); - mu_assert(d == 1.2, "received: %g", d); - - d = lsb_timespec_delta( - &(struct timespec){ .tv_sec = 10, .tv_nsec = 600000000L }, - &(struct timespec){ .tv_sec = 12, .tv_nsec = 300000000L }); - mu_assert(d == 1.7, "received: %g", d); + int iter = 1000000; + unsigned long long start, end; + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_get_time(); + } + t = clock() - t; + printf("benchmark_lsb_get_time(%d) - clock %g seconds\n", iter, + (double)t / CLOCKS_PER_SEC / iter); + + start = lsb_get_time(); + for (int x = 0; x < iter; ++x) { + lsb_get_time(); + } + end = lsb_get_time(); + printf("benchmark_lsb_get_time(%d) - self %g seconds\n", iter, + (double)(end - start) / iter / 1e9); + + start = lsb_get_time(); + lsb_get_time(); + end = lsb_get_time(); + printf("benchmark_lsb_get_time(1) %llu\n", end - start); return NULL; } @@ -53,7 +65,8 @@ static char* all_tests() { mu_run_test(test_stub); mu_run_test(test_lsb_lp2); - mu_run_test(test_lsb_timespec_delta); + + mu_run_test(benchmark_lsb_get_time); return NULL; } diff --git a/src/util/util.c b/src/util/util.c index 134d53d..20fbf76 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -6,10 +6,15 @@ /** General purpose utility functions @file */ - #include #include #include +#include + +#if defined(__MACH__) && defined(__APPLE__) +#include +#include +#endif #include "luasandbox/util/util.h" @@ -27,18 +32,6 @@ size_t lsb_lp2(unsigned long long x) } -double lsb_timespec_delta(const struct timespec* s, const struct timespec* e) -{ - double delta; - if (e->tv_nsec - s->tv_nsec < 0) { - delta = e->tv_sec - s->tv_sec - (e->tv_nsec - s->tv_nsec) / -1e9; - } else { - delta = e->tv_sec - s->tv_sec + (e->tv_nsec - s->tv_nsec) / 1e9; - } - return delta; -} - - char* lsb_read_file(const char *fn) { char *str = NULL; @@ -63,3 +56,23 @@ char* lsb_read_file(const char *fn) return str; } + + +unsigned long long lsb_get_time() +{ +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000000000ULL + ts.tv_nsec; +#elif defined(__MACH__) && defined(__APPLE__) + static unsigned long long convert = 0; + if (convert == 0) { + mach_timebase_info_data_t tbi; + (void) mach_timebase_info(&tbi); + convert = tbi.numer / tbi.denom; + } + return mach_absolute_time() * convert; +#endif +// todo add Win support + return 0; +} From e1633705f0566c689ca901258427a9bb3eb6a0f5 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Fri, 29 Jan 2016 19:38:30 -0800 Subject: [PATCH 099/235] Fix up the Windows MSVC 32/64 bit builds --- CMakeLists.txt | 1 + docs/{heka_sandbox => heka}/analysis.md | 0 docs/{heka_sandbox => heka}/index.md | 0 docs/{heka_sandbox => heka}/input.md | 0 docs/{heka_sandbox => heka}/message.md | 0 .../{heka_sandbox => heka}/message_matcher.md | 0 docs/{heka_sandbox => heka}/output.md | 0 docs/{heka_sandbox => heka}/stream_reader.md | 0 docs/index.md | 2 +- docs/sandbox.md | 2 +- include/luasandbox.h | 1 - include/luasandbox/heka/message_matcher.h | 17 +++--- include/luasandbox/heka/sandbox.h | 54 ++++++++++++------- include/luasandbox/util/heka_message.h | 46 ++++++++-------- include/luasandbox/util/input_buffer.h | 10 ++-- include/luasandbox/util/output_buffer.h | 29 +++++----- include/luasandbox/util/protobuf.h | 25 +++++---- include/luasandbox/util/running_stats.h | 8 +-- include/luasandbox/util/string.h | 4 +- include/luasandbox/util/string_matcher.h | 4 +- include/luasandbox/util/util.h | 23 +++++--- src/CMakeLists.txt | 16 ++++-- src/{heka_sandbox => heka}/CMakeLists.txt | 15 ++++-- src/{heka_sandbox => heka}/message.c | 16 +++--- src/{heka_sandbox => heka}/message_impl.h | 0 src/{heka_sandbox => heka}/message_matcher.c | 8 +-- src/{heka_sandbox => heka}/sandbox.c | 17 +++--- src/{heka_sandbox => heka}/sandbox_impl.h | 4 +- src/{heka_sandbox => heka}/stream_reader.c | 2 +- .../stream_reader_impl.h | 4 +- .../test/CMakeLists.txt | 5 +- src/{heka_sandbox => heka}/test/lua/aim.lua | 0 .../test/lua/analysis.lua | 5 ++ .../test/lua/decode_message.lua | 0 .../test/lua/decode_message_benchmark.lua | 0 .../test/lua/encode_message.lua | 0 src/{heka_sandbox => heka}/test/lua/iim.lua | 0 src/{heka_sandbox => heka}/test/lua/input.lua | 0 .../test/lua/output.lua | 0 .../test/lua/pm_no_return.lua | 0 .../test/lua/read_message.lua | 0 src/{heka_sandbox => heka}/test/test.h.in | 0 .../test/test_message_matcher.c | 0 .../test/test_sandbox.c | 0 src/luasandbox.c | 10 ++-- src/test/CMakeLists.txt | 4 +- src/test/test_luasandbox.c | 6 +-- src/util/CMakeLists.txt | 8 ++- src/util/heka_message.c | 6 +-- src/util/output_buffer.c | 6 +++ src/util/protobuf.c | 4 +- src/util/test/CMakeLists.txt | 11 ++++ src/util/test/test_heka_message.c | 16 +++--- src/util/test/test_input_buffer.c | 2 +- src/util/test/test_output_buffer.c | 13 +++-- src/util/test/test_running_stats.c | 6 +++ src/util/test/test_util.c | 1 + src/util/util.c | 30 +++++++++-- 58 files changed, 275 insertions(+), 166 deletions(-) rename docs/{heka_sandbox => heka}/analysis.md (100%) rename docs/{heka_sandbox => heka}/index.md (100%) rename docs/{heka_sandbox => heka}/input.md (100%) rename docs/{heka_sandbox => heka}/message.md (100%) rename docs/{heka_sandbox => heka}/message_matcher.md (100%) rename docs/{heka_sandbox => heka}/output.md (100%) rename docs/{heka_sandbox => heka}/stream_reader.md (100%) rename src/{heka_sandbox => heka}/CMakeLists.txt (52%) rename src/{heka_sandbox => heka}/message.c (98%) rename src/{heka_sandbox => heka}/message_impl.h (100%) rename src/{heka_sandbox => heka}/message_matcher.c (98%) rename src/{heka_sandbox => heka}/sandbox.c (98%) rename src/{heka_sandbox => heka}/sandbox_impl.h (92%) rename src/{heka_sandbox => heka}/stream_reader.c (99%) rename src/{heka_sandbox => heka}/stream_reader_impl.h (88%) rename src/{heka_sandbox => heka}/test/CMakeLists.txt (80%) rename src/{heka_sandbox => heka}/test/lua/aim.lua (100%) rename src/{heka_sandbox => heka}/test/lua/analysis.lua (77%) rename src/{heka_sandbox => heka}/test/lua/decode_message.lua (100%) rename src/{heka_sandbox => heka}/test/lua/decode_message_benchmark.lua (100%) rename src/{heka_sandbox => heka}/test/lua/encode_message.lua (100%) rename src/{heka_sandbox => heka}/test/lua/iim.lua (100%) rename src/{heka_sandbox => heka}/test/lua/input.lua (100%) rename src/{heka_sandbox => heka}/test/lua/output.lua (100%) rename src/{heka_sandbox => heka}/test/lua/pm_no_return.lua (100%) rename src/{heka_sandbox => heka}/test/lua/read_message.lua (100%) rename src/{heka_sandbox => heka}/test/test.h.in (100%) rename src/{heka_sandbox => heka}/test/test_message_matcher.c (100%) rename src/{heka_sandbox => heka}/test/test_sandbox.c (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b172ff..4ccc608 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,6 +23,7 @@ if(LUA_JIT) endif() find_library(LIBM_LIBRARY m) + find_package(Git REQUIRED) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") diff --git a/docs/heka_sandbox/analysis.md b/docs/heka/analysis.md similarity index 100% rename from docs/heka_sandbox/analysis.md rename to docs/heka/analysis.md diff --git a/docs/heka_sandbox/index.md b/docs/heka/index.md similarity index 100% rename from docs/heka_sandbox/index.md rename to docs/heka/index.md diff --git a/docs/heka_sandbox/input.md b/docs/heka/input.md similarity index 100% rename from docs/heka_sandbox/input.md rename to docs/heka/input.md diff --git a/docs/heka_sandbox/message.md b/docs/heka/message.md similarity index 100% rename from docs/heka_sandbox/message.md rename to docs/heka/message.md diff --git a/docs/heka_sandbox/message_matcher.md b/docs/heka/message_matcher.md similarity index 100% rename from docs/heka_sandbox/message_matcher.md rename to docs/heka/message_matcher.md diff --git a/docs/heka_sandbox/output.md b/docs/heka/output.md similarity index 100% rename from docs/heka_sandbox/output.md rename to docs/heka/output.md diff --git a/docs/heka_sandbox/stream_reader.md b/docs/heka/stream_reader.md similarity index 100% rename from docs/heka_sandbox/stream_reader.md rename to docs/heka/stream_reader.md diff --git a/docs/index.md b/docs/index.md index b7adeba..3448f4f 100644 --- a/docs/index.md +++ b/docs/index.md @@ -9,7 +9,7 @@ infrastructure and make it embeddable into any tool or language. ### Sandbox User APIs * [Sandbox](sandbox.md) -* [Heka Sandbox](heka_sandox/index.md) +* [Heka Sandbox](heka/index.md) ### Sandbox Developer APIs * todo - Links to the Doxygen output diff --git a/docs/sandbox.md b/docs/sandbox.md index 6e09b3f..de79406 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -127,5 +127,5 @@ The best place to start is with some examples: ### Heka Sandbox -[Heka Sandbox](heka_sandbox/index.html) +[Heka Sandbox](heka/index.html) diff --git a/include/luasandbox.h b/include/luasandbox.h index 3c9e739..eacf926 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -68,7 +68,6 @@ typedef void (*lsb_logger)(const char* component, ...); typedef struct lsb_lua_sandbox lsb_lua_sandbox; -typedef struct lsb_output_buffer lsb_output_buffer; #ifdef __cplusplus extern "C" { diff --git a/include/luasandbox/heka/message_matcher.h b/include/luasandbox/heka/message_matcher.h index 777b7ae..4b6ed18 100644 --- a/include/luasandbox/heka/message_matcher.h +++ b/include/luasandbox/heka/message_matcher.h @@ -6,12 +6,13 @@ /** Hindsight/Heka message matcher @file */ -#ifndef luasandbox_heka_sandbox_message_matcher_h_ -#define luasandbox_heka_sandbox_message_matcher_h_ +#ifndef luasandbox_heka_message_matcher_h_ +#define luasandbox_heka_message_matcher_h_ #include -#include "luasandbox/util/heka_message.h" +#include "../util/heka_message.h" +#include "sandbox.h" typedef struct lsb_message_matcher lsb_message_matcher; typedef struct lsb_message_match_builder lsb_message_match_builder; @@ -31,7 +32,7 @@ extern "C" { * match parser in the luasandboxutil lib * */ -LSB_EXPORT lsb_message_match_builder* +LSB_HEKA_EXPORT lsb_message_match_builder* lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath); /** @@ -39,7 +40,7 @@ lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath); * * @param mmb Message matcher builder */ -LSB_EXPORT void +LSB_HEKA_EXPORT void lsb_destroy_message_match_builder(lsb_message_match_builder *mmb); /** @@ -50,7 +51,7 @@ lsb_destroy_message_match_builder(lsb_message_match_builder *mmb); * * @return lsb_message_matcher* */ -LSB_EXPORT lsb_message_matcher* +LSB_HEKA_EXPORT lsb_message_matcher* lsb_create_message_matcher(const lsb_message_match_builder *mmb, const char *exp); @@ -59,7 +60,7 @@ lsb_create_message_matcher(const lsb_message_match_builder *mmb, * * @param mm Message matcher */ -LSB_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); +LSB_HEKA_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); /** * Evaluates the message matcher against the provided message @@ -69,7 +70,7 @@ LSB_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); * * @return bool True if the message is a match */ -LSB_EXPORT bool lsb_eval_message_matcher(lsb_message_matcher *mm, +LSB_HEKA_EXPORT bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m); #ifdef __cplusplus diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 0ece047..3e364c2 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -6,16 +6,30 @@ /** Heka sandbox implementation @file */ -#ifndef luasandbox_heka_sandbox_sandbox_h_ -#define luasandbox_heka_sandbox_sandbox_h_ +#ifndef luasandbox_heka_sandbox_h_ +#define luasandbox_heka_sandbox_h_ #include #include -#include "luasandbox.h" -#include "luasandbox/lua.h" -#include "luasandbox/lauxlib.h" -#include "luasandbox/util/heka_message.h" +#include "../../luasandbox.h" +#include "../lauxlib.h" +#include "../lua.h" +#include "../util/heka_message.h" + +#ifdef _WIN32 +#ifdef luasandboxheka_EXPORTS +#define LSB_HEKA_EXPORT __declspec(dllexport) +#else +#define LSB_HEKA_EXPORT __declspec(dllimport) +#endif +#else +#if __GNUC__ >= 4 +#define LSB_HEKA_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_HEKA_EXPORT +#endif +#endif #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" @@ -107,7 +121,7 @@ typedef int (*lsb_heka_update_checkpoint)(void *parent, void *sequence_id); * @param im inject_message call back * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ -LSB_EXPORT +LSB_HEKA_EXPORT lsb_heka_sandbox* lsb_heka_create_input(void *parent, const char *lua_file, const char *state_file, @@ -131,7 +145,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, * <0 non-fatal error (status message is logged) * */ -LSB_EXPORT +LSB_HEKA_EXPORT int lsb_heka_pm_input(lsb_heka_sandbox *hsb, lsb_heka_message *msg, double cp_numeric, @@ -151,7 +165,7 @@ int lsb_heka_pm_input(lsb_heka_sandbox *hsb, * @param im inject_message call back * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ -LSB_EXPORT +LSB_HEKA_EXPORT lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, const char *lua_file, const char *state_file, @@ -172,7 +186,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, * <0 non-fatal error (status message is logged) * */ -LSB_EXPORT +LSB_HEKA_EXPORT int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, lsb_heka_message *msg, bool profile); @@ -191,7 +205,7 @@ int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, * * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ -LSB_EXPORT +LSB_HEKA_EXPORT lsb_heka_sandbox* lsb_heka_create_output(void *parent, const char *lua_file, const char *state_file, @@ -218,7 +232,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, * -5 async output * */ -LSB_EXPORT +LSB_HEKA_EXPORT int lsb_heka_pm_output(lsb_heka_sandbox *hsb, lsb_heka_message *msg, void *sequence_id, @@ -234,7 +248,7 @@ int lsb_heka_pm_output(lsb_heka_sandbox *hsb, * @return * */ -LSB_EXPORT void +LSB_HEKA_EXPORT void lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb); /** @@ -243,7 +257,7 @@ lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb); * @param hsb Heka sandbox to terminate * @param err Reason for termination */ -LSB_EXPORT void +LSB_HEKA_EXPORT void lsb_heka_terminate_sandbox(lsb_heka_sandbox *lsb, const char *err); /** @@ -256,7 +270,7 @@ lsb_heka_terminate_sandbox(lsb_heka_sandbox *lsb, const char *err); * FREED by the caller. * */ -LSB_EXPORT char* +LSB_HEKA_EXPORT char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); /** @@ -269,7 +283,7 @@ lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); * * @return int 0 on success */ -LSB_EXPORT +LSB_HEKA_EXPORT int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown); @@ -280,7 +294,7 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown); * * @return const char* error message */ -LSB_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); +LSB_HEKA_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); /** * Returns the filename of the Lua source. @@ -289,7 +303,7 @@ LSB_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); * * @return const char* filename. */ -LSB_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); +LSB_HEKA_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); /** * Retrieve the sandbox profiling/monitoring statistics. This call accesses @@ -299,7 +313,7 @@ LSB_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); * * @return lsb_heka_stats A copy of the stats structure */ -LSB_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); +LSB_HEKA_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); /** * Queries the state of the sandbox. @@ -308,7 +322,7 @@ LSB_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); * * @return True if the sandbox has not been terminated */ -LSB_EXPORT bool lsb_heka_is_running(lsb_heka_sandbox *hsb); +LSB_HEKA_EXPORT bool lsb_heka_is_running(lsb_heka_sandbox *hsb); #ifdef __cplusplus } diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index 7b8cdac..e2c51c3 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -12,11 +12,11 @@ #include #include -#include "luasandbox.h" -#include "luasandbox/util/input_buffer.h" -#include "luasandbox/util/output_buffer.h" -#include "luasandbox/util/string.h" -#include "luasandbox/util/util.h" +#include "../../luasandbox.h" +#include "input_buffer.h" +#include "output_buffer.h" +#include "string.h" +#include "util.h" #define LSB_UUID_SIZE 16 #define LSB_UUID_STR_SIZE 36 @@ -122,7 +122,7 @@ extern "C" { * @return int 0 on success * */ -LSB_EXPORT int lsb_init_heka_message(lsb_heka_message *m, int num_fields); +LSB_UTIL_EXPORT int lsb_init_heka_message(lsb_heka_message *m, int num_fields); /** * Frees the memory allocated for the message fields @@ -130,7 +130,7 @@ LSB_EXPORT int lsb_init_heka_message(lsb_heka_message *m, int num_fields); * @param m Heka message structure * */ -LSB_EXPORT void lsb_free_heka_message(lsb_heka_message *m); +LSB_UTIL_EXPORT void lsb_free_heka_message(lsb_heka_message *m); /** * Resets the message headers and fields zeroing the allocated memory @@ -138,7 +138,7 @@ LSB_EXPORT void lsb_free_heka_message(lsb_heka_message *m); * @param m Heka message structure * */ -LSB_EXPORT void lsb_clear_heka_message(lsb_heka_message *m); +LSB_UTIL_EXPORT void lsb_clear_heka_message(lsb_heka_message *m); /** * Locates a framed Heka message in an input buffer @@ -151,11 +151,11 @@ LSB_EXPORT void lsb_clear_heka_message(lsb_heka_message *m); * * @return bool True on success */ -LSB_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, - lsb_input_buffer *ib, - bool decode, - size_t *discarded_bytes, - lsb_logger logger); +LSB_UTIL_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, + lsb_input_buffer *ib, + bool decode, + size_t *discarded_bytes, + lsb_logger logger); /** * Decodes an array of bytes into a Heka message. The message structure is @@ -169,10 +169,10 @@ LSB_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, * @return bool True on success * */ -LSB_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, - const char *buf, - size_t len, - lsb_logger logger); +LSB_UTIL_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, + const char *buf, + size_t len, + lsb_logger logger); /** * Reads a dynamic field from the Heka message @@ -185,11 +185,11 @@ LSB_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, * * @return bool True on success */ -LSB_EXPORT bool lsb_read_heka_field(lsb_heka_message *m, - lsb_const_string *name, - int fi, - int ai, - lsb_read_value *val); +LSB_UTIL_EXPORT bool lsb_read_heka_field(lsb_heka_message *m, + lsb_const_string *name, + int fi, + int ai, + lsb_read_value *val); /** * Writes a binary UUID to the output buffer @@ -201,7 +201,7 @@ LSB_EXPORT bool lsb_read_heka_field(lsb_heka_message *m, * * @return int 0 on success */ -LSB_EXPORT int +LSB_UTIL_EXPORT int lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len); #ifdef __cplusplus diff --git a/include/luasandbox/util/input_buffer.h b/include/luasandbox/util/input_buffer.h index 1153fc7..4fc3c14 100644 --- a/include/luasandbox/util/input_buffer.h +++ b/include/luasandbox/util/input_buffer.h @@ -12,7 +12,7 @@ #include #include -#include "luasandbox.h" +#include "util.h" typedef struct lsb_input_buffer { @@ -38,15 +38,15 @@ extern "C" { * * @return int 0 on success */ -LSB_EXPORT int lsb_init_input_buffer(lsb_input_buffer *b, - size_t max_message_size); +LSB_UTIL_EXPORT int lsb_init_input_buffer(lsb_input_buffer *b, + size_t max_message_size); /** * Frees the memory internally allocated by the buffer and resets the state * * @param b Input buffer */ -LSB_EXPORT void lsb_free_input_buffer(lsb_input_buffer *b); +LSB_UTIL_EXPORT void lsb_free_input_buffer(lsb_input_buffer *b); /** * Expands the input buffer (if necessary) to accomadate the requested number of @@ -58,7 +58,7 @@ LSB_EXPORT void lsb_free_input_buffer(lsb_input_buffer *b); * * @return int 0 on success */ -LSB_EXPORT int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len); +LSB_UTIL_EXPORT int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len); #ifdef __cplusplus } diff --git a/include/luasandbox/util/output_buffer.h b/include/luasandbox/util/output_buffer.h index 2a3b13e..5d1d4a2 100644 --- a/include/luasandbox/util/output_buffer.h +++ b/include/luasandbox/util/output_buffer.h @@ -11,17 +11,17 @@ #include -#include "luasandbox.h" +#include "util.h" #define LSB_OUTPUT_SIZE 1024 -struct lsb_output_buffer { +typedef struct lsb_output_buffer { char *buf; size_t maxsize; size_t size; size_t pos; int err; -}; +} lsb_output_buffer; #ifdef __cplusplus extern "C" { @@ -36,7 +36,7 @@ extern "C" { * * @return int 0 on success */ -LSB_EXPORT +LSB_UTIL_EXPORT int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); /** @@ -44,14 +44,14 @@ int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); * * @param b Output buffer */ -LSB_EXPORT void lsb_free_output_buffer(lsb_output_buffer *b); +LSB_UTIL_EXPORT void lsb_free_output_buffer(lsb_output_buffer *b); /** * Resets the position and error state of the buffer * * @param b Output buffer */ -LSB_EXPORT void lsb_clear_output_buffer(lsb_output_buffer *b); +LSB_UTIL_EXPORT void lsb_clear_output_buffer(lsb_output_buffer *b); /** * Resize the output buffer when more space is needed. @@ -61,7 +61,8 @@ LSB_EXPORT void lsb_clear_output_buffer(lsb_output_buffer *b); * * @return int 0 on success */ -LSB_EXPORT int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed); +LSB_UTIL_EXPORT int lsb_expand_output_buffer(lsb_output_buffer *b, + size_t needed); /** * Append a character to the output buffer. @@ -71,7 +72,7 @@ LSB_EXPORT int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed); * * @return int 0 on success, false if out of memory. */ -LSB_EXPORT int lsb_outputc(lsb_output_buffer *b, char ch); +LSB_UTIL_EXPORT int lsb_outputc(lsb_output_buffer *b, char ch); /** * Append a formatted string to the output buffer. @@ -81,7 +82,7 @@ LSB_EXPORT int lsb_outputc(lsb_output_buffer *b, char ch); * * @return int 0 on success, false if out of memory. */ -LSB_EXPORT int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); +LSB_UTIL_EXPORT int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); /** * Append a fixed string to the output buffer. @@ -92,9 +93,9 @@ LSB_EXPORT int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); * * @return int 0 on success, false if out of memory. */ -LSB_EXPORT int lsb_outputs(lsb_output_buffer *b, - const char *str, - size_t len); +LSB_UTIL_EXPORT int lsb_outputs(lsb_output_buffer *b, + const char *str, + size_t len); /** * More efficient output of a double to a string. NaN/Inf check and then calls @@ -105,7 +106,7 @@ LSB_EXPORT int lsb_outputs(lsb_output_buffer *b, * * @return int 0 on success, false if out of memory. */ -LSB_EXPORT int lsb_outputd(lsb_output_buffer *output, double d); +LSB_UTIL_EXPORT int lsb_outputd(lsb_output_buffer *output, double d); /** @@ -116,7 +117,7 @@ LSB_EXPORT int lsb_outputd(lsb_output_buffer *output, double d); * * @return int 0 on success, false if out of memory. */ -LSB_EXPORT int lsb_outputfd(lsb_output_buffer *ob, double d); +LSB_UTIL_EXPORT int lsb_outputfd(lsb_output_buffer *ob, double d); #ifdef __cplusplus } diff --git a/include/luasandbox/util/protobuf.h b/include/luasandbox/util/protobuf.h index e7619ea..cf90ca9 100644 --- a/include/luasandbox/util/protobuf.h +++ b/include/luasandbox/util/protobuf.h @@ -9,12 +9,11 @@ #ifndef luasandbox_util_protobuf_h_ #define luasandbox_util_protobuf_h_ -#include "luasandbox.h" - #include #include -#include "luasandbox/util/output_buffer.h" +#include "output_buffer.h" +#include "util.h" #define LSB_MAX_VARINT_BYTES 10 @@ -41,7 +40,7 @@ extern "C" { * * @return LSB_EXPORT const char* */ -LSB_EXPORT +LSB_UTIL_EXPORT const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype); /** @@ -53,7 +52,7 @@ const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype); * * @return int 0 on success */ -LSB_EXPORT int +LSB_UTIL_EXPORT int lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, unsigned char wiretype); @@ -66,7 +65,7 @@ lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, * * @return const char* Position in the buffer after the varint */ -LSB_EXPORT +LSB_UTIL_EXPORT const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi); @@ -78,7 +77,7 @@ const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi); * * @return int Number of bytes written */ -LSB_EXPORT int lsb_pb_output_varint(char* buf, unsigned long long i); +LSB_UTIL_EXPORT int lsb_pb_output_varint(char *buf, unsigned long long i); /** * Writes a varint encoded number to the output buffer. @@ -88,7 +87,7 @@ LSB_EXPORT int lsb_pb_output_varint(char* buf, unsigned long long i); * * @return int 0 on success */ -LSB_EXPORT int +LSB_UTIL_EXPORT int lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i); /** @@ -99,7 +98,7 @@ lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i); * * @return int 0 on success */ -LSB_EXPORT int lsb_pb_write_bool(lsb_output_buffer *ob, int i); +LSB_UTIL_EXPORT int lsb_pb_write_bool(lsb_output_buffer *ob, int i); /** * Writes a double to the output buffer. @@ -109,7 +108,7 @@ LSB_EXPORT int lsb_pb_write_bool(lsb_output_buffer *ob, int i); * * @return int 0 on success */ -LSB_EXPORT int lsb_pb_write_double(lsb_output_buffer *ob, double i); +LSB_UTIL_EXPORT int lsb_pb_write_double(lsb_output_buffer *ob, double i); /** * Writes a string to the output buffer. @@ -121,8 +120,8 @@ LSB_EXPORT int lsb_pb_write_double(lsb_output_buffer *ob, double i); * * @return int 0 on success */ -LSB_EXPORT int -lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char* s, size_t len); +LSB_UTIL_EXPORT int +lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len); /** * Updates the field length in the output buffer once the size is known, this @@ -134,7 +133,7 @@ lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char* s, size_t len); * * @return int 0 on success */ -LSB_EXPORT int +LSB_UTIL_EXPORT int lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos); #ifdef __cplusplus diff --git a/include/luasandbox/util/running_stats.h b/include/luasandbox/util/running_stats.h index f1bf231..af04082 100644 --- a/include/luasandbox/util/running_stats.h +++ b/include/luasandbox/util/running_stats.h @@ -10,7 +10,7 @@ #ifndef lsb_util_running_stats_h_ #define lsb_util_running_stats_h_ -#include "luasandbox.h" +#include "util.h" typedef struct lsb_running_stats { @@ -28,7 +28,7 @@ extern "C" { * * @param s Stat structure to zero out */ -LSB_EXPORT void lsb_init_running_stats(lsb_running_stats *s); +LSB_UTIL_EXPORT void lsb_init_running_stats(lsb_running_stats *s); /** * Value to add to the running stats @@ -36,7 +36,7 @@ LSB_EXPORT void lsb_init_running_stats(lsb_running_stats *s); * @param s Stat structure * @param d Value to add */ -LSB_EXPORT void lsb_update_running_stats(lsb_running_stats *s, double d); +LSB_UTIL_EXPORT void lsb_update_running_stats(lsb_running_stats *s, double d); /** * Return the standard deviation of the stats @@ -45,7 +45,7 @@ LSB_EXPORT void lsb_update_running_stats(lsb_running_stats *s, double d); * * @return double Standard deviation of the stats up to this point */ -LSB_EXPORT double lsb_sd_running_stats(lsb_running_stats *s); +LSB_UTIL_EXPORT double lsb_sd_running_stats(lsb_running_stats *s); #ifdef __cplusplus } diff --git a/include/luasandbox/util/string.h b/include/luasandbox/util/string.h index 9c2803f..2d334d8 100644 --- a/include/luasandbox/util/string.h +++ b/include/luasandbox/util/string.h @@ -9,7 +9,7 @@ #ifndef luasandbox_util_string_h_ #define luasandbox_util_string_h_ -#include "luasandbox.h" +#include "util.h" typedef struct lsb_const_string { @@ -28,7 +28,7 @@ extern "C" { * @param s Pointer to the struct * */ -LSB_EXPORT void lsb_init_const_string(lsb_const_string *s); +LSB_UTIL_EXPORT void lsb_init_const_string(lsb_const_string *s); #ifdef __cplusplus } diff --git a/include/luasandbox/util/string_matcher.h b/include/luasandbox/util/string_matcher.h index 1758e27..3f7dddd 100644 --- a/include/luasandbox/util/string_matcher.h +++ b/include/luasandbox/util/string_matcher.h @@ -12,7 +12,7 @@ #include #include -#include "luasandbox.h" +#include "util.h" #ifdef __cplusplus extern "C" { @@ -28,7 +28,7 @@ extern "C" { * * @return bool True if the sting matches the pattern */ -LSB_EXPORT bool lsb_string_match(const char* s, size_t len, const char* p); +LSB_UTIL_EXPORT bool lsb_string_match(const char *s, size_t len, const char *p); #ifdef __cplusplus } diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 7b0ea31..5d8b263 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -9,10 +9,21 @@ #ifndef luasandbox_util_util_h_ #define luasandbox_util_util_h_ -#include +#include -#include "luasandbox.h" -#include "output_buffer.h" +#ifdef _WIN32 +#ifdef luasandboxutil_EXPORTS +#define LSB_UTIL_EXPORT __declspec(dllexport) +#else +#define LSB_UTIL_EXPORT __declspec(dllimport) +#endif +#else +#if __GNUC__ >= 4 +#define LSB_UTIL_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_UTIL_EXPORT +#endif +#endif #ifdef __cplusplus extern "C" { @@ -25,7 +36,7 @@ extern "C" { * * @return size_t Least power of 2 greater than or equal to x */ -LSB_EXPORT size_t lsb_lp2(unsigned long long x); +LSB_UTIL_EXPORT size_t lsb_lp2(unsigned long long x); /** * Read a file into a string @@ -35,14 +46,14 @@ LSB_EXPORT size_t lsb_lp2(unsigned long long x); * @return char* NULL on failure otherwise a pointer to the file contents (must * be freed by the caller). */ -LSB_EXPORT char* lsb_read_file(const char *fn); +LSB_UTIL_EXPORT char* lsb_read_file(const char *fn); /** * Retrieves the highest resolution time available converted to nanoseconds * * @return unsigned long long */ -LSB_EXPORT unsigned long long lsb_get_time(); +LSB_UTIL_EXPORT unsigned long long lsb_get_time(); #ifdef __cplusplus } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index aa6ae7c..bcaf288 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,32 +20,38 @@ add_dependencies(lua_circular_buffer luasandbox) add_dependencies(lua_hyperloglog luasandbox) add_dependencies(lua_cuckoo_filter luasandbox) +if(WIN32) + set(LIB_DIR bin) +else() + set(LIB_DIR lib) +endif() + if(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/luasb.lib") - install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib COMPONENT core PATTERN "*.dll") + install(DIRECTORY "${EP_BASE}/${LIB_DIR}/" DESTINATION ${LIB_DIR} COMPONENT core PATTERN "*.dll") elseif(MINGW) add_definitions(-D_MINGW) set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/libluasb.dll") set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) - install(DIRECTORY "${EP_BASE}/bin/" DESTINATION lib COMPONENT core PATTERN "*.dll") + install(DIRECTORY "${EP_BASE}/${LIB_DIR}/" DESTINATION ${LIB_DIR} COMPONENT core PATTERN "*.dll") else() if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") set(LINK_DL "-ldl") endif() - set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} -lm) + set(LUA_SANDBOX_LIBS "${EP_BASE}/${LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} ${LIBM_LIBRARY}) endif() set_target_properties(luasandbox PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) target_link_libraries(luasandbox luasandboxutil ${LUA_SANDBOX_LIBS}) -install(TARGETS luasandbox DESTINATION lib COMPONENT core) +install(TARGETS luasandbox DESTINATION ${LIB_DIR} COMPONENT core) install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib COMPONENT core PATTERN "lua" EXCLUDE) install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) install(DIRECTORY "${EP_BASE}/io/lib/lua/" DESTINATION lib/${PROJECT_NAME}/io_modules COMPONENT core) install(DIRECTORY "${EP_BASE}/include/" DESTINATION include COMPONENT core) add_subdirectory(util) -add_subdirectory(heka_sandbox) +add_subdirectory(heka) add_subdirectory(test) diff --git a/src/heka_sandbox/CMakeLists.txt b/src/heka/CMakeLists.txt similarity index 52% rename from src/heka_sandbox/CMakeLists.txt rename to src/heka/CMakeLists.txt index dfd876f..983d8a5 100644 --- a/src/heka_sandbox/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -2,16 +2,23 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -set(HEKA_SANDBOX_SRC +set(HEKA_SRC message.c message_matcher.c sandbox.c stream_reader.c ) -add_library(luasandboxheka SHARED ${HEKA_SANDBOX_SRC}) +add_library(luasandboxheka SHARED ${HEKA_SRC}) set_target_properties(luasandboxheka PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) -target_link_libraries(luasandboxheka luasandbox ${LIBM_LIBRARY}) +target_compile_definitions(luasandboxheka PUBLIC -Dluasandboxheka_EXPORTS) +target_link_libraries(luasandboxheka luasandbox) +if(WIN32) + target_link_libraries(luasandboxheka ws2_32) +endif() +if(LIBM_LIBRARY) + target_link_libraries(luasandboxheka ${LIBM_LIBRARY}) +endif() -install(TARGETS luasandboxheka DESTINATION lib COMPONENT core) +install(TARGETS luasandboxheka DESTINATION ${LIB_DIR} COMPONENT core) add_subdirectory(test) diff --git a/src/heka_sandbox/message.c b/src/heka/message.c similarity index 98% rename from src/heka_sandbox/message.c rename to src/heka/message.c index 0e77bb6..9fabd41 100644 --- a/src/heka_sandbox/message.c +++ b/src/heka/message.c @@ -63,7 +63,7 @@ static const char* read_string(lua_State *lua, if (!p || len < 0 || p + len > e) { return NULL; } - lua_pushlstring(lua, p, len); + lua_pushlstring(lua, p, (size_t)len); p += len; return p; } @@ -84,7 +84,7 @@ static const char* process_varint(lua_State *lua, if (!p) { return NULL; } - lua_pushnumber(lua, val); + lua_pushnumber(lua, (lua_Number)val); lua_setfield(lua, stack_index, name); return p; } @@ -144,7 +144,7 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) case 0: p = lsb_pb_read_varint(p, p + len, &val); if (!p) break; - lua_pushnumber(lua, val); + lua_pushnumber(lua, (lua_Number)val); lua_rawseti(lua, 5, ++value_count); break; case 2: @@ -880,7 +880,7 @@ int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) lua_getfield(lsb->lua, idx, LSB_TIMESTAMP); long long ts; if (lua_isnumber(lsb->lua, -1)) { - ts = lua_tonumber(lsb->lua, -1); + ts = (long long)lua_tonumber(lsb->lua, -1); } else { ts = time(NULL) * 1000000000LL; } @@ -922,9 +922,9 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) } size_t field_len; const char *field = luaL_checklstring(lua, 1, &field_len); - int fi = luaL_optinteger(lua, 2, 0); + int fi = (int)luaL_optinteger(lua, 2, 0); luaL_argcheck(lua, fi >= 0, 2, "field index must be >= 0"); - int ai = luaL_optinteger(lua, 3, 0); + int ai = (int)luaL_optinteger(lua, 3, 0); luaL_argcheck(lua, ai >= 0, 3, "array index must be >= 0"); if (strcmp(field, LSB_UUID) == 0) { @@ -934,7 +934,7 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) lua_pushnil(lua); } } else if (strcmp(field, LSB_TIMESTAMP) == 0) { - lua_pushnumber(lua, m->timestamp); + lua_pushnumber(lua, (lua_Number)m->timestamp); } else if (strcmp(field, LSB_TYPE) == 0) { if (m->type.s) { lua_pushlstring(lua, m->type.s, m->type.len); @@ -998,7 +998,7 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) lua_pushnumber(lua, v.u.d); break; case LSB_READ_BOOL: - lua_pushboolean(lua, v.u.d); + lua_pushboolean(lua, v.u.d ? 1 : 0); break; default: lua_pushnil(lua); diff --git a/src/heka_sandbox/message_impl.h b/src/heka/message_impl.h similarity index 100% rename from src/heka_sandbox/message_impl.h rename to src/heka/message_impl.h diff --git a/src/heka_sandbox/message_matcher.c b/src/heka/message_matcher.c similarity index 98% rename from src/heka_sandbox/message_matcher.c rename to src/heka/message_matcher.c index 0f4008a..77f9e57 100644 --- a/src/heka_sandbox/message_matcher.c +++ b/src/heka/message_matcher.c @@ -242,7 +242,7 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) default: switch (mn->id) { case LSB_PB_TIMESTAMP: - return numeric_test(mn, m->timestamp); + return numeric_test(mn, (double)m->timestamp); case LSB_PB_TYPE: return string_test(mn, &m->type); case LSB_PB_LOGGER: @@ -372,11 +372,11 @@ static void load_expression_node(lua_State *L, match_node *mn) if (lua_type(L, -1) == LUA_TNUMBER) { mn->id = LSB_PB_FIELDS; } - mn->fi = lua_tointeger(L, -1); + mn->fi = (int)lua_tointeger(L, -1); lua_pop(L, 1); lua_getfield(L, -1, "ai"); - mn->ai = lua_tointeger(L, -1); + mn->ai = (int)lua_tointeger(L, -1); lua_pop(L, 1); lua_getfield(L, -1, "variable"); @@ -503,7 +503,7 @@ lsb_create_message_matcher(const lsb_message_match_builder *mmb, if (lua_type(mmb->parser, 1) != LUA_TTABLE) { return NULL; } - int size = lua_tointeger(mmb->parser, 2); + int size = (int)lua_tointeger(mmb->parser, 2); lsb_message_matcher *mm = calloc(sizeof(lsb_message_matcher) + (sizeof(match_node) * size), 1); if (!mm) { diff --git a/src/heka_sandbox/sandbox.c b/src/heka/sandbox.c similarity index 98% rename from src/heka_sandbox/sandbox.c rename to src/heka/sandbox.c index 6d44fea..a49afee 100644 --- a/src/heka_sandbox/sandbox.c +++ b/src/heka/sandbox.c @@ -12,17 +12,18 @@ #include #include -#include "luasandbox/heka/message_matcher.h" #include "luasandbox.h" +#include "luasandbox/heka/message_matcher.h" +#include "luasandbox/util/protobuf.h" +#include "luasandbox/util/running_stats.h" #include "luasandbox_output.h" #include "message_impl.h" #include "sandbox_impl.h" #include "stream_reader_impl.h" -#include "luasandbox/util/protobuf.h" -#include "luasandbox/util/running_stats.h" #ifdef _WIN32 #include +#define snprintf _snprintf #else #include #endif @@ -227,7 +228,7 @@ static int update_checkpoint(lua_State *lua) switch (n) { case 2: // async case luaL_checktype(lua, 2, LUA_TNUMBER); - hsb->stats.pm_failures += lua_tonumber(lua, 2); + hsb->stats.pm_failures += (unsigned long long)lua_tonumber(lua, 2); // fall thru case 1: luaL_checktype(lua, 1, LUA_TLIGHTUSERDATA); @@ -428,7 +429,7 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, } if (profile) { end = lsb_get_time(); - lsb_update_running_stats(&hsb->stats.pm, end - start); + lsb_update_running_stats(&hsb->stats.pm, (double)(end - start)); } hsb->msg = NULL; @@ -719,7 +720,9 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) { static const char *func_name = "timer_event"; - if (!hsb || (hsb->type != 'o' && hsb->type != 'a')) return 1; + if (!hsb || (hsb->type != 'o' && hsb->type != 'a')) { + return 1; + } lua_State *lua = lsb_get_lua(hsb->lsb); if (!lua) return 1; @@ -747,7 +750,7 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) return 1; } end = lsb_get_time(); - lsb_update_running_stats(&hsb->stats.te, end - start); + lsb_update_running_stats(&hsb->stats.te, (double)(end - start)); lsb_pcall_teardown(hsb->lsb); lua_gc(lua, LUA_GCCOLLECT, 0); return 0; diff --git a/src/heka_sandbox/sandbox_impl.h b/src/heka/sandbox_impl.h similarity index 92% rename from src/heka_sandbox/sandbox_impl.h rename to src/heka/sandbox_impl.h index 267e2ac..f26e6aa 100644 --- a/src/heka_sandbox/sandbox_impl.h +++ b/src/heka/sandbox_impl.h @@ -6,8 +6,8 @@ /** Hindsight Heka sandbox private implementation @file */ -#ifndef luasandbox_heka_sandbox_sandbox_impl_h_ -#define luasandbox_heka_sandbox_sandbox_impl_h_ +#ifndef luasandbox_heka_sandbox_impl_h_ +#define luasandbox_heka_sandbox_impl_h_ #include "luasandbox.h" #include "luasandbox/heka/sandbox.h" diff --git a/src/heka_sandbox/stream_reader.c b/src/heka/stream_reader.c similarity index 99% rename from src/heka_sandbox/stream_reader.c rename to src/heka/stream_reader.c index e0735d8..eb70e83 100644 --- a/src/heka_sandbox/stream_reader.c +++ b/src/heka/stream_reader.c @@ -171,7 +171,7 @@ static int hsr_find_message(lua_State* lua) hsr->buf.size - hsr->buf.readpos, fh); hsr->buf.readpos += nread; - lua_pushnumber(lua, nread); + lua_pushnumber(lua, (lua_Number)nread); } } else { // update bytes needed if (found) { diff --git a/src/heka_sandbox/stream_reader_impl.h b/src/heka/stream_reader_impl.h similarity index 88% rename from src/heka_sandbox/stream_reader_impl.h rename to src/heka/stream_reader_impl.h index 10df60f..6512fba 100644 --- a/src/heka_sandbox/stream_reader_impl.h +++ b/src/heka/stream_reader_impl.h @@ -6,8 +6,8 @@ /** Hindsight Heka stream reader structures @file */ -#ifndef luasandbox_heka_sandbox_stream_reader_h_ -#define luasandbox_heka_sandbox_stream_reader_h_ +#ifndef luasandbox_heka_stream_reader_h_ +#define luasandbox_heka_stream_reader_h_ #include "luasandbox/lua.h" diff --git a/src/heka_sandbox/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt similarity index 80% rename from src/heka_sandbox/test/CMakeLists.txt rename to src/heka/test/CMakeLists.txt index 1bbc17f..9b759bf 100644 --- a/src/heka_sandbox/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -4,6 +4,7 @@ configure_file(test.h.in test.h ESCAPE_QUOTES) include_directories(${CMAKE_CURRENT_BINARY_DIR}) +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/${LIB_DIR};${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/heka") add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) @@ -16,11 +17,11 @@ target_link_libraries(test_message_matcher luasandboxheka luasandboxutil) add_test(NAME test_message_matcher COMMAND test_message_matcher) if(WIN32) - set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/heka_sandbox") + STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) set_tests_properties(test_message_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) elseif(APPLE) - set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib:${CMAKE_BINARY_DIR}/src:${CMAKE_BINARY_DIR}/src/util:${CMAKE_BINARY_DIR}/src/heka_sandbox") + STRING(REPLACE ";" ":" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) set_tests_properties(test_message_matcher PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() diff --git a/src/heka_sandbox/test/lua/aim.lua b/src/heka/test/lua/aim.lua similarity index 100% rename from src/heka_sandbox/test/lua/aim.lua rename to src/heka/test/lua/aim.lua diff --git a/src/heka_sandbox/test/lua/analysis.lua b/src/heka/test/lua/analysis.lua similarity index 77% rename from src/heka_sandbox/test/lua/analysis.lua rename to src/heka/test/lua/analysis.lua index 26fa72c..1211f27 100644 --- a/src/heka_sandbox/test/lua/analysis.lua +++ b/src/heka/test/lua/analysis.lua @@ -7,6 +7,11 @@ function process_message() end function timer_event(ns, shutdown) + local cnt = 0 + for i=1, 10000 do + cnt = cnt + 1 -- make sure we have something to measure even with a low res clock + end + if ns == 1e9 then assert(shutdown, "not shutting down") return diff --git a/src/heka_sandbox/test/lua/decode_message.lua b/src/heka/test/lua/decode_message.lua similarity index 100% rename from src/heka_sandbox/test/lua/decode_message.lua rename to src/heka/test/lua/decode_message.lua diff --git a/src/heka_sandbox/test/lua/decode_message_benchmark.lua b/src/heka/test/lua/decode_message_benchmark.lua similarity index 100% rename from src/heka_sandbox/test/lua/decode_message_benchmark.lua rename to src/heka/test/lua/decode_message_benchmark.lua diff --git a/src/heka_sandbox/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua similarity index 100% rename from src/heka_sandbox/test/lua/encode_message.lua rename to src/heka/test/lua/encode_message.lua diff --git a/src/heka_sandbox/test/lua/iim.lua b/src/heka/test/lua/iim.lua similarity index 100% rename from src/heka_sandbox/test/lua/iim.lua rename to src/heka/test/lua/iim.lua diff --git a/src/heka_sandbox/test/lua/input.lua b/src/heka/test/lua/input.lua similarity index 100% rename from src/heka_sandbox/test/lua/input.lua rename to src/heka/test/lua/input.lua diff --git a/src/heka_sandbox/test/lua/output.lua b/src/heka/test/lua/output.lua similarity index 100% rename from src/heka_sandbox/test/lua/output.lua rename to src/heka/test/lua/output.lua diff --git a/src/heka_sandbox/test/lua/pm_no_return.lua b/src/heka/test/lua/pm_no_return.lua similarity index 100% rename from src/heka_sandbox/test/lua/pm_no_return.lua rename to src/heka/test/lua/pm_no_return.lua diff --git a/src/heka_sandbox/test/lua/read_message.lua b/src/heka/test/lua/read_message.lua similarity index 100% rename from src/heka_sandbox/test/lua/read_message.lua rename to src/heka/test/lua/read_message.lua diff --git a/src/heka_sandbox/test/test.h.in b/src/heka/test/test.h.in similarity index 100% rename from src/heka_sandbox/test/test.h.in rename to src/heka/test/test.h.in diff --git a/src/heka_sandbox/test/test_message_matcher.c b/src/heka/test/test_message_matcher.c similarity index 100% rename from src/heka_sandbox/test/test_message_matcher.c rename to src/heka/test/test_message_matcher.c diff --git a/src/heka_sandbox/test/test_sandbox.c b/src/heka/test/test_sandbox.c similarity index 100% rename from src/heka_sandbox/test/test_sandbox.c rename to src/heka/test/test_sandbox.c diff --git a/src/luasandbox.c b/src/luasandbox.c index a73a15e..dd508c1 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -157,7 +157,7 @@ static int unprotected_panic(lua_State *lua) static int get_usage_config(lua_State *lua, int idx, const char *item) { lua_getfield(lua, idx, item); - int i = lua_tointeger(lua, -1); + int i = (int)lua_tointeger(lua, -1); lua_pop(lua, 1); return i; } @@ -276,7 +276,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) if (kt == LUA_TSTRING) { lua_setfield(sb, -2, lua_tostring(cfg, -2)); } else { - lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); } } } @@ -286,7 +286,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) if (kt == LUA_TSTRING) { lua_setfield(sb, -2, lua_tostring(cfg, -2)); } else { - lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); } break; case LUA_TBOOLEAN: @@ -294,7 +294,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) if (kt == LUA_TSTRING) { lua_setfield(sb, -2, lua_tostring(cfg, -2)); } else { - lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); } break; case LUA_TTABLE: @@ -323,7 +323,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) lua_setfield(sb, -2, lua_tostring(cfg, -2)); break; case LUA_TNUMBER: - lua_rawseti(sb, -2, lua_tointeger(cfg, -2)); + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); break; } } diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index aa2b4a3..a6d1d36 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -4,7 +4,7 @@ add_executable(test_luasandbox test_luasandbox.c) target_link_libraries(test_luasandbox luasandbox luasandboxutil) -set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util") +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/${LIB_DIR};${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util") add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) @@ -12,8 +12,10 @@ add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE add_test(NAME test_luasandbox COMMAND test_luasandbox) if(WIN32) + STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_luasandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) elseif(APPLE) + STRING(REPLACE ";" ":" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_luasandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 7c8d9e1..4983e12 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -27,7 +27,7 @@ size_t written_data_len = 0; #ifdef _WIN32 -#define MODULE_PATH "path = 'modules\\?.lua';cpath = 'modules\\?.dll'\n" +#define MODULE_PATH "path = 'modules\\\\?.lua';cpath = 'modules\\\\?.dll'\n" #else #define MODULE_PATH "path = 'modules/?.lua';cpath = 'modules/?.so'\n" #endif @@ -710,8 +710,8 @@ static char* test_errors() "instruction_limit = 1000;" "output_limit = 128;" #ifdef _WIN32 - "path = 'lua\\?.lua'" - "cpath = 'lua\\?.dll'", + "path = 'lua\\\\?.lua'" + "cpath = 'lua\\\\?.dll'", #else "path = 'lua/?.lua'" "cpath = 'lua/?.so'", diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index eab9c23..518f63b 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -15,8 +15,12 @@ util.c add_library(luasandboxutil SHARED ${UTIL_SRC}) set_target_properties(luasandboxutil PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) -target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) +target_compile_definitions(luasandboxutil PUBLIC -Dluasandboxutil_EXPORTS) -install(TARGETS luasandboxutil DESTINATION lib COMPONENT core) +if(LIBM_LIBRARY) + target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) +endif() + +install(TARGETS luasandboxutil DESTINATION ${LIB_DIR} COMPONENT core) add_subdirectory(test) diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 2f0933b..ee7cbd4 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -28,7 +28,7 @@ static size_t decode_header(char *buf, size_t len, size_t max_message_size) long long vi; if (lsb_pb_read_varint(p + 1, buf + len, &vi)) { if (vi > 0 && vi <= (long long)max_message_size) { - return vi; + return (size_t)vi; } } } @@ -49,7 +49,7 @@ read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) return NULL; } s->s = p; - s->len = vi; + s->len = (size_t)vi; return p + vi; } @@ -295,7 +295,7 @@ bool lsb_decode_heka_message(lsb_heka_message *m, if (!cp) { if (logger) { - logger(__FUNCTION__, 4, "tag:%d wiretype:%d position:%td", tag, + logger(__FUNCTION__, 4, "tag:%d wiretype:%d position:%d", tag, wiretype, lp - buf); } return false; diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index 725a791..0afb48a 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -9,12 +9,18 @@ #include "luasandbox/util/output_buffer.h" #include +#include #include #include #include #include "luasandbox/util/util.h" +#ifdef _MSC_VER +// To silence the +/-INFINITY warning +#pragma warning( disable : 4756 ) +#pragma warning( disable : 4056 ) +#endif int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) { diff --git a/src/util/protobuf.c b/src/util/protobuf.c index 51922db..1084ca4 100644 --- a/src/util/protobuf.c +++ b/src/util/protobuf.c @@ -109,7 +109,9 @@ lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len) int lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos) { - if (len_pos >= ob->pos) return 1; + if (len_pos >= ob->pos) { + return 1; + } size_t len = ob->pos - len_pos - 1; if (len < 128) { diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt index f41496d..968686c 100644 --- a/src/util/test/CMakeLists.txt +++ b/src/util/test/CMakeLists.txt @@ -29,3 +29,14 @@ add_test(NAME test_util COMMAND test_util) add_executable(test_heka_message test_heka_message.c) target_link_libraries(test_heka_message luasandboxutil) add_test(NAME test_heka_message COMMAND test_heka_message) + +if(WIN32) + set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src/util") + set_tests_properties(test_input_buffer PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_output_buffer PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_protobuf PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_string_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_running_stats PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_util PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_heka_message PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) +endif() diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 7e990c9..c34ea7a 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -27,15 +27,15 @@ struct log_message { char msg[1024]; }; -static struct log_message log = { .component = NULL, .severity = 0, .msg = {0} }; +static struct log_message lm = { .component = NULL, .severity = 0, .msg = {0} }; void logger(const char *component, int severity, const char *fmt, ...) { - log.component = component; - log.severity = severity; + lm.component = component; + lm.severity = severity; va_list args; va_start(args, fmt); - vsnprintf(log.msg, sizeof log.msg, fmt, args); + vsnprintf(lm.msg, sizeof lm.msg, fmt, args); va_end(args); } @@ -87,7 +87,7 @@ static char* test_decode() mu_assert(!lsb_init_heka_message(&m, 1), "failed"); for (unsigned i = 0; i < sizeof tests / sizeof(lsb_const_string); ++i){ bool ok = lsb_decode_heka_message(&m, tests[i].s, tests[i].len, logger); - mu_assert(ok, "test: %d failed err: %s", i, log.msg); + mu_assert(ok, "test: %d failed err: %s", i, lm.msg); } lsb_free_heka_message(&m); return NULL; @@ -136,8 +136,8 @@ static char* test_decode_failure() for (unsigned i = 0; i < sizeof(tests) / sizeof(struct decode_failure); ++i) { bool ok = lsb_decode_heka_message(&m, tests[i].s, strlen(tests[i].s), logger); mu_assert(!ok, "test: %u no error generated", i); - mu_assert(!strcmp(log.msg, tests[i].e), "test: %u expected: %s received: %s", - i, tests[i].e, log.msg); + mu_assert(!strcmp(lm.msg, tests[i].e), "test: %u expected: %s received: %s", + i, tests[i].e, lm.msg); } lsb_free_heka_message(&m); return NULL; @@ -152,7 +152,7 @@ static char* test_find_message() size_t d; }; -#define add_input(str, result, discard) {.s = (lsb_const_string){.s = str, .len = sizeof str - 1}, .b = result, .d = discard}, +#define add_input(str, result, discard) {.s = {.s = str, .len = sizeof str - 1}, .b = result, .d = discard}, struct find_message tests[] = { add_input("\x1e\x02\x08\x14\x1f" TEST_UUID TEST_NS, true, 0) // full message add_input("\x1e", false, 0) diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c index db3f98c..7220577 100644 --- a/src/util/test/test_input_buffer.c +++ b/src/util/test/test_input_buffer.c @@ -20,7 +20,7 @@ static char* test_stub() static char* test_init_small_buf() { - size_t size = 300; + size_t size = 100; lsb_input_buffer b; mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); mu_assert(b.size == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index a33f0f0..5f6271a 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -6,6 +6,7 @@ /** @brief lsb_output_buffer unit tests @file */ +#include #include #include @@ -13,6 +14,12 @@ #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/heka_message.h" +#ifdef _MSC_VER +// To silence the +/-INFINITY warning +#pragma warning( disable : 4756 ) +#pragma warning( disable : 4056 ) +#endif + static char* test_stub() { return NULL; @@ -137,13 +144,13 @@ static char* test_outputd() lsb_outputd(&b, d - 1); mu_assert(strcmp("10.12147483648-2147483649", b.buf) == 0, "received: %s", b.buf); - lsb_outputd(&b, 0.0/0.0); + lsb_outputd(&b, NAN); mu_assert(strcmp("10.12147483648-2147483649nan", b.buf) == 0, "received: %s", b.buf); - lsb_outputd(&b, 1.0/0.0); + lsb_outputd(&b, INFINITY); mu_assert(strcmp("10.12147483648-2147483649naninf", b.buf) == 0, "received: %s", b.buf); - lsb_outputd(&b, -1.0/0.0); + lsb_outputd(&b, -INFINITY); mu_assert(strcmp("10.12147483648-2147483649naninf-inf", b.buf) == 0, "received: %s", b.buf); lsb_free_output_buffer(&b); diff --git a/src/util/test/test_running_stats.c b/src/util/test/test_running_stats.c index 22a59c7..6ce53e3 100644 --- a/src/util/test/test_running_stats.c +++ b/src/util/test/test_running_stats.c @@ -11,6 +11,12 @@ #include "../../test/mu_test.h" #include "luasandbox/util/running_stats.h" +#ifdef _MSC_VER +// To silence the +/-INFINITY warning +#pragma warning( disable : 4756 ) +#pragma warning( disable : 4056 ) +#endif + static char* test_stub() { return NULL; diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index c6d9dd8..2a664c2 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -7,6 +7,7 @@ /** @brief lsb_util unit tests @file */ #include +#include #include "../../test/mu_test.h" #include "luasandbox/util/util.h" diff --git a/src/util/util.c b/src/util/util.c index 20fbf76..7082a4a 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -11,6 +11,13 @@ #include #include +#ifdef _WIN32 +#include +#include +#else +#include +#endif + #if defined(__MACH__) && defined(__APPLE__) #include #include @@ -35,6 +42,7 @@ size_t lsb_lp2(unsigned long long x) char* lsb_read_file(const char *fn) { char *str = NULL; + size_t b; FILE *fh = fopen(fn, "rb"); if (!fh) return str; @@ -46,14 +54,13 @@ char* lsb_read_file(const char *fn) str = malloc(pos + 1); if (!str) goto cleanup; - size_t b = fread(str, 1, pos, fh); + b = fread(str, 1, pos, fh); if ((long)b == pos) { str[pos] = 0; } cleanup: fclose(fh); - return str; } @@ -72,7 +79,22 @@ unsigned long long lsb_get_time() convert = tbi.numer / tbi.denom; } return mach_absolute_time() * convert; +#elif defined(_WIN32) + static unsigned long long qpf = ULLONG_MAX; + static_assert(sizeof(LARGE_INTEGER) == sizeof qpf, "size mismatch"); + + unsigned long long t; + if (qpf == ULLONG_MAX) QueryPerformanceFrequency((LARGE_INTEGER*)&qpf); + if (qpf){ + QueryPerformanceCounter((LARGE_INTEGER*)&t); + return (t / qpf * 1000000000ULL) + ((t % qpf) * 1000000000ULL / qpf); + } else { + GetSystemTimeAsFileTime((FILETIME*)&t); + return t * 100ULL; + } +#else + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000000ULL + tv.tv_usec * 1000ULL; #endif -// todo add Win support - return 0; } From 641cafa03fd155389e6708486f6dcfe8cd0f5122 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 29 Jan 2016 19:45:30 -0800 Subject: [PATCH 100/235] Add in the missing limits.h header --- src/util/output_buffer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index 0afb48a..1b35e09 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -8,6 +8,7 @@ #include "luasandbox/util/output_buffer.h" +#include #include #include #include From d0199ddca6815c016cd4ca6e7c4903c11a4d086b Mon Sep 17 00:00:00 2001 From: jefff Date: Mon, 1 Feb 2016 16:27:57 -0800 Subject: [PATCH 101/235] Allow negative numbers in integer definition --- modules/common_log_format.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index e5967e1..3264820 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -24,7 +24,7 @@ local unreserved = l.alnum + l.S"-._~" local sub_delims = l.S"!$&'()*+,;=" local last_literal = l.space -local integer = l.digit^1 / tonumber +local integer = l.P"-"^-1 * l.digit^1 / tonumber local double = l.digit^1 * "." * l.digit^1 / tonumber local msec_time = double / dt.seconds_to_ns local host = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) From 53bdd64026d863ccb7df972594968c95a8b3f7d8 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 4 Feb 2016 08:11:38 -0800 Subject: [PATCH 102/235] Alter the API error returns, introduce lsb_err_value - build out the unit tests and update the coverage stats --- CMakeLists.txt | 2 +- cmake/externals.cmake | 2 +- covfn.txt | 308 +++++++++---------- include/luasandbox.h | 31 +- include/luasandbox/heka/sandbox.h | 43 ++- include/luasandbox/util/heka_message.h | 12 +- include/luasandbox/util/input_buffer.h | 11 +- include/luasandbox/util/output_buffer.h | 55 ++-- include/luasandbox/util/protobuf.h | 26 +- include/luasandbox/util/util.h | 7 + include/luasandbox_serialize.h | 29 +- src/heka/message.c | 323 ++++++++++---------- src/heka/message_impl.h | 4 +- src/heka/message_matcher.c | 5 + src/heka/sandbox.c | 60 ++-- src/heka/sandbox_impl.h | 4 +- src/heka/stream_reader.c | 100 +++--- src/heka/test/lua/aim.lua | 16 + src/heka/test/lua/analysis.lua | 3 + src/heka/test/lua/encode_message.lua | 70 ++++- src/heka/test/lua/iim.lua | 59 ++++ src/heka/test/lua/input.lua | 12 +- src/heka/test/lua/output.lua | 9 + src/heka/test/test_message_matcher.c | 15 + src/heka/test/test_sandbox.c | 115 ++++++- src/luasandbox.c | 94 +++--- src/luasandbox_impl.h | 18 ++ src/luasandbox_output.c | 11 +- src/luasandbox_serialize.c | 212 ++++++------- src/test/mu_test.h | 2 +- src/test/test_luasandbox.c | 385 +++++++++++++----------- src/util/heka_message.c | 54 ++-- src/util/input_buffer.c | 20 +- src/util/output_buffer.c | 101 +++---- src/util/protobuf.c | 95 +++--- src/util/test/test_heka_message.c | 37 ++- src/util/test/test_input_buffer.c | 15 +- src/util/test/test_output_buffer.c | 27 +- src/util/test/test_protobuf.c | 61 ++-- src/util/test/test_string_matcher.c | 11 +- src/util/test/test_util.c | 17 ++ src/util/util.c | 5 + 42 files changed, 1481 insertions(+), 1005 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4ccc608..fe56424 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 12) +set(CPACK_PACKAGE_VERSION_MINOR 13) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 50aa3bf..c64e13b 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -131,7 +131,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 4111ca2d22f47ed451b5e87dd192d53e9965800f + GIT_TAG 745756887dd4c277bee0337ab307eae9ea95890e CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/covfn.txt b/covfn.txt index fce484b..986d544 100644 --- a/covfn.txt +++ b/covfn.txt @@ -1,152 +1,156 @@ -Function Source Line FnCov C/D Coverage ------------------------------------------------------------------------------ --------------------------------------------------- ----- --------------------- -lsb_get_parent(lua_sandbox*) ../src/luasandbox.c 457 0 / 1 0 / 0 -circular_buffer_version(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 858 0 / 1 0 / 0 -cf_version(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 330 0 / 1 0 / 0 -hyperloglog_version(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 152 0 / 1 0 / 0 -lsb_restore_global_data(lua_sandbox*,const char*) ../src/lsb_serialize.c 488 1 / 1 2 / 12 = 16% -add_table_ref(table_ref_array*,const void*,size_t) ../src/lsb_serialize.c 168 1 / 1 1 / 4 = 25% -pb_write_double(lsb_output_data*,double) ../src/lsb_serialize_protobuf.c 58 1 / 1 1 / 4 = 25% -pb_write_tag(lsb_output_data*,unsigned char,unsigned char) ../src/lsb_serialize_protobuf.c 105 1 / 1 1 / 4 = 25% -pb_write_string(lsb_output_data*,char,const char*,size_t) ../src/lsb_serialize_protobuf.c 164 1 / 1 3 / 8 = 37% -MurmurHash64A(const void*,int,unsigned int) ...Source/lua_hyperloglog/redis_hyperloglog.c 264 1 / 1 4 / 10 = 40% -lsb_appendf(lsb_output_data*,const char*,...) ../src/lsb_output.c 67 1 / 1 9 / 22 = 40% -cf_delete(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 262 1 / 1 3 / 7 = 42% -serialize_data(lua_sandbox*,int,lsb_output_data*) ../src/lsb_serialize.c 227 1 / 1 10 / 22 = 45% -serialize_bloom_filter(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 195 1 / 1 8 / 16 = 50% -serialize_cuckoo_filter(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 339 1 / 1 8 / 16 = 50% -serialize_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 160 1 / 1 8 / 16 = 50% -pb_write_bool(lsb_output_data*,int) ../src/lsb_serialize_protobuf.c 80 1 / 1 3 / 6 = 50% -output_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 183 1 / 1 3 / 6 = 50% -file_exists(const char*) ../src/lsb_serialize.c 477 1 / 1 1 / 2 = 50% -instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 123 1 / 1 1 / 2 = 50% -cf_fromstring(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 316 1 / 1 1 / 2 = 50% -lsb_serialize_table_as_pb(lua_sandbox*,int) ../src/lsb_serialize_protobuf.c 581 1 / 1 20 / 38 = 52% -expand_path(const char*,int,const char*,char*) ../src/luasandbox.c 165 1 / 1 10 / 18 = 55% -circular_buffer_set(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 326 1 / 1 14 / 25 = 56% -ignore_value_type(lua_sandbox*,serialization_data*,int,lua_CFunction*) ../src/lsb_serialize.c 109 1 / 1 8 / 14 = 57% -lsb_serialize_binary(const void*,size_t,lsb_output_data*) ../src/lsb_serialize.c 535 1 / 1 10 / 17 = 58% -update_field_length(lsb_output_data*,size_t) ../src/lsb_serialize_protobuf.c 128 1 / 1 6 / 10 = 60% -cf_add(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 195 1 / 1 3 / 5 = 60% -cf_query(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 228 1 / 1 3 / 5 = 60% -lsb_create_custom(void*,const char*,const char*) ../src/luasandbox.c 226 1 / 1 16 / 26 = 61% -pb_write_varint(lsb_output_data*,unsigned long long) ../src/lsb_serialize_protobuf.c 29 1 / 1 5 / 8 = 62% -lsb_pcall_setup(lua_sandbox*,const char*) ../src/luasandbox.c 493 1 / 1 5 / 8 = 62% -serialize_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 1017 1 / 1 19 / 30 = 63% -output_circular_buffer_cbufd(lua_State*,circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 889 1 / 1 14 / 22 = 63% -lsb_create(void*,const char*,const char*,unsigned,unsigned,unsigned) ../src/luasandbox.c 182 1 / 1 9 / 14 = 64% -serialize_circular_buffer_delta(lua_State*,circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 974 1 / 1 13 / 20 = 65% -encode_fields(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 519 1 / 1 24 / 36 = 66% -encode_field_array(lua_sandbox*,lsb_output_data*,int,const char*,int) ../src/lsb_serialize_protobuf.c 269 1 / 1 20 / 30 = 66% -hyperloglog_fromstring(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 132 1 / 1 4 / 6 = 66% -process_fields(lua_State*,unsigned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 104 1 / 1 51 / 74 = 68% -nlz(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 70 1 / 1 7 / 10 = 70% -output_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 933 1 / 1 17 / 24 = 70% -lsb_serialize_double(lsb_output_data*,double) ../src/lsb_serialize.c 561 1 / 1 27 / 38 = 71% -encode_field_value(lua_sandbox*,lsb_output_data*,int,const char*,int) ../src/lsb_serialize_protobuf.c 336 1 / 1 75 / 105 = 71% -lsb_init(lua_sandbox*,const char*) ../src/luasandbox.c 305 1 / 1 20 / 28 = 71% -circular_buffer_set_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 373 1 / 1 10 / 14 = 71% -lsb_preserve_global_data(lua_sandbox*,const char*) ../src/lsb_serialize.c 366 1 / 1 19 / 26 = 73% -serialize_kvp(lua_sandbox*,serialization_data*,size_t) ../src/lsb_serialize.c 287 1 / 1 18 / 24 = 75% -hllCount(hyperloglog*) ...Source/lua_hyperloglog/redis_hyperloglog.c 454 1 / 1 12 / 16 = 75% -read_string(lua_State*,int,unsigned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 61 1 / 1 6 / 8 = 75% -lsb_usage(lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 405 1 / 1 6 / 8 = 75% -circular_buffer_add_delta(lua_State*,circular_buffer*,double,int,double) .../lua_circular_buffer/lua_circular_buffer.c 219 1 / 1 6 / 8 = 75% -process_varint(lua_State*,const char*...ned const char*,unsigned const char*) ../src/lsb_deserialize_protobuf.c 82 1 / 1 3 / 4 = 75% -lsb_appendc(lsb_output_data*,char) ../src/lsb_output.c 55 1 / 1 3 / 4 = 75% -encode_int(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 225 1 / 1 3 / 4 = 75% -output(lua_State*) ../src/luasandbox.c 131 1 / 1 3 / 4 = 75% -lsb_set_error(lua_sandbox*,const char*) ../src/luasandbox.c 438 1 / 1 3 / 4 = 75% -lsb_get_output(lua_sandbox*,size_t*) ../src/luasandbox.c 481 1 / 1 3 / 4 = 75% -bucket_delete(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 136 1 / 1 3 / 4 = 75% -circular_buffer_mannwhitneyu(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 662 1 / 1 23 / 30 = 76% -compute_max(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 525 1 / 1 8 / 10 = 80% -lsb_realloc_output(lsb_output_data*,size_t) ../src/lsb_output.c 132 1 / 1 13 / 16 = 81% -bucket_insert(cuckoo_filter*,unsigned,unsigned,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 160 1 / 1 13 / 16 = 81% -read_double(char**,double*) .../lua_circular_buffer/lua_circular_buffer.c 781 1 / 1 18 / 22 = 81% -lsb_decode_protobuf(lua_State*) ../src/lsb_deserialize_protobuf.c 262 1 / 1 54 / 65 = 83% -lsb_output(lua_sandbox*,int,int,int) ../src/lsb_output.c 153 1 / 1 25 / 30 = 83% -lsb_output_protobuf(lua_sandbox*,int,int) ../src/lsb_output.c 215 1 / 1 5 / 6 = 83% -compute_sum(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 415 1 / 1 5 / 6 = 83% -compute_avg(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 438 1 / 1 5 / 6 = 83% -circular_buffer_compute(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 552 1 / 1 11 / 13 = 84% -output_circular_buffer_full(circular_buffer*,lsb_output_data*) .../lua_circular_buffer/lua_circular_buffer.c 867 1 / 1 12 / 14 = 85% -compute_variance(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 463 1 / 1 7 / 8 = 87% -circular_buffer_delta_fromstring(lua_State*,circular_buffer*,char**) .../lua_circular_buffer/lua_circular_buffer.c 804 1 / 1 7 / 8 = 87% -hllDenseSum(uint8_t*,double*,int*) ...Source/lua_hyperloglog/redis_hyperloglog.c 356 1 / 1 34 / 38 = 89% -read_length(unsigned const char*,unsigned const char*,size_t*) ../src/lsb_deserialize_protobuf.c 27 1 / 1 9 / 10 = 90% -read_varint(unsigned const char*,unsigned const char*,long long*) ../src/lsb_deserialize_protobuf.c 44 1 / 1 9 / 10 = 90% -compute_min(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 498 1 / 1 9 / 10 = 90% -memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 86 1 / 1 11 / 12 = 91% -circular_buffer_add(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 267 1 / 1 11 / 12 = 91% -lsb_destroy(lua_sandbox*,const char*) ../src/luasandbox.c 381 1 / 1 13 / 14 = 92% -read_key(unsigned const char*,int*,int*) ../src/lsb_deserialize_protobuf.c 18 1 / 1 0 / 0 -update_output_stats(lua_sandbox*) ../src/lsb_output.c 24 1 / 1 2 / 2 = 100% -lsb_add_output_function(lua_State*,lua_CFunction) ../src/lsb_output.c 35 1 / 1 0 / 0 -lsb_get_output_function(lua_State*,int) ../src/lsb_output.c 43 1 / 1 0 / 0 -lsb_appends(lsb_output_data*,const char*,size_t) ../src/lsb_output.c 119 1 / 1 4 / 4 = 100% -get_serialize_function(lua_State*,int) ../src/lsb_serialize.c 87 1 / 1 0 / 0 -find_table_ref(table_ref_array*,const void*) ../src/lsb_serialize.c 147 1 / 1 4 / 4 = 100% -get_preservation_version(lua_State*) ../src/lsb_serialize.c 193 1 / 1 4 / 4 = 100% -serialize_table(lua_sandbox*,serialization_data*,size_t) ../src/lsb_serialize.c 212 1 / 1 6 / 6 = 100% -lsb_add_serialize_function(lua_State*,lua_CFunction) ../src/lsb_serialize.c 527 1 / 1 0 / 0 -encode_string(lua_sandbox*,lsb_output_data*,char,const char*,int) ../src/lsb_serialize_protobuf.c 196 1 / 1 2 / 2 = 100% -encode_field_object(lua_sandbox*,lsb_output_data*) ../src/lsb_serialize_protobuf.c 315 1 / 1 4 / 4 = 100% -libsize(const luaL_Reg*) ../src/luasandbox.c 52 1 / 1 2 / 2 = 100% -preload_modules(lua_State*) ../src/luasandbox.c 59 1 / 1 2 / 2 = 100% -instruction_usage(lua_sandbox*) ../src/luasandbox.c 117 1 / 1 0 / 0 -unprotected_panic(lua_State*) ../src/luasandbox.c 148 1 / 1 0 / 0 -get_usage_config(lua_State*,int,const char*) ../src/luasandbox.c 156 1 / 1 0 / 0 -lsb_get_error(lua_sandbox*) ../src/luasandbox.c 429 1 / 1 2 / 2 = 100% -lsb_get_lua(lua_sandbox*) ../src/luasandbox.c 451 1 / 1 0 / 0 -lsb_get_state(lua_sandbox*) ../src/luasandbox.c 463 1 / 1 2 / 2 = 100% -lsb_add_function(lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 472 1 / 1 0 / 0 -lsb_pcall_teardown(lua_sandbox*) ../src/luasandbox.c 512 1 / 1 2 / 2 = 100% -lsb_terminate(lua_sandbox*,const char*) ../src/luasandbox.c 524 1 / 1 4 / 4 = 100% -bloom_filter_new(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 36 1 / 1 0 / 0 -check_bloom_filter(lua_State*,int) ...Source/lua_bloom_filter/lua_bloom_filter.c 66 1 / 1 0 / 0 -bloom_filter_add(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 76 1 / 1 9 / 9 = 100% -bloom_filter_query(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 115 1 / 1 9 / 9 = 100% -bloom_filter_count(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 147 1 / 1 0 / 0 -bloom_filter_clear(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 155 1 / 1 0 / 0 -bloom_filter_fromstring(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 164 1 / 1 4 / 4 = 100% -bloom_filter_version(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 186 1 / 1 0 / 0 -luaopen_bloom_filter(lua_State*) ...Source/lua_bloom_filter/lua_bloom_filter.c 242 1 / 1 0 / 0 -get_start_time(circular_buffer*) .../lua_circular_buffer/lua_circular_buffer.c 81 1 / 1 0 / 0 -copy_cleared_row(circular_buffer*,double*,size_t) .../lua_circular_buffer/lua_circular_buffer.c 87 1 / 1 4 / 4 = 100% -clear_rows(circular_buffer*,unsigned) .../lua_circular_buffer/lua_circular_buffer.c 106 1 / 1 10 / 10 = 100% -circular_buffer_new(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 130 1 / 1 4 / 4 = 100% -check_circular_buffer(lua_State*,int) .../lua_circular_buffer/lua_circular_buffer.c 177 1 / 1 0 / 0 -check_row(circular_buffer*,double,int) .../lua_circular_buffer/lua_circular_buffer.c 187 1 / 1 12 / 12 = 100% -check_column(lua_State*,circular_buffer*,int) .../lua_circular_buffer/lua_circular_buffer.c 209 1 / 1 0 / 0 -circular_buffer_get(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 298 1 / 1 2 / 2 = 100% -circular_buffer_get_configuration(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 315 1 / 1 0 / 0 -circular_buffer_get_header(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 402 1 / 1 0 / 0 -compute_sd(circular_buffer*,unsigned,unsigned,unsigned,unsigned*) .../lua_circular_buffer/lua_circular_buffer.c 490 1 / 1 0 / 0 -append_values(circular_buffer*,unsigned,unsigned,unsigned,double[]) .../lua_circular_buffer/lua_circular_buffer.c 602 1 / 1 4 / 4 = 100% -rank_data(double*[],size_t) .../lua_circular_buffer/lua_circular_buffer.c 618 1 / 1 16 / 16 = 100% -double_pp_compare(const void*,const void*) .../lua_circular_buffer/lua_circular_buffer.c 647 1 / 1 14 / 14 = 100% -circular_buffer_current_time(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 754 1 / 1 0 / 0 -circular_buffer_format(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 761 1 / 1 0 / 0 -read_time_row(char**,circular_buffer*) .../lua_circular_buffer/lua_circular_buffer.c 774 1 / 1 0 / 0 -circular_buffer_fromstring(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 830 1 / 1 12 / 12 = 100% -luaopen_circular_buffer(lua_State*) .../lua_circular_buffer/lua_circular_buffer.c 1102 1 / 1 0 / 0 -clp2(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 51 1 / 1 0 / 0 -cf_new(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 85 1 / 1 0 / 0 -check_cuckoo_filter(lua_State*,int) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 110 1 / 1 0 / 0 -fingerprint(unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 120 1 / 1 2 / 2 = 100% -bucket_lookup(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 127 1 / 1 4 / 4 = 100% -bucket_add(cuckoo_bucket*,unsigned) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 148 1 / 1 4 / 4 = 100% -cf_count(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 299 1 / 1 0 / 0 -cf_clear(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 307 1 / 1 0 / 0 -luaopen_cuckoo_filter(lua_State*) ...urce/lua_cuckoo_filter/lua_cuckoo_filter.c 387 1 / 1 0 / 0 -hyperloglog_new(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 29 1 / 1 0 / 0 -check_hyperloglog(lua_State*,int) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 49 1 / 1 0 / 0 -hyperloglog_add(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 59 1 / 1 5 / 5 = 100% -hyperloglog_count(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 90 1 / 1 2 / 2 = 100% -hyperloglog_clear(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 123 1 / 1 0 / 0 -luaopen_hyperloglog(lua_State*) ...e/Source/lua_hyperloglog/lua_hyperloglog.c 213 1 / 1 0 / 0 -hllPatLen(unsigned char*,size_t,long*) ...Source/lua_hyperloglog/redis_hyperloglog.c 323 1 / 1 2 / 2 = 100% -hllDenseAdd(uint8_t*,unsigned char*,size_t) ...Source/lua_hyperloglog/redis_hyperloglog.c 426 1 / 1 2 / 2 = 100% ------------------------------------------------------------------------------ --------------------------------------------------- ----- --------------------- -Total 97% 1028 / 1419 = 72% +Function Source Line FnCov C/D Coverage +----------------------------- ------------------ ----- --------------------- +add_table_ref...void*,size_t) ...rialize.c 170 1 / 1 1 / 4 = 25% +restore_globa...lua_sandbox*) ...rialize.c 495 1 / 1 7 / 18 = 38% +lsb_create(vo...*,lsb_logger) ...sandbox.c 379 1 / 1 11 / 26 = 42% +serialize_dat...tput_buffer*) ...rialize.c 229 1 / 1 8 / 18 = 44% +hsr_new(lua_State*) ..._reader.c 21 1 / 1 4 / 8 = 50% +lsb_create_me...,const char*) ...matcher.c 443 1 / 1 3 / 6 = 50% +lsb_outputc(l...buffer*,char) ..._buffer.c 78 1 / 1 2 / 4 = 50% +instruction_m...*,lua_Debug*) ...sandbox.c 115 1 / 1 1 / 2 = 50% +read_config(lua_State*) ...sandbox.c 139 1 / 1 1 / 2 = 50% +set_tz() ...sandbox.c 336 1 / 1 1 / 2 = 50% +heka_encode_m...sandbox*,int) ...message.c 871 1 / 1 14 / 24 = 58% +set_random_seed() ...sandbox.c 351 1 / 1 7 / 12 = 58% +lsb_read_file(const char*) ...il/util.c 47 1 / 1 6 / 10 = 60% +lsb_heka_crea...eka_im_input) ...sandbox.c 341 1 / 1 11 / 18 = 61% +lsb_heka_crea..._im_analysis) ...sandbox.c 529 1 / 1 11 / 18 = 61% +lsb_heka_crea...e_checkpoint) ...sandbox.c 612 1 / 1 11 / 18 = 61% +min_expand(Ma...,const char*) ...matcher.c 171 1 / 1 5 / 8 = 62% +preserve_glob...lua_sandbox*) ...rialize.c 366 1 / 1 27 / 42 = 64% +lsb_init_heka...message*,int) ...message.c 417 1 / 1 4 / 6 = 66% +process_field...,const char*) ...message.c 93 1 / 1 55 / 82 = 67% +encode_field_...st char*,int) ...message.c 330 1 / 1 19 / 28 = 67% +lsb_heka_pm_i... char*,_Bool) ...sandbox.c 497 1 / 1 11 / 16 = 68% +read_string(l...,const char*) ...message.c 52 1 / 1 7 / 10 = 70% +set_restricti...eka_sandbox*) ...sandbox.c 251 1 / 1 19 / 27 = 70% +check_int(lua...st char*,int) ...sandbox.c 192 1 / 1 5 / 7 = 71% +encode_field_...st char*,int) ...message.c 403 1 / 1 78 / 105 = 74% +process_messa...e*,int,_Bool) ...sandbox.c 412 1 / 1 29 / 39 = 74% +encode_fields...st char*,int) ...message.c 608 1 / 1 27 / 36 = 75% +serialize_kvp...data*,size_t) ...rialize.c 286 1 / 1 18 / 24 = 75% +decode_header...ize_t,size_t) ...message.c 19 1 / 1 12 / 16 = 75% +read_message(lua_State*) ...sandbox.c 35 1 / 1 6 / 8 = 75% +hsr_decode_me...e(lua_State*) ..._reader.c 77 1 / 1 6 / 8 = 75% +process_varin...,const char*) ...message.c 72 1 / 1 3 / 4 = 75% +encode_int(ls...st char*,int) ...message.c 287 1 / 1 3 / 4 = 75% +load_op_node(...,match_node*) ...matcher.c 321 1 / 1 3 / 4 = 75% +lsb_destroy_m...tch_builder*) ...matcher.c 480 1 / 1 3 / 4 = 75% +output(lua_State*) ...sandbox.c 123 1 / 1 3 / 4 = 75% +lsb_stop_sand...lua_sandbox*) ...sandbox.c 561 1 / 1 3 / 4 = 75% +lsb_get_outpu...box*,size_t*) ..._output.c 110 1 / 1 3 / 4 = 75% +lsb_clear_hek...eka_message*) ...message.c 431 1 / 1 3 / 4 = 75% +lsb_outputs(l...char*,size_t) ..._buffer.c 144 1 / 1 3 / 4 = 75% +lsb_outputfd(...ffer*,double) ..._buffer.c 173 1 / 1 31 / 40 = 77% +lsb_init(lsb_...,const char*) ...sandbox.c 461 1 / 1 28 / 36 = 77% +lsb_pcall_set...,const char*) ...sandbox.c 676 1 / 1 14 / 18 = 77% +hsr_find_message(lua_State*) ..._reader.c 106 1 / 1 25 / 32 = 78% +read_string(i...onst_string*) ...message.c 39 1 / 1 8 / 10 = 80% +read_string_v..._read_value*) ...message.c 57 1 / 1 8 / 10 = 80% +read_integer_..._read_value*) ...message.c 78 1 / 1 8 / 10 = 80% +check_string(...,const char*) ...sandbox.c 170 1 / 1 4 / 5 = 80% +lsb_heka_time...time_t,_Bool) ...sandbox.c 725 1 / 1 13 / 16 = 81% +matchbalance(...,const char*) ...matcher.c 137 1 / 1 13 / 16 = 81% +lsb_serialize...void*,size_t) ...rialize.c 547 1 / 1 9 / 11 = 81% +lsb_outputf(l...st char*,...) ..._buffer.c 90 1 / 1 23 / 28 = 82% +lsb_create_me...,const char*) ...matcher.c 493 1 / 1 24 / 29 = 82% +heka_decode_m...e(lua_State*) ...message.c 683 1 / 1 54 / 65 = 83% +heka_read_mes...eka_message*) ...message.c 912 1 / 1 45 / 54 = 83% +set_missing_h...eka_sandbox*) ...message.c 32 1 / 1 10 / 12 = 83% +lsb_heka_pm_a...ssage*,_Bool) ...sandbox.c 593 1 / 1 10 / 12 = 83% +ignore_value_...a_CFunction*) ...rialize.c 114 1 / 1 10 / 12 = 83% +inject_messag...s(lua_State*) ...sandbox.c 130 1 / 1 5 / 6 = 83% +lsb_destroy(lsb_lua_sandbox*) ...sandbox.c 574 1 / 1 5 / 6 = 83% +lsb_pb_output...ed long long) ...rotobuf.c 56 1 / 1 5 / 6 = 83% +lsb_output(ls...,int,int,int) ..._output.c 42 1 / 1 27 / 32 = 84% +heka_encode_m...e(lua_State*) ...message.c 810 1 / 1 11 / 13 = 84% +match_class(int,int) ...matcher.c 68 1 / 1 11 / 13 = 84% +match(MatchSt...,const char*) ...matcher.c 183 1 / 1 39 / 46 = 84% +lsb_pb_read_v...*,long long*) ...rotobuf.c 36 1 / 1 17 / 20 = 85% +inject_messag...t(lua_State*) ...sandbox.c 51 1 / 1 18 / 21 = 85% +lsb_heka_pm_o...,void*,_Bool) ...sandbox.c 699 1 / 1 12 / 14 = 85% +lsb_expand_in...ffer*,size_t) ..._buffer.c 49 1 / 1 12 / 14 = 85% +numeric_test(...node*,double) ...matcher.c 213 1 / 1 6 / 7 = 85% +load_sandbox_...*,lsb_logger) ...sandbox.c 223 1 / 1 19 / 22 = 86% +matchbracketc...,const char*) ...matcher.c 99 1 / 1 19 / 22 = 86% +eval_tree(mat...eka_message*) ...matcher.c 297 1 / 1 14 / 16 = 87% +update_checkpoint(lua_State*) ...sandbox.c 218 1 / 1 7 / 8 = 87% +lsb_init_inpu...ffer*,size_t) ..._buffer.c 20 1 / 1 7 / 8 = 87% +lsb_outputd(l...ffer*,double) ..._buffer.c 156 1 / 1 7 / 8 = 87% +process_field...,const char*) ...message.c 121 1 / 1 50 / 57 = 87% +copy_table(lu...*,lsb_logger) ...sandbox.c 263 1 / 1 22 / 25 = 88% +load_expressi...,match_node*) ...matcher.c 335 1 / 1 42 / 47 = 89% +lsb_decode_he...t,lsb_logger) ...message.c 201 1 / 1 51 / 57 = 89% +inject_payload(lua_State*) ...sandbox.c 160 1 / 1 18 / 20 = 90% +lsb_add_funct...,const char*) ...sandbox.c 664 1 / 1 9 / 10 = 90% +lsb_init_outp...ffer*,size_t) ..._buffer.c 27 1 / 1 9 / 10 = 90% +lsb_find_heka...*,lsb_logger) ...message.c 319 1 / 1 29 / 32 = 90% +eval_node(mat...eka_message*) ...matcher.c 235 1 / 1 31 / 34 = 91% +memory_manage...ize_t,size_t) ...sandbox.c 78 1 / 1 11 / 12 = 91% +lsb_read_heka..._read_value*) ...message.c 462 1 / 1 24 / 26 = 92% +classend(const char*) ...matcher.c 43 1 / 1 16 / 17 = 94% +lsb_expand_ou...ffer*,size_t) ..._buffer.c 52 1 / 1 17 / 18 = 94% +string_test(m...onst_string*) ...matcher.c 167 1 / 1 18 / 19 = 94% +encode_string...st char*,int) ...message.c 258 1 / 1 2 / 2 = 100% +encode_field_...tput_buffer*) ...message.c 381 1 / 1 4 / 4 = 100% +lsb_destroy_m...age_matcher*) ...matcher.c 559 1 / 1 6 / 6 = 100% +lsb_eval_mess...eka_message*) ...matcher.c 578 1 / 1 0 / 0 +lsb_heka_stop...eka_sandbox*) ...sandbox.c 675 1 / 1 0 / 0 +lsb_heka_term...,const char*) ...sandbox.c 681 1 / 1 0 / 0 +lsb_heka_dest...eka_sandbox*) ...sandbox.c 687 1 / 1 2 / 2 = 100% +lsb_heka_get_...eka_sandbox*) ...sandbox.c 766 1 / 1 2 / 2 = 100% +lsb_heka_get_...eka_sandbox*) ...sandbox.c 772 1 / 1 2 / 2 = 100% +lsb_heka_get_...eka_sandbox*) ...sandbox.c 778 1 / 1 2 / 2 = 100% +lsb_heka_is_r...eka_sandbox*) ...sandbox.c 799 1 / 1 4 / 4 = 100% +check_hsr(lua_State*,int) ..._reader.c 68 1 / 1 0 / 0 +hsr_read_message(lua_State*) ..._reader.c 196 1 / 1 6 / 6 = 100% +hsr_gc(lua_State*) ..._reader.c 208 1 / 1 0 / 0 +luaopen_heka_...r(lua_State*) ..._reader.c 230 1 / 1 0 / 0 +libsize(const luaL_Reg*) ...sandbox.c 44 1 / 1 2 / 2 = 100% +preload_modules(lua_State*) ...sandbox.c 51 1 / 1 2 / 2 = 100% +instruction_u...lua_sandbox*) ...sandbox.c 109 1 / 1 0 / 0 +unprotected_panic(lua_State*) ...sandbox.c 153 1 / 1 0 / 0 +get_usage_con...,const char*) ...sandbox.c 161 1 / 1 0 / 0 +stop_hook(lua...*,lua_Debug*) ...sandbox.c 553 1 / 1 0 / 0 +lsb_usage(lsb...b_usage_stat) ...sandbox.c 597 1 / 1 10 / 10 = 100% +lsb_get_error...lua_sandbox*) ...sandbox.c 621 1 / 1 2 / 2 = 100% +lsb_set_error...,const char*) ...sandbox.c 627 1 / 1 4 / 4 = 100% +lsb_get_lua(lsb_lua_sandbox*) ...sandbox.c 640 1 / 1 2 / 2 = 100% +lsb_get_lua_f...lua_sandbox*) ...sandbox.c 646 1 / 1 2 / 2 = 100% +lsb_get_paren...lua_sandbox*) ...sandbox.c 652 1 / 1 2 / 2 = 100% +lsb_get_state...lua_sandbox*) ...sandbox.c 658 1 / 1 2 / 2 = 100% +lsb_pcall_tea...lua_sandbox*) ...sandbox.c 700 1 / 1 4 / 4 = 100% +lsb_terminate...,const char*) ...sandbox.c 714 1 / 1 6 / 6 = 100% +lsb_add_outpu...ua_CFunction) ..._output.c 22 1 / 1 0 / 0 +lsb_get_outpu...a_State*,int) ..._output.c 30 1 / 1 0 / 0 +get_serialize...a_State*,int) ...rialize.c 92 1 / 1 0 / 0 +find_table_re...,const void*) ...rialize.c 149 1 / 1 4 / 4 = 100% +get_preservat...n(lua_State*) ...rialize.c 195 1 / 1 4 / 4 = 100% +serialize_tab...data*,size_t) ...rialize.c 214 1 / 1 6 / 6 = 100% +file_exists(const char*) ...rialize.c 484 1 / 1 2 / 2 = 100% +lsb_add_seria...ua_CFunction) ...rialize.c 539 1 / 1 0 / 0 +lsb_serialize...ffer*,double) ...rialize.c 574 1 / 1 6 / 6 = 100% +read_double_v..._read_value*) ...message.c 97 1 / 1 2 / 2 = 100% +process_varin...*,long long*) ...message.c 110 1 / 1 4 / 4 = 100% +lsb_free_heka...eka_message*) ...message.c 451 1 / 1 2 / 2 = 100% +lsb_write_hek...char*,size_t) ...message.c 507 1 / 1 22 / 22 = 100% +lsb_free_inpu...nput_buffer*) ..._buffer.c 36 1 / 1 2 / 2 = 100% +lsb_free_outp...tput_buffer*) ..._buffer.c 42 1 / 1 2 / 2 = 100% +lsb_pb_read_k...r*,int*,int*) ...rotobuf.c 15 1 / 1 8 / 8 = 100% +lsb_pb_write_...nsigned char) ...rotobuf.c 25 1 / 1 2 / 2 = 100% +lsb_pb_write_...ed long long) ...rotobuf.c 75 1 / 1 2 / 2 = 100% +lsb_pb_write_..._buffer*,int) ...rotobuf.c 85 1 / 1 4 / 4 = 100% +lsb_pb_write_...ffer*,double) ...rotobuf.c 99 1 / 1 2 / 2 = 100% +lsb_pb_write_...char*,size_t) ...rotobuf.c 114 1 / 1 6 / 6 = 100% +lsb_pb_update...ffer*,size_t) ...rotobuf.c 127 1 / 1 8 / 8 = 100% +lsb_init_runn...nning_stats*) ...g_stats.c 13 1 / 1 0 / 0 +lsb_update_ru...tats*,double) ...g_stats.c 21 1 / 1 4 / 4 = 100% +lsb_sd_runnin...nning_stats*) ...g_stats.c 37 1 / 1 2 / 2 = 100% +lsb_init_cons...onst_string*) .../string.c 11 1 / 1 0 / 0 +singlematch(i...,const char*) ...matcher.c 119 1 / 1 4 / 4 = 100% +max_expand(Ma...,const char*) ...matcher.c 156 1 / 1 10 / 10 = 100% +lsb_string_ma...,const char*) ...matcher.c 261 1 / 1 10 / 10 = 100% +lsb_lp2(unsigned long long) ...il/util.c 33 1 / 1 2 / 2 = 100% +lsb_get_time() ...il/util.c 73 1 / 1 0 / 0 +----------------------------- ------------------ ----- --------------------- +Total 100% 1567 / 1958 = 80% diff --git a/include/luasandbox.h b/include/luasandbox.h index eacf926..94cfe80 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -10,6 +10,7 @@ #define luasandbox_h_ #include "luasandbox/lua.h" +#include "luasandbox/error.h" #ifdef _WIN32 #ifdef luasandbox_EXPORTS @@ -25,14 +26,9 @@ #endif #endif -#if defined(_MSC_VER) -#define PRIuSIZE "Iu" -#else -#define PRIuSIZE "zu" -#endif - #define LSB_ERROR_SIZE 256 +#define LSB_SHUTTING_DOWN "shutting down" #define LSB_CONFIG_TABLE "lsb_config" #define LSB_MEMORY_LIMIT "memory_limit" #define LSB_INSTRUCTION_LIMIT "instruction_limit" @@ -62,15 +58,15 @@ typedef enum { LSB_UT_MAX } lsb_usage_type; -typedef void (*lsb_logger)(const char* component, - int level, - const char* fmt, - ...); - typedef struct lsb_lua_sandbox lsb_lua_sandbox; +LSB_EXPORT extern lsb_err_id LSB_ERR_INIT; +LSB_EXPORT extern lsb_err_id LSB_ERR_LUA; +LSB_EXPORT extern lsb_err_id LSB_ERR_TERMINATED; + #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /** @@ -118,9 +114,11 @@ lsb_create(void *parent, const char *lua_file, const char *cfg, * global data schema. If no version is set the check will * always succeed and a version of zero is assigned. * - * @return int Zero on success, non-zero on failure. + * @return lsb_err_value NULL on success error message on failure + * */ -LSB_EXPORT int lsb_init(lsb_lua_sandbox *lsb, const char *state_file); +LSB_EXPORT lsb_err_value +lsb_init(lsb_lua_sandbox *lsb, const char *state_file); /** * Aborts the running sandbox from a different thread of execution. A "shutting @@ -228,9 +226,10 @@ LSB_EXPORT void lsb_add_function(lsb_lua_sandbox *lsb, * @param lsb Pointer to the sandbox. * @param func_name Name of the function to load * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_EXPORT int lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name); +LSB_EXPORT lsb_err_value +lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name); /** * Helper function to update the statistics after the call diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 3e364c2..c90205e 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -12,7 +12,7 @@ #include #include -#include "../../luasandbox.h" +#include "../error.h" #include "../lauxlib.h" #include "../lua.h" #include "../util/heka_message.h" @@ -33,6 +33,8 @@ #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" +LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; + enum lsb_heka_pm_rv { LSB_HEKA_PM_SENT = 0, LSB_HEKA_PM_FAIL = -1, @@ -78,11 +80,11 @@ extern "C" * * @return 0 on success */ -typedef int (*lsb_heka_inject_message_input)(void *parent, - const char *pb, - size_t pb_len, - double cp_numeric, - const char *cp_string); +typedef int (*lsb_heka_im_input)(void *parent, + const char *pb, + size_t pb_len, + double cp_numeric, + const char *cp_string); /** * inject_message callback function provided by the host. @@ -93,9 +95,9 @@ typedef int (*lsb_heka_inject_message_input)(void *parent, * * @return 0 on success */ -typedef int (*lsb_heka_inject_message_analysis)(void *parent, - const char *pb, - size_t pb_len); +typedef int (*lsb_heka_im_analysis)(void *parent, + const char *pb, + size_t pb_len); /** * update_checkpoint callback function provided by the host. @@ -127,14 +129,13 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, const char *state_file, const char *lsb_cfg, lsb_logger logger, - lsb_heka_inject_message_input im); + lsb_heka_im_input im); /** * Host access to the input sandbox process_message API. If a numeric * checkpoint is set the string checkpoint is ignored. * * @param hsb Heka input sandbox - * @param msg Heka message to process * @param cp_numeric NAN if no numeric checkpoint * @param cp_string NULL if no string checkpoint * @param profile Take a timing sample on this execution @@ -147,10 +148,9 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, */ LSB_HEKA_EXPORT int lsb_heka_pm_input(lsb_heka_sandbox *hsb, - lsb_heka_message *msg, - double cp_numeric, - const char *cp_string, - bool profile); + double cp_numeric, + const char *cp_string, + bool profile); /** * Create a sandbox supporting the Heka Analysis Plugin API @@ -171,7 +171,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, const char *state_file, const char *lsb_cfg, lsb_logger logger, - lsb_heka_inject_message_analysis im); + lsb_heka_im_analysis im); /** * Host access to the analysis sandbox process_message API @@ -188,8 +188,8 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, */ LSB_HEKA_EXPORT int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, - lsb_heka_message *msg, - bool profile); + lsb_heka_message *msg, + bool profile); /** * Create a sandbox supporting the Heka Output Plugin API @@ -234,9 +234,9 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, */ LSB_HEKA_EXPORT int lsb_heka_pm_output(lsb_heka_sandbox *hsb, - lsb_heka_message *msg, - void *sequence_id, - bool profile); + lsb_heka_message *msg, + void *sequence_id, + bool profile); /** * Aborts the running sandbox from a different thread of execution. A "shutting @@ -286,7 +286,6 @@ lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); LSB_HEKA_EXPORT int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown); - /** * Return the last error in human readable form. * diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index e2c51c3..6c77435 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -110,7 +110,8 @@ typedef struct { } lsb_read_value; #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /** @@ -119,10 +120,11 @@ extern "C" { * @param m Heka message structure * @param num_fields Preallocated number of fields (must be >0) * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure * */ -LSB_UTIL_EXPORT int lsb_init_heka_message(lsb_heka_message *m, int num_fields); +LSB_UTIL_EXPORT lsb_err_value +lsb_init_heka_message(lsb_heka_message *m, int num_fields); /** * Frees the memory allocated for the message fields @@ -199,9 +201,9 @@ LSB_UTIL_EXPORT bool lsb_read_heka_field(lsb_heka_message *m, * @param len Length of UUID (16 or 36 anything else will cause a new UUID to be * created). * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int +LSB_UTIL_EXPORT lsb_err_value lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len); #ifdef __cplusplus diff --git a/include/luasandbox/util/input_buffer.h b/include/luasandbox/util/input_buffer.h index 4fc3c14..48d83ef 100644 --- a/include/luasandbox/util/input_buffer.h +++ b/include/luasandbox/util/input_buffer.h @@ -36,10 +36,10 @@ extern "C" { * before erroring (the internal buffer will contain extra space * for the header) * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_init_input_buffer(lsb_input_buffer *b, - size_t max_message_size); +LSB_UTIL_EXPORT lsb_err_value +lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size); /** * Frees the memory internally allocated by the buffer and resets the state @@ -56,9 +56,10 @@ LSB_UTIL_EXPORT void lsb_free_input_buffer(lsb_input_buffer *b); * @param b Input buffer * @param len The length of the data being added to the buffer * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len); +LSB_UTIL_EXPORT lsb_err_value +lsb_expand_input_buffer(lsb_input_buffer *b, size_t len); #ifdef __cplusplus } diff --git a/include/luasandbox/util/output_buffer.h b/include/luasandbox/util/output_buffer.h index 5d1d4a2..2adac33 100644 --- a/include/luasandbox/util/output_buffer.h +++ b/include/luasandbox/util/output_buffer.h @@ -16,15 +16,15 @@ #define LSB_OUTPUT_SIZE 1024 typedef struct lsb_output_buffer { - char *buf; - size_t maxsize; - size_t size; - size_t pos; - int err; + char *buf; + size_t maxsize; + size_t size; + size_t pos; } lsb_output_buffer; #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /** @@ -34,10 +34,10 @@ extern "C" { * @param max_message_size The maximum message size the buffer will handle * before erroring * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT -int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); +LSB_UTIL_EXPORT lsb_err_value +lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); /** * Frees the memory internally allocated by the buffer and resets the state @@ -46,23 +46,16 @@ int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); */ LSB_UTIL_EXPORT void lsb_free_output_buffer(lsb_output_buffer *b); -/** - * Resets the position and error state of the buffer - * - * @param b Output buffer - */ -LSB_UTIL_EXPORT void lsb_clear_output_buffer(lsb_output_buffer *b); - /** * Resize the output buffer when more space is needed. * * @param b Output buffer to resize. * @param needed Number of additional bytes needed. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_expand_output_buffer(lsb_output_buffer *b, - size_t needed); +LSB_UTIL_EXPORT lsb_err_value lsb_expand_output_buffer(lsb_output_buffer *b, + size_t needed); /** * Append a character to the output buffer. @@ -70,9 +63,9 @@ LSB_UTIL_EXPORT int lsb_expand_output_buffer(lsb_output_buffer *b, * @param b Pointer the b buffer. * @param ch Character to append to the b. * - * @return int 0 on success, false if out of memory. + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_outputc(lsb_output_buffer *b, char ch); +LSB_UTIL_EXPORT lsb_err_value lsb_outputc(lsb_output_buffer *b, char ch); /** * Append a formatted string to the output buffer. @@ -80,9 +73,10 @@ LSB_UTIL_EXPORT int lsb_outputc(lsb_output_buffer *b, char ch); * @param b Pointer the b buffer. * @param fmt Printf format specifier. * - * @return int 0 on success, false if out of memory. + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); +LSB_UTIL_EXPORT lsb_err_value +lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); /** * Append a fixed string to the output buffer. @@ -91,11 +85,10 @@ LSB_UTIL_EXPORT int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); * @param str String to append to the b. * @param len Length of the string to append * - * @return int 0 on success, false if out of memory. + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_outputs(lsb_output_buffer *b, - const char *str, - size_t len); +LSB_UTIL_EXPORT lsb_err_value +lsb_outputs(lsb_output_buffer *b, const char *str, size_t len); /** * More efficient output of a double to a string. NaN/Inf check and then calls @@ -104,9 +97,9 @@ LSB_UTIL_EXPORT int lsb_outputs(lsb_output_buffer *b, * @param output Pointer the output buffer. * @param d Double value to convert to a string. * - * @return int 0 on success, false if out of memory. + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_outputd(lsb_output_buffer *output, double d); +LSB_UTIL_EXPORT lsb_err_value lsb_outputd(lsb_output_buffer *output, double d); /** @@ -115,9 +108,9 @@ LSB_UTIL_EXPORT int lsb_outputd(lsb_output_buffer *output, double d); * @param output Pointer the output buffer. * @param d Double value to convert to a string. * - * @return int 0 on success, false if out of memory. + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_outputfd(lsb_output_buffer *ob, double d); +LSB_UTIL_EXPORT lsb_err_value lsb_outputfd(lsb_output_buffer *ob, double d); #ifdef __cplusplus } diff --git a/include/luasandbox/util/protobuf.h b/include/luasandbox/util/protobuf.h index cf90ca9..a03ba06 100644 --- a/include/luasandbox/util/protobuf.h +++ b/include/luasandbox/util/protobuf.h @@ -26,7 +26,6 @@ typedef enum { LSB_PB_WT_FIXED32 = 5 } lsb_pb_wire_types; - #ifdef __cplusplus extern "C" { #endif @@ -50,9 +49,9 @@ const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype); * @param tag Field identifier. * @param wiretype Field wire type. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int +LSB_UTIL_EXPORT lsb_err_value lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, unsigned char wiretype); @@ -85,9 +84,9 @@ LSB_UTIL_EXPORT int lsb_pb_output_varint(char *buf, unsigned long long i); * @param ob Pointer to the output data buffer. * @param i Number to be encoded. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int +LSB_UTIL_EXPORT lsb_err_value lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i); /** @@ -96,9 +95,9 @@ lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i); * @param ob Pointer to the output data buffer. * @param i Number to be encoded. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_pb_write_bool(lsb_output_buffer *ob, int i); +LSB_UTIL_EXPORT lsb_err_value lsb_pb_write_bool(lsb_output_buffer *ob, int i); /** * Writes a double to the output buffer. @@ -106,9 +105,10 @@ LSB_UTIL_EXPORT int lsb_pb_write_bool(lsb_output_buffer *ob, int i); * @param ob Pointer to the output data buffer. * @param i Double to be encoded. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int lsb_pb_write_double(lsb_output_buffer *ob, double i); +LSB_UTIL_EXPORT lsb_err_value +lsb_pb_write_double(lsb_output_buffer *ob, double i); /** * Writes a string to the output buffer. @@ -118,9 +118,9 @@ LSB_UTIL_EXPORT int lsb_pb_write_double(lsb_output_buffer *ob, double i); * @param s String to output. * @param len Length of s. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int +LSB_UTIL_EXPORT lsb_err_value lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len); /** @@ -131,9 +131,9 @@ lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len); * @param len_pos Position in the output buffer where the length should be * written. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT int +LSB_UTIL_EXPORT lsb_err_value lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos); #ifdef __cplusplus diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 5d8b263..d040570 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -11,6 +11,8 @@ #include +#include "../error.h" + #ifdef _WIN32 #ifdef luasandboxutil_EXPORTS #define LSB_UTIL_EXPORT __declspec(dllexport) @@ -25,6 +27,11 @@ #endif #endif +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_NULL; +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_OOM; +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_FULL; +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_PRANGE; + #ifdef __cplusplus extern "C" { #endif diff --git a/include/luasandbox_serialize.h b/include/luasandbox_serialize.h index e402335..40dfda4 100644 --- a/include/luasandbox_serialize.h +++ b/include/luasandbox_serialize.h @@ -18,24 +18,6 @@ extern "C" { #endif -/** - * Serialize all user global data to disk. - * - * @param lsb Pointer to the sandbox. - * - * @return int Zero on success, non-zero on failure. - */ -int lsb_preserve_global_data(lsb_lua_sandbox *lsb); - -/** - * Restores previously serialized data from disk. - * - * @param lsb Pointer to the sandbox. - * - * @return int Zero on success, non-zero on failure. - */ -int lsb_restore_global_data(lsb_lua_sandbox *lsb); - /** * Add a serialization function to the environment table. The environment table * must be on the top of the stack. This function will receive the userdata, @@ -49,7 +31,7 @@ int lsb_restore_global_data(lsb_lua_sandbox *lsb); * @param lua Pointer the Lua state. * @param fp Function pointer to the serializer. * - * @return int Zero on success, non-zero on failure. + * @return lsb_err_value NULL on success error message on failure */ LSB_EXPORT void lsb_add_serialize_function(lua_State *lua, lua_CFunction fp); @@ -61,9 +43,9 @@ lsb_add_serialize_function(lua_State *lua, lua_CFunction fp); * @param src Pointer to the binary data. * @param len Length in bytes of the data to output. * - * @return int Zero on success, non-zero on failure. + * @return lsb_err_value NULL on success error message on failure */ -LSB_EXPORT int +LSB_EXPORT lsb_err_value lsb_serialize_binary(lsb_output_buffer *output, const void *src, size_t len); /** @@ -72,9 +54,10 @@ lsb_serialize_binary(lsb_output_buffer *output, const void *src, size_t len); * @param output Pointer the output buffer. * @param d Double value to convert to a string. * - * @return int Zero on success, non-zero on failure. + * @return lsb_err_value NULL on success error message on failure */ -LSB_EXPORT int lsb_serialize_double(lsb_output_buffer *output, double d); +LSB_EXPORT lsb_err_value +lsb_serialize_double(lsb_output_buffer *output, double d); #ifdef __cplusplus } diff --git a/src/heka/message.c b/src/heka/message.c index 9fabd41..7f769f5 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -252,21 +252,21 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) * @param name Key used for the Lua table entry lookup. * @param index Lua stack index of the table. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value encode_string(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, const char *name, int index) { - int result = 0; + lsb_err_value ret = NULL; lua_getfield(lsb->lua, index, name); if (lua_isstring(lsb->lua, -1)) { size_t len; const char *s = lua_tolstring(lsb->lua, -1, &len); - result = lsb_pb_write_string(ob, tag, s, len); + ret = lsb_pb_write_string(ob, tag, s, len); } lua_pop(lsb->lua, 1); - return result; + return ret; } @@ -281,22 +281,21 @@ encode_string(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, * @param name Key used for the Lua table entry lookup. * @param index Lua stack index of the table. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value encode_int(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, const char *name, int index) { - int result = 0; + lsb_err_value ret = NULL; lua_getfield(lsb->lua, index, name); if (lua_isnumber(lsb->lua, -1)) { unsigned long long i = (unsigned long long)lua_tonumber(lsb->lua, -1); - if ((result = lsb_pb_write_key(ob, tag, LSB_PB_WT_VARINT))) { - result = lsb_pb_write_varint(ob, i); - } + ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, i); } lua_pop(lsb->lua, 1); - return result; + return ret; } @@ -310,9 +309,9 @@ encode_int(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, * @param representation String representation of the field i.e., "ms" * @param value_type Protobuf value type * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, const char *representation, int value_type); @@ -325,24 +324,24 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, * @param t Lua type of the array values. * @param representation String representation of the field i.e., "ms" * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, const char *representation, int value_type) { - int result = 0; + lsb_err_value ret; int first = (int)lua_objlen(lsb->lua, -1); int multiple = first > 1 ? first : 0; size_t len_pos = 0; lua_checkstack(lsb->lua, 2); lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { + while (!ret && lua_next(lsb->lua, -2) != 0) { if (lua_type(lsb->lua, -1) != t) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); - return 3; + return LSB_ERR_HEKA_INPUT; } - result = encode_field_value(lsb, ob, first, representation, value_type); + ret = encode_field_value(lsb, ob, first, representation, value_type); if (first) { len_pos = ob->pos; first = 0; @@ -350,7 +349,7 @@ encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for // the next interation. } - if (multiple && value_type == LSB_PB_INTEGER) { + if (!ret && multiple && value_type == LSB_PB_INTEGER) { // fix up the varint packed length size_t i = len_pos - 2; int y = 0; @@ -359,14 +358,14 @@ encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, --i; ++y; } - result = lsb_pb_update_field_length(ob, i); - if (y == LSB_MAX_VARINT_BYTES || result) { + if (y == LSB_MAX_VARINT_BYTES) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "unable set the length of the packed integer array"); - return 1; + return LSB_ERR_LUA; } + ret = lsb_pb_update_field_length(ob, i); } - return result; + return ret; } @@ -376,9 +375,10 @@ encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, * @param lsb Pointer to the sandbox. * @param ob Pointer to the output data buffer. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -static int encode_field_object(lsb_lua_sandbox *lsb, lsb_output_buffer *ob) +static lsb_err_value +encode_field_object(lsb_lua_sandbox *lsb, lsb_output_buffer *ob) { const char *representation = NULL; lua_getfield(lsb->lua, -1, "representation"); @@ -393,17 +393,17 @@ static int encode_field_object(lsb_lua_sandbox *lsb, lsb_output_buffer *ob) } lua_getfield(lsb->lua, -3, "value"); - int result = encode_field_value(lsb, ob, 1, representation, value_type); + lsb_err_value ret = encode_field_value(lsb, ob, 1, representation, value_type); lua_pop(lsb->lua, 3); // remove representation, value_type and value - return result; + return ret; } -static int +static lsb_err_value encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, const char *representation, int value_type) { - int result = 0; + lsb_err_value ret = NULL; size_t len; const char *s; @@ -419,27 +419,28 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "invalid string value_type: %d", value_type); - return 3; + return LSB_ERR_HEKA_INPUT; } if (first) { // this uglyness keeps the protobuf fields in order without // additional lookups if (value_type == LSB_PB_BYTES) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); - if (result) return result; - result = lsb_pb_write_varint(ob, value_type); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, value_type); + if (ret) return ret; } if (representation) { - result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, - strlen(representation)); - if (result) return result; + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; } } s = lua_tolstring(lsb->lua, -1, &len); if (value_type == LSB_PB_BYTES) { - result = lsb_pb_write_string(ob, LSB_PB_VALUE_BYTES, s, len); + ret = lsb_pb_write_string(ob, LSB_PB_VALUE_BYTES, s, len); + if (ret) return ret; } else { - result = lsb_pb_write_string(ob, LSB_PB_VALUE_STRING, s, len); + ret = lsb_pb_write_string(ob, LSB_PB_VALUE_STRING, s, len); + if (ret) return ret; } break; case LUA_TNUMBER: @@ -452,76 +453,69 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "invalid numeric value_type: %d", value_type); - return 3; + return LSB_ERR_HEKA_INPUT; } if (first) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); - if (result) return result; - - result = lsb_pb_write_varint(ob, value_type); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, value_type); + if (ret) return ret; if (representation) { - result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, - strlen(representation)); - if (result) return result; + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; } if (1 == first) { if (value_type == LSB_PB_INTEGER) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_VARINT); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_VARINT); } else { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_FIXED64); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_FIXED64); } + if (ret) return ret; } else { // pack array if (value_type == LSB_PB_INTEGER) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_LENGTH); - if (result) return result; - result = lsb_pb_write_varint(ob, 0); // length tbd later - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, 0); // length tbd later } else { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_LENGTH); - if (result) return result; - result = lsb_pb_write_varint(ob, first * sizeof(double)); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, first * sizeof(double)); } + if (ret) return ret; } } if (value_type == LSB_PB_INTEGER) { - result = lsb_pb_write_varint(ob, lua_tointeger(lsb->lua, -1)); + ret = lsb_pb_write_varint(ob, lua_tointeger(lsb->lua, -1)); } else { - result = lsb_pb_write_double(ob, lua_tonumber(lsb->lua, -1)); + ret = lsb_pb_write_double(ob, lua_tonumber(lsb->lua, -1)); } + if (ret) return ret; break; case LUA_TBOOLEAN: if (value_type != -1 && value_type != LSB_PB_BOOL) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "invalid boolean value_type: %d", value_type); - return 3; + return LSB_ERR_HEKA_INPUT; } if (first) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); - if (result) return result; - result = lsb_pb_write_varint(ob, LSB_PB_BOOL); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, LSB_PB_BOOL); + if (ret) return ret; + if (representation) { - result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, - strlen(representation)); - if (result) return result; + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; } if (1 == first) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_VARINT); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_VARINT); } else { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_LENGTH); - if (result) return result; - result = lsb_pb_write_varint(ob, first); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, first); } + if (ret) return ret; } - result = lsb_pb_write_bool(ob, lua_toboolean(lsb->lua, -1)); + ret = lsb_pb_write_bool(ob, lua_toboolean(lsb->lua, -1)); break; case LUA_TTABLE: @@ -531,18 +525,17 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, lua_pop(lsb->lua, 1); // remove the array test value switch (t) { case LUA_TNIL: - result = encode_field_object(lsb, ob); + ret = encode_field_object(lsb, ob); break; case LUA_TNUMBER: case LUA_TSTRING: case LUA_TBOOLEAN: - result = encode_field_array(lsb, ob, t, representation, value_type); + ret = encode_field_array(lsb, ob, t, representation, value_type); break; default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported array type: %s", lua_typename(lsb->lua, t)); - result = 3; - break; + return LSB_ERR_LUA; } } break; @@ -554,40 +547,48 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, if (!fp) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "user data object does not implement lsb_output"); - return 3; + return LSB_ERR_LUA; } if (first) { - result = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (ret) return ret; + // encode userdata as a byte array - result = lsb_pb_write_varint(ob, LSB_PB_BYTES); - if (result) return result; + ret = lsb_pb_write_varint(ob, LSB_PB_BYTES); + if (ret) return ret; + if (representation) { - result = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, - strlen(representation)); - if (result) return result; + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; } } - result = lsb_pb_write_key(ob, LSB_PB_VALUE_BYTES, LSB_PB_WT_LENGTH); - if (result) return result; + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_BYTES, LSB_PB_WT_LENGTH); + if (ret) return ret; + len_pos = ob->pos; - result = lsb_pb_write_varint(ob, 0); // length tbd later - if (result) return result; + ret = lsb_pb_write_varint(ob, 0); // length tbd later + if (ret) return ret; + lua_pushlightuserdata(lsb->lua, ob); - if (fp(lsb->lua)) return 3; + int result = fp(lsb->lua); lua_pop(lsb->lua, 1); // remove output function - result = lsb_pb_update_field_length(ob, len_pos); + if (result) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "user data output callback failed: %d", result); + return LSB_ERR_LUA; + } + ret = lsb_pb_update_field_length(ob, len_pos); } break; default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", lua_typename(lsb->lua, t)); - result = 3; - break; + return LSB_ERR_LUA; } - return result; + return ret; } @@ -601,16 +602,16 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, * @param name Key used for the Lua table entry lookup. * @param index Lua stack index of the table. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, const char *name, int index) { - int result = 0; + lsb_err_value ret = NULL; lua_getfield(lsb->lua, index, name); if (!lua_istable(lsb->lua, -1)) { - return result; + return ret; } lua_rawgeti(lsb->lua, -1, 1); // test for the array notation @@ -618,60 +619,64 @@ encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, if (lua_istable(lsb->lua, -1)) { int i = 1; do { - result = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); - if (result) return result; + ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (ret) return ret; + len_pos = ob->pos; - result = lsb_pb_write_varint(ob, 0); // length tbd later - if (result) return result; + ret = lsb_pb_write_varint(ob, 0); // length tbd later + if (ret) return ret; + lua_getfield(lsb->lua, -1, "name"); if (lua_isstring(lsb->lua, -1)) { const char *s = lua_tolstring(lsb->lua, -1, &len); - result = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); - if (result) return result; + ret = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, "field name must be a string"); - return 3; + ret = LSB_ERR_HEKA_INPUT; } lua_pop(lsb->lua, 1); // remove the name + if (ret) return ret; - result = encode_field_object(lsb, ob); - if (result) return result; - result = lsb_pb_update_field_length(ob, len_pos); - if (result) return result; + ret = encode_field_object(lsb, ob); + if (!ret) ret = lsb_pb_update_field_length(ob, len_pos); + if (ret) return ret; lua_pop(lsb->lua, 1); // remove the current field object lua_rawgeti(lsb->lua, -1, ++i); // grab the next field object - } while (!lua_isnil(lsb->lua, -1)); + } while (!ret && !lua_isnil(lsb->lua, -1)); } else { lua_pop(lsb->lua, 1); // remove the array test value lua_checkstack(lsb->lua, 2); lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - result = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); - if (result) return result; + while (lua_next(lsb->lua, -2) != 0) { + ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (ret) return ret; + len_pos = ob->pos; - result = lsb_pb_write_varint(ob, 0); // length tbd later - if (result) return result; + ret = lsb_pb_write_varint(ob, 0); // length tbd later + if (ret) return ret; + if (lua_isstring(lsb->lua, -2)) { const char *s = lua_tolstring(lsb->lua, -2, &len); - result = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); - if (result) return result; + ret = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, "field name must be a string"); - return 3; + ret = LSB_ERR_HEKA_INPUT; } - result = encode_field_value(lsb, ob, 1, NULL, -1); - if (result) return result; - result = lsb_pb_update_field_length(ob, len_pos); - if (result) return result; + if (ret) return ret; + + ret = encode_field_value(lsb, ob, 1, NULL, -1); + if (!ret) ret = lsb_pb_update_field_length(ob, len_pos); + if (ret) return ret; + lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for // the next interation. } } lua_pop(lsb->lua, 1); // remove the fields table - return result; + return ret; } @@ -788,7 +793,7 @@ int heka_decode_message(lua_State *lua) wiretype, (const char *)lp - pbstr); } - if (!(has_uuid && has_timestamp)) { + if (!has_uuid || !has_timestamp) { return luaL_error(lua, "missing required field uuid: %s timestamp: %s", has_uuid ? "found" : "not found", has_timestamp ? "found" : "not found"); @@ -827,10 +832,11 @@ int heka_encode_message(lua_State *lua) lsb_heka_sandbox *hsb = lsb_get_parent(lsb); set_missing_headers(lua, 1, hsb); - lsb_clear_output_buffer(&lsb->output); - if (heka_encode_message_table(lsb, 1)) { + lsb->output.pos = 0; + lsb_err_value ret = heka_encode_message_table(lsb, 1); + if (ret) { const char *err = lsb_get_error(lsb); - if (strlen(err) == 0) err = "exceeded output_limit"; + if (strlen(err) == 0) err = ret; return luaL_error(lua, "encode_message() failed: %s", err); } @@ -856,25 +862,25 @@ int heka_encode_message(lua_State *lua) if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; } return 1; } -int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) +lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) { - int result = 0; + lsb_err_value ret = NULL; lsb_output_buffer *ob = &lsb->output; - lsb_clear_output_buffer(ob); + ob->pos = 0; // use existing or create a type 4 uuid lua_getfield(lsb->lua, idx, LSB_UUID); size_t len; const char *uuid = lua_tolstring(lsb->lua, -1, &len); - result = lsb_write_heka_uuid(ob, uuid, len); - if (result) return result; + ret = lsb_write_heka_uuid(ob, uuid, len); lua_pop(lsb->lua, 1); // remove uuid + if (ret) return ret; // use existing or create a timestamp lua_getfield(lsb->lua, idx, LSB_TIMESTAMP); @@ -886,31 +892,20 @@ int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) } lua_pop(lsb->lua, 1); // remove timestamp - result = lsb_pb_write_key(ob, LSB_PB_TIMESTAMP, LSB_PB_WT_VARINT); - if (result) return result; - result = lsb_pb_write_varint(ob, ts); - if (result) return result; - result = encode_string(lsb, ob, LSB_PB_TYPE, LSB_TYPE, idx); - if (result) return result; - result = encode_string(lsb, ob, LSB_PB_LOGGER, LSB_LOGGER, idx); - if (result) return result; - result = encode_int(lsb, ob, LSB_PB_SEVERITY, LSB_SEVERITY, idx); - if (result) return result; - result = encode_string(lsb, ob, LSB_PB_PAYLOAD, LSB_PAYLOAD, idx); - if (result) return result; - result = encode_string(lsb, ob, LSB_PB_ENV_VERSION, LSB_ENV_VERSION, idx); - if (result) return result; - result = encode_int(lsb, ob, LSB_PB_PID, LSB_PID, idx); - if (result) return result; - result = encode_string(lsb, ob, LSB_PB_HOSTNAME, LSB_HOSTNAME, idx); - if (result) return result; - result = encode_fields(lsb, ob, LSB_PB_FIELDS, LSB_FIELDS, idx); - if (result) return result; - result = lsb_expand_output_buffer(ob, 1); - if (result) return result; - ob->buf[ob->pos] = 0; // NULL terminate incase someone tries to treat this - // as a string - return result; + ret = lsb_pb_write_key(ob, LSB_PB_TIMESTAMP, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, ts); + if (!ret) ret = encode_string(lsb, ob, LSB_PB_TYPE, LSB_TYPE, idx); + if (!ret) ret = encode_string(lsb, ob, LSB_PB_LOGGER, LSB_LOGGER, idx); + if (!ret) ret = encode_int(lsb, ob, LSB_PB_SEVERITY, LSB_SEVERITY, idx); + if (!ret) ret = encode_string(lsb, ob, LSB_PB_PAYLOAD, LSB_PAYLOAD, idx); + if (!ret) ret = encode_string(lsb, ob, LSB_PB_ENV_VERSION, LSB_ENV_VERSION, + idx); + if (!ret) ret = encode_int(lsb, ob, LSB_PB_PID, LSB_PID, idx); + if (!ret) ret = encode_string(lsb, ob, LSB_PB_HOSTNAME, LSB_HOSTNAME, idx); + if (!ret) ret = encode_fields(lsb, ob, LSB_PB_FIELDS, LSB_FIELDS, idx); + if (!ret) ret = lsb_expand_output_buffer(ob, 1); + ob->buf[ob->pos] = 0; // prevent possible overrun if treated as a string + return ret; } diff --git a/src/heka/message_impl.h b/src/heka/message_impl.h index 156792e..ee4b286 100644 --- a/src/heka/message_impl.h +++ b/src/heka/message_impl.h @@ -45,9 +45,9 @@ int heka_encode_message(lua_State *lua); * @param lsb Pointer to the sandbox. * @param idx Lua stack index of the message table. * - * @return int 0 on success + * @return lsb_err_value NULL on success error message on failure */ -int heka_encode_message_table(lsb_lua_sandbox *lsb, int idx); +lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, int idx); /** diff --git a/src/heka/message_matcher.c b/src/heka/message_matcher.c index 77f9e57..1c0512a 100644 --- a/src/heka/message_matcher.c +++ b/src/heka/message_matcher.c @@ -479,6 +479,8 @@ lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath) void lsb_destroy_message_match_builder(lsb_message_match_builder *mmb) { + if (!mmb) return; + if (mmb->parser) { lua_close(mmb->parser); mmb->parser = NULL; @@ -491,6 +493,9 @@ lsb_message_matcher* lsb_create_message_matcher(const lsb_message_match_builder *mmb, const char *exp) { + if (!mmb || !exp) { + return NULL; + } lua_getglobal(mmb->parser, "parse"); if (!lua_isfunction(mmb->parser, -1)) { return NULL; diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index a49afee..c2629c0 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -23,11 +23,12 @@ #ifdef _WIN32 #include -#define snprintf _snprintf #else #include #endif +lsb_err_id LSB_ERR_HEKA_INPUT = "invalid input"; + static const char *pm_func_name = "process_message"; static const char *im_func_name = "inject_message"; @@ -342,7 +343,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, const char *state_file, const char *lsb_cfg, lsb_logger logger, - lsb_heka_inject_message_input im) + lsb_heka_im_input im) { if (!lua_file) { if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); @@ -419,8 +420,11 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, } if (lua_pcall(lua, nargs, 2, 0) != 0) { char err[LSB_ERROR_SIZE]; - size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", pm_func_name, - lua_tostring(lua, -1)); + const char *em = lua_tostring(lua, -1); + if (hsb->type == 'i' && strcmp(em, LSB_SHUTTING_DOWN) == 0) { + return 0; + } + size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", pm_func_name, em); if (len >= LSB_ERROR_SIZE) { err[LSB_ERROR_SIZE - 1] = 0; } @@ -447,7 +451,6 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, int status = (int)lua_tointeger(lua, 1); switch (lua_type(lua, 2)) { - case LUA_TNONE: case LUA_TNIL: lsb_set_error(hsb->lsb, NULL); break; @@ -492,17 +495,20 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, int lsb_heka_pm_input(lsb_heka_sandbox *hsb, - lsb_heka_message *msg, - double cp_numeric, - const char *cp_string, - bool profile) + double cp_numeric, + const char *cp_string, + bool profile) { if (!hsb || hsb->type != 'i') return 1; - if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { - char err[LSB_ERROR_SIZE]; - snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", pm_func_name); - lsb_terminate(hsb->lsb, err); + lsb_err_value ret = lsb_pcall_setup(hsb->lsb, pm_func_name); + if (ret) { + if (ret != LSB_ERR_TERMINATED) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", + pm_func_name); + lsb_terminate(hsb->lsb, err); + } return 1; } @@ -516,7 +522,7 @@ int lsb_heka_pm_input(lsb_heka_sandbox *hsb, } else { lua_pushnil(lua); } - return process_message(hsb, msg, lua, 1, profile); + return process_message(hsb, NULL, lua, 1, profile); } @@ -525,7 +531,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, const char *state_file, const char *lsb_cfg, lsb_logger logger, - lsb_heka_inject_message_analysis im) + lsb_heka_im_analysis im) { if (!lua_file) { if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); @@ -585,10 +591,10 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, - lsb_heka_message *msg, - bool profile) + lsb_heka_message *msg, + bool profile) { - if (!hsb || hsb->type != 'a') return 1; + if (!hsb || !msg || hsb->type != 'a') return 1; if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { char err[LSB_ERROR_SIZE]; @@ -681,8 +687,8 @@ void lsb_heka_terminate_sandbox(lsb_heka_sandbox *hsb, const char *err) char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) { if (!hsb) return NULL; - char *msg = lsb_destroy(hsb->lsb); + char *msg = lsb_destroy(hsb->lsb); free(hsb->hostname); free(hsb->name); free(hsb); @@ -691,11 +697,11 @@ char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) int lsb_heka_pm_output(lsb_heka_sandbox *hsb, - lsb_heka_message *msg, - void *sequence_id, - bool profile) + lsb_heka_message *msg, + void *sequence_id, + bool profile) { - if (!hsb || hsb->type != 'o') return 1; + if (!hsb || !msg || hsb->type != 'o') return 1; if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { char err[LSB_ERROR_SIZE]; @@ -759,21 +765,19 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) const char* lsb_heka_get_error(lsb_heka_sandbox *hsb) { - if (!hsb) return ""; - return lsb_get_error(hsb->lsb); + return hsb ? lsb_get_error(hsb->lsb) : ""; } const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb) { - if (!hsb) return NULL; - return lsb_get_lua_file(hsb->lsb); + return hsb ? lsb_get_lua_file(hsb->lsb) : NULL; } lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb) { - if (!hsb) return (struct lsb_heka_stats){0,0,0,0,0,0,0,0,0,0,0,0}; + if (!hsb) return (struct lsb_heka_stats){ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; return (struct lsb_heka_stats){ .mem_cur = lsb_usage(hsb->lsb, LSB_UT_MEMORY, LSB_US_CURRENT), diff --git a/src/heka/sandbox_impl.h b/src/heka/sandbox_impl.h index f26e6aa..048940d 100644 --- a/src/heka/sandbox_impl.h +++ b/src/heka/sandbox_impl.h @@ -33,8 +33,8 @@ struct lsb_heka_sandbox { char *name; char *hostname; union { - lsb_heka_inject_message_input iim; - lsb_heka_inject_message_analysis aim; + lsb_heka_im_input iim; + lsb_heka_im_analysis aim; lsb_heka_update_checkpoint ucp; } cb; char type; diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c index eb70e83..4d32c34 100644 --- a/src/heka/stream_reader.c +++ b/src/heka/stream_reader.c @@ -15,19 +15,19 @@ #include "message_impl.h" #include "luasandbox/lauxlib.h" -const char* mozsvc_heka_stream_reader = "mozsvc.heka_stream_reader"; +const char *mozsvc_heka_stream_reader = "mozsvc.heka_stream_reader"; const char *mozsvc_heka_stream_reader_table = "heka_stream_reader"; -static int hsr_new(lua_State* lua) +static int hsr_new(lua_State *lua) { int n = lua_gettop(lua); luaL_argcheck(lua, n == 1, 0, "incorrect number of arguments"); size_t len; - const char* name = luaL_checklstring(lua, 1, &len); + const char *name = luaL_checklstring(lua, 1, &len); luaL_argcheck(lua, len < 255, 1, "name is too long"); size_t nbytes = sizeof(heka_stream_reader); - heka_stream_reader* hsr = lua_newuserdata(lua, nbytes); + heka_stream_reader *hsr = lua_newuserdata(lua, nbytes); size_t mms = 0; lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); @@ -65,33 +65,33 @@ static int hsr_new(lua_State* lua) } -static heka_stream_reader* check_hsr(lua_State* lua, int args) +static heka_stream_reader* check_hsr(lua_State *lua, int args) { - heka_stream_reader* hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); + heka_stream_reader *hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); luaL_argcheck(lua, args == lua_gettop(lua), 0, "incorrect number of arguments"); return hsr; } -static int hsr_decode_message(lua_State* lua) +static int hsr_decode_message(lua_State *lua) { - heka_stream_reader* hsr = check_hsr(lua, 2); - lsb_input_buffer* b = &hsr->buf; + heka_stream_reader *hsr = check_hsr(lua, 2); + lsb_input_buffer *b = &hsr->buf; b->readpos = b->scanpos = b->msglen = 0; size_t len; if (lua_type(lua, 2) == LUA_TSTRING) { - const char* s = lua_tolstring(lua, 2, &len); - if (len > 0) { - if (lsb_expand_input_buffer(b, len)) { - return luaL_error(lua, "buffer reallocation failed\tname:%s", - hsr->name); - } - memcpy(b->buf, s, len); - } else { - return luaL_error(lua, "empty protobuf string"); + const char *s = lua_tolstring(lua, 2, &len); + if (len > 0) { + if (lsb_expand_input_buffer(b, len)) { + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); } + memcpy(b->buf, s, len); + } else { + return luaL_error(lua, "empty protobuf string"); + } } else { return luaL_error(lua, "buffer must be string"); } @@ -103,22 +103,26 @@ static int hsr_decode_message(lua_State* lua) } -static int hsr_find_message(lua_State* lua) +static int hsr_find_message(lua_State *lua) { int n = lua_gettop(lua); - luaL_argcheck(lua, n > 1 && n < 4, 0,"incorrect number of arguments"); + luaL_argcheck(lua, n > 1 && n < 4, 0, "incorrect number of arguments"); - heka_stream_reader* hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); - lsb_input_buffer* b = &hsr->buf; - FILE* fh = NULL; + heka_stream_reader *hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); + lsb_input_buffer *b = &hsr->buf; + if (hsr->msg.raw.s) { + lsb_clear_heka_message(&hsr->msg); + b->msglen = b->readpos = b->scanpos = 0; + } + FILE *fh = NULL; switch (lua_type(lua, 2)) { case LUA_TNIL: // scan the existing buffer break; case LUA_TSTRING: // add data to the buffer { size_t len; - const char* s = lua_tolstring(lua, 2, &len); + const char *s = lua_tolstring(lua, 2, &len); if (len > 0) { if (lsb_expand_input_buffer(b, len)) { return luaL_error(lua, "buffer reallocation failed\tname:%s", @@ -130,7 +134,7 @@ static int hsr_find_message(lua_State* lua) } break; case LUA_TUSERDATA: // add data from the provided file handle to the buffer - fh = *(FILE**)luaL_checkudata(lua, 2, "FILE*"); + fh = *(FILE **)luaL_checkudata(lua, 2, "FILE*"); if (!fh) luaL_error(lua, "attempt to use a closed file"); break; default: @@ -138,7 +142,8 @@ static int hsr_find_message(lua_State* lua) } bool decode = true; - if (n == 3 && lua_type(lua, 3) == LUA_TBOOLEAN) { + if (n == 3) { + luaL_checktype(lua, 3, LUA_TBOOLEAN); decode = (bool)lua_toboolean(lua, 3); } @@ -147,6 +152,7 @@ static int hsr_find_message(lua_State* lua) size_t discarded = 0; bool found = lsb_find_heka_message(&hsr->msg, b, decode, &discarded, NULL); + size_t need = b->size; if (found) { lua_pushboolean(lua, 1); // found lua_pushinteger(lua, b->scanpos - pos_s); // consumed @@ -157,13 +163,19 @@ static int hsr_find_message(lua_State* lua) } else { lua_pushinteger(lua, b->scanpos - pos_s); } + if (b->msglen) { + need = b->msglen + (size_t)b->buf[b->scanpos + 1] + LSB_HDR_FRAME_SIZE - + b->readpos; + } else { + need = b->scanpos + b->size - b->readpos; + } } if (fh) { // update bytes read if (found) { lua_pushinteger(lua, 0); } else { - if (lsb_expand_input_buffer(&hsr->buf, 0)) { + if (lsb_expand_input_buffer(&hsr->buf, need)) { luaL_error(lua, "%s buffer reallocation failed", hsr->name); } size_t nread = fread(hsr->buf.buf + hsr->buf.readpos, @@ -174,39 +186,28 @@ static int hsr_find_message(lua_State* lua) lua_pushnumber(lua, (lua_Number)nread); } } else { // update bytes needed - if (found) { - if (b->scanpos != b->readpos) { - lua_pushinteger(lua, 0); - } else { - lua_pushinteger(lua, b->size); - } - } else { - if (b->msglen + LSB_MAX_HDR_SIZE > b->size) { - lua_pushinteger(lua, b->msglen); - } else { - lua_pushinteger(lua, b->scanpos + b->size - b->readpos); - } - } + if (found && b->scanpos != b->readpos) need = 0; + lua_pushinteger(lua, need); } return 3; } -static int hsr_read_message(lua_State* lua) +static int hsr_read_message(lua_State *lua) { int n = lua_gettop(lua); if (n < 1 || n > 4) { return luaL_error(lua, "read_message() incorrect number of arguments"); } - heka_stream_reader* hsr = check_hsr(lua, n); + heka_stream_reader *hsr = check_hsr(lua, n); lua_remove(lua, 1); // remove the hsr user data return heka_read_message(lua, &hsr->msg); } -static int hsr_gc(lua_State* lua) +static int hsr_gc(lua_State *lua) { - heka_stream_reader* hsr = check_hsr(lua, 1); + heka_stream_reader *hsr = check_hsr(lua, 1); free(hsr->name); lsb_free_heka_message(&hsr->msg); lsb_free_input_buffer(&hsr->buf); @@ -216,22 +217,17 @@ static int hsr_gc(lua_State* lua) static const struct luaL_reg heka_stream_readerlib_f[] = { - { "new", hsr_new } - , { NULL, NULL } + { "new", hsr_new }, { NULL, NULL } }; static const struct luaL_reg heka_stream_readerlib_m[] = { - { "find_message", hsr_find_message } - , { "decode_message", hsr_decode_message } - , { "read_message", hsr_read_message } - , { "__gc", hsr_gc } - , { NULL, NULL } + { "find_message", hsr_find_message }, { "decode_message", hsr_decode_message }, { "read_message", hsr_read_message }, { "__gc", hsr_gc }, { NULL, NULL } }; -int luaopen_heka_stream_reader(lua_State* lua) +int luaopen_heka_stream_reader(lua_State *lua) { luaL_newmetatable(lua, mozsvc_heka_stream_reader); lua_pushvalue(lua, -1); diff --git a/src/heka/test/lua/aim.lua b/src/heka/test/lua/aim.lua index eac1d27..353edd7 100644 --- a/src/heka/test/lua/aim.lua +++ b/src/heka/test/lua/aim.lua @@ -38,3 +38,19 @@ ok, err = pcall(inject_payload, "txt", "name", add_to_payload) if ok then error("cannot output functions") end eerr = "bad argument #3 to '?' (unsupported type)" assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_payload, true, "name", "data") +assert(not ok) +assert("inject_payload() payload_type argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_payload, "txt", true, "data") +assert(not ok) +assert("inject_payload() payload_name argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_message, {Fields = {foo = {value = {"s", true}}}}) +assert(not ok) +assert("inject_message() failed: array has mixed types" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_message, {}) +assert(not ok) +assert("inject_message() failed: rejected by the callback" == err, string.format("received: %s", err)) diff --git a/src/heka/test/lua/analysis.lua b/src/heka/test/lua/analysis.lua index 1211f27..8726898 100644 --- a/src/heka/test/lua/analysis.lua +++ b/src/heka/test/lua/analysis.lua @@ -17,6 +17,9 @@ function timer_event(ns, shutdown) return end + local uuid = read_message("Uuid") + assert(not uuid) + if ns == 2e9 then error("boom") end if shutdown then error("should not have a shutdown signal") end diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua index 0cc6f7e..bdc3211 100644 --- a/src/heka/test/lua/encode_message.lua +++ b/src/heka/test/lua/encode_message.lua @@ -4,6 +4,9 @@ require "string" require "table" +require "circular_buffer" + +local cb = circular_buffer.new(2,1,1) local msgs = { { @@ -19,6 +22,57 @@ local msgs = { rv = "\030\002\008\028\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl\074\002\sh", framed = true, }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h", + Fields = { + number = 1, + numbers = {value = {1,2,3}, representation = "count"}, + string = "string", + strings = {"s1","s2","s3"}, + bool = true, + bools = {true,false,false}}}, + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0\82\10\10\4\98\111\111\108\16\4\64\1\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51" + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h", + Fields = { + {name = "number" ,value = 1}, + {name = "numbers" ,value = {1,2,3}, representation="count"}, + {name = "string" ,value = "string"}, + {name = "strings" ,value = {"s1","s2","s3"}}, + {name = "bool" ,value = true}, + {name = "bools" ,value = {true,false,false}}} + }, + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51\82\10\10\4\98\111\111\108\16\4\64\1\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0" + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = "value", value_type = 0, representation = "widget"}}}, -- string + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\20\10\3\107\101\121\26\6\119\105\100\103\101\116\34\5\118\97\108\117\101", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = "value", value_type = 1, representation = "widget"}}}, -- bytes + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\22\10\3\107\101\121\16\1\26\6\119\105\100\103\101\116\42\5\118\97\108\117\101", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = 34, value_type = 2, representation = "widget"}}}, -- int + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\17\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\48\34", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = 34, value_type = 3, representation = "widget"}}}, -- double + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\24\10\3\107\101\121\16\3\26\6\119\105\100\103\101\116\57\0\0\0\0\0\0\65\64", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = true, value_type = 4, representation = "widget"}}}, -- bool + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\17\10\3\107\101\121\16\4\26\6\119\105\100\103\101\116\64\1", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = {1,2,3}, value_type = 2, representation = "widget"}}}, -- int + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\20\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\50\3\1\2\3", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = cb}}, -- userdata + rv = '\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\141\1\10\3\107\101\121\16\1\42\131\1\123\34\116\105\109\101\34\58\48\44\34\114\111\119\115\34\58\50\44\34\99\111\108\117\109\110\115\34\58\49\44\34\115\101\99\111\110\100\115\95\112\101\114\95\114\111\119\34\58\49\44\34\99\111\108\117\109\110\95\105\110\102\111\34\58\91\123\34\110\97\109\101\34\58\34\67\111\108\117\109\110\95\49\34\44\34\117\110\105\116\34\58\34\99\111\117\110\116\34\44\34\97\103\103\114\101\103\97\116\105\111\110\34\58\34\115\117\109\34\125\93\125\10\110\97\110\10\110\97\110\10' + }, } for i, v in ipairs(msgs) do @@ -30,9 +84,9 @@ for i, v in ipairs(msgs) do end if v.rv ~= rv then - local et = {string.byte(v.rv, 1, -1)} + --local et = {string.byte(v.rv, 1, -1)} local rt = {string.byte(rv, 1, -1)} - assert(v.rv == rv, string.format("test: %d\nexpected: %s\nreceived: %s", i, table.concat(et, " "), table.concat(rt, " "))) + assert(v.rv == rv, string.format("test: %d\received: %s", i, table.concat(rt, " "))) end end @@ -41,3 +95,15 @@ local ok, err = pcall(encode_message, "foo") if ok then error("encode_message should not accept a string") end local eerr = "bad argument #1 to '?' (table expected, got string)" assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(encode_message, "", true, "extra") +assert(not ok) +assert("bad argument #3 to '?' (incorrect number of arguments)" == err, string.format("received: %s", err)) + +ok, err = pcall(encode_message, {Fields = {foo = {value = {"s", true}}}}) +assert(not ok) +assert("encode_message() failed: array has mixed types" == err, string.format("received: %s", err)) + +ok, err = pcall(encode_message, {Fields = { {noname = "foo", value = 1}} }) +assert(not ok) +assert("encode_message() failed: field name must be a string" == err, string.format("received: %s", err)) diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index 836a652..3496926 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -43,14 +43,73 @@ for i, v in ipairs(err_msgs) do end -- Stream Reader tests +require "io" require "heka_stream_reader" local hsr = heka_stream_reader.new("test") ok, err = pcall(inject_message, hsr) local eerr = "inject_message() attempted to inject a nil message" assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) +ok, err = pcall(hsr.decode_message, hsr, true) +assert(not ok) +assert("buffer must be string" == err, string.format("received: %s", err)) +ok, err = pcall(hsr.decode_message, hsr, "") +assert(not ok) +assert("empty protobuf string" == err, string.format("received: %s", err)) hsr:decode_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\004") inject_message(hsr) +local ts = hsr:read_message("Timestamp") +assert(4 == ts, string.format("received: %g", ts)) + +local framed = "\030\002\008\020\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\006" +local fh = assert(io.open("hekamsg.pb", "w+")) +fh:write(framed) +fh:seek("set") +local found, consumed, read = hsr:find_message(fh) +assert(not found) +assert(0 == consumed, string.format("expected: 0 received %d", consumed)) +assert(25 == read, string.format("expected: 25 received: %d", read)) +found, consumed, read = hsr:find_message(fh) +assert(found) +assert(25 == consumed, string.format("expected: 25 received %d", consumed)) +assert(0 == read, string.format("expected: >0 received: %d", read)) +fh:close() + +local found, consumed, need = hsr:find_message(framed) +assert(true == found) +assert(25 == consumed, string.format("expected: 25 received %d", consumed)) +assert(0 < need, string.format("expected: >0 received: %d", need)) + +local found, consumed, need = hsr:find_message("") +assert(not found) + +ok, err = pcall(hsr.find_message, hsr, assert) +assert(not ok, "accepted function as second arg") + +ok, err = pcall(hsr.find_message, hsr, nil, "str") +assert(not ok, "accepted string as third arg") + +ok, err = pcall(hsr.find_message, hsr, hsr) +assert(not ok, "accepted non FILE userdata") + +fh = assert(io.open("lua/iim.lua")) +found, consumed, read = hsr:find_message(fh) +fh:close() +assert(false == found) +assert(0 == consumed, string.format("expected: 0 received %d", consumed)) +assert(0 < need, string.format("expected: >0 received: %d", need)) + +ok, err = pcall(hsr.read_message) +assert(not ok) +assert("read_message() incorrect number of arguments" == err, string.format("received: %s", err)) + +ok, err = pcall(hsr.read_message, hsr, 1, 2, 3, 4) +assert(not ok) +assert("read_message() incorrect number of arguments" == err, string.format("received: %s", err)) + +ok, err = pcall(hsr.read_message, 1, 2) +assert(not ok) +assert("bad argument #1 to '?' (mozsvc.heka_stream_reader expected, got number)" == err, string.format("received: %s", err)) -- String tests inject_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\005") diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index e027b35..56bcfac 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -1,8 +1,8 @@ -- This Source Code Form is subject to the terms of the Mozilla Public -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local msg = {Timestamp = 8} +require "string" +msg = {Timestamp = 8} function process_message(cp) if cp == 0 then @@ -11,6 +11,14 @@ function process_message(cp) return -1, "failed" elseif cp == 2 then return 0, "ok" + elseif cp == 3 then + error("boom") + elseif cp == 4 then + return 0, msg + elseif cp == 5 then + return msg + elseif cp == 6 then + error(string.rep("a", 255)) elseif cp == "string" then return 0, "string" end diff --git a/src/heka/test/lua/output.lua b/src/heka/test/lua/output.lua index 9e884f1..18b1110 100644 --- a/src/heka/test/lua/output.lua +++ b/src/heka/test/lua/output.lua @@ -7,13 +7,22 @@ local sid function process_message(sequence_id) if not sequence_id then update_checkpoint() + local ok, err = pcall(update_checkpoint, 1, 2, 3) + assert(not ok) return 0 else if not sid then update_checkpoint(sequence_id, 7) + update_checkpoint(sequence_id) + local ok, err = pcall(update_checkpoint, true) + assert(not ok) + ok, err = pcall(update_checkpoint, sequence_id, "foo") + assert(not ok) sid = sequence_id return -5 else + local ok, err = pcall(update_checkpoint, sequence_id) + assert(not ok) return -3 -- retry end end diff --git a/src/heka/test/test_message_matcher.c b/src/heka/test/test_message_matcher.c index 46d2daf..187a7e9 100644 --- a/src/heka/test/test_message_matcher.c +++ b/src/heka/test/test_message_matcher.c @@ -29,6 +29,20 @@ static char* test_stub() } +static char* test_api_assertion() +{ + mu_assert(NULL == lsb_create_message_matcher(NULL, "TRUE"), "not null"); + lsb_destroy_message_match_builder(NULL); + lsb_destroy_message_matcher(NULL); + + lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); + mu_assert(mmb, "failed to create mmb"); + mu_assert(NULL == lsb_create_message_matcher(mmb, NULL), "not null"); + lsb_destroy_message_match_builder(mmb); + return NULL; +} + + static char* test_true_matcher() { char* tests[] = { @@ -277,6 +291,7 @@ static char* benchmark_match() static char* all_tests() { mu_run_test(test_stub); + mu_run_test(test_api_assertion); mu_run_test(test_true_matcher); mu_run_test(test_false_matcher); mu_run_test(test_malformed_matcher); diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index a8f7078..979efc0 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -48,7 +48,7 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, struct im_result results[] = { { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 67, .cp_numeric = 99, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 156, .cp_numeric = NAN, .cp_string = "foo.log:123" }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, @@ -175,7 +175,7 @@ static int ucp(void *parent, void *sequence_id) { static int cnt = 0; if (parent) return 1; - void *results[] = { NULL, (void *)99 }; + void *results[] = { NULL, (void *)99, (void *)99 }; if (cnt >= (int)(sizeof results / sizeof results[0])) { fprintf(stderr, "tests and results are mis-matched\n"); @@ -191,12 +191,50 @@ static int ucp(void *parent, void *sequence_id) } +static char* test_api_assertion() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + + lsb_heka_sandbox *isb, *asb, *osb; + isb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); + asb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, aim); + osb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); + + mu_assert(isb, "lsb_heka_create_input failed"); + mu_assert_rv(1, lsb_heka_pm_input(NULL, 0, NULL, false)); + mu_assert_rv(1, lsb_heka_timer_event(isb, 0, false)); + + mu_assert(isb, "lsb_heka_create_analysis failed"); + mu_assert_rv(1, lsb_heka_pm_analysis(NULL, NULL, false)); + mu_assert_rv(1, lsb_heka_pm_analysis(asb, NULL, false)); + mu_assert_rv(1, lsb_heka_timer_event(NULL, 0, false)); + mu_assert_rv(1, lsb_heka_pm_analysis(isb, &m, false)); + + mu_assert(isb, "lsb_heka_create_output failed"); + mu_assert_rv(1, lsb_heka_pm_output(NULL, NULL, NULL, false)); + mu_assert_rv(1, lsb_heka_pm_output(osb, NULL, NULL, false)); + mu_assert_rv(1, lsb_heka_pm_output(asb, &m, NULL, false)); + + lsb_heka_destroy_sandbox(isb); + lsb_heka_destroy_sandbox(asb); + lsb_heka_destroy_sandbox(osb); + lsb_free_heka_message(&m); + + mu_assert(strcmp(lsb_heka_get_error(NULL), "") == 0, "not empty"); + mu_assert(lsb_heka_get_lua_file(NULL) == NULL, "not null"); + lsb_heka_stats stats = lsb_heka_get_stats(NULL); + mu_assert(0 == stats.mem_cur, "received: %llu", stats.mem_cur); + mu_assert(!lsb_heka_is_running(NULL), "running is true"); + return NULL; +} + + static char* test_create_input_sandbox() { static const char *lua_file = "lua/input.lua"; lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, - iim); + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); const char* lfn = lsb_heka_get_lua_file(hsb); mu_assert(strcmp(lua_file, lfn) == 0, "expected %s received %s", lua_file, @@ -316,6 +354,20 @@ static char* test_timer_event() } +static char* test_stop_input() +{ + static const char* state_file = "stop.data"; + remove(state_file); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, dlog, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + lsb_heka_stop_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "received %s", e); + return NULL; +} + + static char* test_pm_input() { struct pm_result { @@ -333,14 +385,11 @@ static char* test_pm_input() { .ncp = NAN, .scp = NULL, .rv = 0, .err = "no cp" }, }; - lsb_heka_message m; - mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); - for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ - int rv = lsb_heka_pm_input(hsb, &m, results[i].ncp, - results[i].scp, true); + for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i) { + int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); const char *err = lsb_heka_get_error(hsb); mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", results[i].err, err); @@ -351,10 +400,45 @@ static char* test_pm_input() lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(5 == stats.pm_cnt, "expected %llu", stats.pm_cnt); mu_assert(1 == stats.pm_failures, "expected %llu", stats.pm_failures); + mu_assert(0 < stats.mem_cur, "expected %llu", stats.mem_cur); mu_assert(0 < stats.pm_avg, "received %g", stats.pm_avg); mu_assert(0 < stats.pm_sd, "received %g", stats.pm_sd); e = lsb_heka_destroy_sandbox(hsb); - lsb_free_heka_message(&m); + return NULL; +} + +static char* test_pm_error() +{ + struct pm_result { + double ncp; + const char *scp; + int rv; + const char *err; + }; + + struct pm_result results[] = { + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:15: boom" }, + { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, + { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:21: aaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, // >max error message + }; + + lsb_heka_sandbox *hsb; + for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i) { + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", + results[i].err, err); + mu_assert(results[i].rv == rv, "test: %u expected: %d received: %d", i, + results[i].rv, + rv); + e = lsb_heka_destroy_sandbox(hsb); + } return NULL; } @@ -442,8 +526,8 @@ static char* test_im_input() "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(6 == stats.im_cnt, "expected %llu", stats.im_cnt); - mu_assert(303 == stats.im_bytes, "expected %llu", stats.im_bytes); + mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(305 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -469,10 +553,12 @@ static char* test_encode_message() { lsb_heka_sandbox *hsb; hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n" "Hostname = 'sh';Logger = 'sl'", dlog, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(33 == stats.out_max, "received %llu", stats.out_max); + mu_assert(172 == stats.out_max, "received %llu", stats.out_max); e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -539,11 +625,14 @@ static char* benchmark_decode_message() static char* all_tests() { + mu_run_test(test_api_assertion); mu_run_test(test_create_input_sandbox); mu_run_test(test_create_analysis_sandbox); mu_run_test(test_create_output_sandbox); mu_run_test(test_timer_event); + mu_run_test(test_stop_input); mu_run_test(test_pm_input); + mu_run_test(test_pm_error); mu_run_test(test_pm_analysis); mu_run_test(test_pm_no_return); mu_run_test(test_pm_output); diff --git a/src/luasandbox.c b/src/luasandbox.c index dd508c1..8fad3df 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -24,6 +24,10 @@ #include "luasandbox_serialize.h" +lsb_err_id LSB_ERR_INIT = "already initialized"; +lsb_err_id LSB_ERR_LUA = "lua error"; // use lsb_get_error for more detail +lsb_err_id LSB_ERR_TERMINATED = "sandbox already terminated"; + static jmp_buf g_jbuf; static const luaL_Reg preload_module_list[] = { @@ -373,9 +377,9 @@ static void set_random_seed() lsb_lua_sandbox* lsb_create(void *parent, - const char *lua_file, - const char *cfg, - lsb_logger logger) + const char *lua_file, + const char *cfg, + lsb_logger logger) { if (!lua_file) { if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); @@ -454,22 +458,22 @@ lsb_lua_sandbox* lsb_create(void *parent, } -int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) +lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) { if (!lsb) { - return 1; + return LSB_ERR_UTIL_NULL; } if (lsb->state != LSB_UNKNOWN) { - lsb_terminate(lsb, "already initialized"); - return 1; + lsb_terminate(lsb, LSB_ERR_INIT); + return LSB_ERR_INIT; } if (state_file && strlen(state_file) > 0) { lsb->state_file = malloc(strlen(state_file) + 1); if (!lsb->state_file) { - lsb_terminate(lsb, "memory allocation failure"); - return 1; + lsb_terminate(lsb, LSB_ERR_UTIL_OOM); + return LSB_ERR_UTIL_OOM; } strcpy(lsb->state_file, state_file); } @@ -492,16 +496,16 @@ int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) lua_getglobal(lsb->lua, "require"); if (!lua_iscfunction(lsb->lua, -1)) { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_init 'require' not found"); + "lsb_init() 'require' not found"); lsb_terminate(lsb, NULL); - return 1; + return LSB_ERR_LUA; } lua_pushstring(lsb->lua, LUA_BASELIBNAME); if (lua_pcall(lsb->lua, 1, 0, 0)) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_init %s", lua_tostring(lsb->lua, -1)); lsb_terminate(lsb, NULL); - return 1; + return LSB_ERR_LUA; } if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { @@ -526,7 +530,7 @@ int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } lsb_terminate(lsb, NULL); - return 2; + return LSB_ERR_LUA; } else { lua_gc(lsb->lua, LUA_GCCOLLECT, 0); lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = instruction_usage(lsb); @@ -536,29 +540,34 @@ int lsb_init(lsb_lua_sandbox *lsb, const char *state_file) lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; } lsb->state = LSB_RUNNING; - if (lsb->state_file != NULL) { - if (lsb_restore_global_data(lsb)) { - return 3; - } + if (lsb->state_file) { + lsb_err_value ret = restore_global_data(lsb); + if (ret) return ret; } } lua_atpanic(lsb->lua, pf); - return 0; + return NULL; } -static void stop_hook(lua_State* lua, lua_Debug* ar) +static void stop_hook(lua_State *lua, lua_Debug *ar) { (void)ar; /* unused arg. */ lua_sethook(lua, NULL, 0, 0); - luaL_error(lua, "shutting down"); + luaL_error(lua, LSB_SHUTTING_DOWN); } -void lsb_stop_sandbox(lsb_lua_sandbox* lsb) +void lsb_stop_sandbox(lsb_lua_sandbox *lsb) { - lua_State* lua = lsb_get_lua(lsb); - lua_sethook(lua, stop_hook, LUA_MASKCALL | LUA_MASKRET | LUA_MASKCOUNT, 1); + if (!lsb) { + return; + } + + lua_State *lua = lsb_get_lua(lsb); + if (lua) { + lua_sethook(lua, stop_hook, LUA_MASKCALL | LUA_MASKRET | LUA_MASKCOUNT, 1); + } } @@ -569,7 +578,7 @@ char* lsb_destroy(lsb_lua_sandbox *lsb) return err; } - if (lsb_preserve_global_data(lsb) != 0) { + if (preserve_global_data(lsb)) { size_t len = strlen(lsb->error_message); err = malloc(len + 1); if (err != NULL) { @@ -588,7 +597,7 @@ char* lsb_destroy(lsb_lua_sandbox *lsb) size_t lsb_usage(lsb_lua_sandbox *lsb, lsb_usage_type utype, lsb_usage_stat ustat) { - if (!lsb || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { + if (!lsb || !lsb->lua || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { return 0; } #ifdef LUA_JIT @@ -611,10 +620,7 @@ size_t lsb_usage(lsb_lua_sandbox *lsb, lsb_usage_type utype, const char* lsb_get_error(lsb_lua_sandbox *lsb) { - if (lsb) { - return lsb->error_message; - } - return ""; + return lsb ? lsb->error_message : ""; } @@ -633,42 +639,45 @@ void lsb_set_error(lsb_lua_sandbox *lsb, const char *err) lua_State* lsb_get_lua(lsb_lua_sandbox *lsb) { - return lsb->lua; + return lsb ? lsb->lua : NULL; } const char* lsb_get_lua_file(lsb_lua_sandbox *lsb) { - return lsb->lua_file; + return lsb ? lsb->lua_file : NULL; } void* lsb_get_parent(lsb_lua_sandbox *lsb) { - return lsb->parent; + return lsb ? lsb->parent : NULL; } lsb_state lsb_get_state(lsb_lua_sandbox *lsb) { - if (lsb) { - return lsb->state; - } - return LSB_UNKNOWN; + return lsb ? lsb->state : LSB_UNKNOWN; } void lsb_add_function(lsb_lua_sandbox *lsb, lua_CFunction func, const char *func_name) { + if (!lsb || !func || !func_name) return; + if (!lsb->lua) return; + lua_pushlightuserdata(lsb->lua, (void *)lsb); lua_pushcclosure(lsb->lua, func, 1); lua_setglobal(lsb->lua, func_name); } -int lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name) +lsb_err_value lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name) { + if (!lsb || !func_name) return LSB_ERR_UTIL_NULL; + if (!lsb->lua) return LSB_ERR_TERMINATED; + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, (int)lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); @@ -677,20 +686,21 @@ int lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name) } lua_getglobal(lsb->lua, func_name); if (!lua_isfunction(lsb->lua, -1)) { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "%s() function was not found", + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "%s() not found", func_name); if (len >= LSB_ERROR_SIZE || len < 0) { lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } - return 1; + return LSB_ERR_LUA; } - return 0; + return NULL; } void lsb_pcall_teardown(lsb_lua_sandbox *lsb) { + if (!lsb) return; + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = (unsigned)instruction_usage(lsb); if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] @@ -703,6 +713,8 @@ void lsb_pcall_teardown(lsb_lua_sandbox *lsb) void lsb_terminate(lsb_lua_sandbox *lsb, const char *err) { + if (!lsb) return; + if (err) { strncpy(lsb->error_message, err, LSB_ERROR_SIZE); lsb->error_message[LSB_ERROR_SIZE - 1] = 0; diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index a005529..af7e60f 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -28,4 +28,22 @@ struct lsb_lua_sandbox { char error_message[LSB_ERROR_SIZE]; }; +/** + * Serialize all user global data to disk. + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_err_value NULL on success error message on failure + */ +lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb); + +/** + * Restores previously serialized data from disk. + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_err_value NULL on success error message on failure + */ +lsb_err_value restore_global_data(lsb_lua_sandbox *lsb); + #endif diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c index 4016704..c2e2ff6 100644 --- a/src/luasandbox_output.c +++ b/src/luasandbox_output.c @@ -42,7 +42,7 @@ lua_CFunction lsb_get_output_function(lua_State *lua, int index) void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) { if (!append) { - lsb_clear_output_buffer(&lsb->output); + lsb->output.pos = 0; } int result = 0; @@ -109,11 +109,8 @@ void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) const char* lsb_get_output(lsb_lua_sandbox *lsb, size_t *len) { - if (len) { - *len = lsb->output.pos; - } - if (lsb->output.err || lsb->output.pos == 0) return ""; - - lsb_clear_output_buffer(&lsb->output); + if (len) *len = lsb->output.pos; + if (lsb->output.pos == 0) return ""; + lsb->output.pos = 0; // internally reset the buffer for the next write return lsb->output.buf; } diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 480c95d..95ee7c0 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -51,9 +51,9 @@ static const char *serialize_function = "lsb_serialize"; * @param data Pointer to the serialization state data. * @param parent Index pointing to the parent's name in the table array. * - * @return int Zero on success, non-zero on failure. + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); /** @@ -61,13 +61,13 @@ serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); * * @param lsb Pointer to the sandbox. * @param index Lua stack index where the data resides. - * @param output Pointer the output collector. + * @param ob Pointer the output buffer. * - * @return int + * @return lsb_err_value NULL on success error message on failure */ -static int serialize_data(lsb_lua_sandbox *lsb, - int index, - lsb_output_buffer *output); +static lsb_err_value serialize_data(lsb_lua_sandbox *lsb, + int index, + lsb_output_buffer *ob); /** * Serializes a table key value pair. @@ -76,9 +76,9 @@ static int serialize_data(lsb_lua_sandbox *lsb, * @param data Pointer to the serialization state data. * @param parent Index pointing to the parent's name in the table array. * - * @return int Zero on success, non-zero on failure. + * @return lsb_err_value NULL on success error message on failure */ -static int +static lsb_err_value serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); /** @@ -115,6 +115,10 @@ ignore_value_type(lsb_lua_sandbox *lsb, serialization_data *data, int index, lua_CFunction *fp) { switch (lua_type(lsb->lua, index)) { + case LUA_TSTRING: + case LUA_TNUMBER: + case LUA_TBOOLEAN: + return 0; case LUA_TTABLE: if (lua_getmetatable(lsb->lua, index) != 0) { lua_pop(lsb->lua, 1); // Remove the metatable. @@ -123,21 +127,14 @@ ignore_value_type(lsb_lua_sandbox *lsb, serialization_data *data, int index, if (lua_topointer(lsb->lua, index) == data->globals) { return 1; } - break; + return 0; case LUA_TUSERDATA: *fp = get_serialize_function(lsb->lua, index); - if (!*fp) { - return 1; - } + return !*fp ? 1 : 0; + default: break; - case LUA_TNONE: - case LUA_TFUNCTION: - case LUA_TTHREAD: - case LUA_TLIGHTUSERDATA: - case LUA_TNIL: - return 1; } - return 0; + return 1; } @@ -213,33 +210,32 @@ static int get_preservation_version(lua_State *lua) } -static int +static lsb_err_value serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) { - int result = 0; + lsb_err_value ret = NULL; lua_checkstack(lsb->lua, 2); lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - result = serialize_kvp(lsb, data, parent); + while (!ret && lua_next(lsb->lua, -2) != 0) { + ret = serialize_kvp(lsb, data, parent); lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for // the next interation. } - return result; + return ret; } -static int -serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *output) +static lsb_err_value +serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *ob) { - lsb_clear_output_buffer(output); + lsb_err_value ret = NULL; + ob->pos = 0; // clear the buffer switch (lua_type(lsb->lua, index)) { case LUA_TNUMBER: - if (lsb_serialize_double(output, lua_tonumber(lsb->lua, index))) { - return 1; - } + ret = lsb_serialize_double(ob, lua_tonumber(lsb->lua, index)); break; case LUA_TSTRING: - // The stack is cleaned up on failure by lsb_preserve_global_data + // The stack is cleaned up on failure by preserve_global_data // but for clarity it is incrementally cleaned up anyway. lua_checkstack(lsb->lua, 4); lua_getglobal(lsb->lua, LUA_STRLIBNAME); @@ -249,14 +245,15 @@ serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *output) "serialize_data cannot access the string format function"); lua_pop(lsb->lua, 2); // Remove the bogus format function and // string table. - return 1; + return LSB_ERR_LUA; } lua_pushstring(lsb->lua, "%q"); lua_pushvalue(lsb->lua, index - 3); if (lua_pcall(lsb->lua, 2, 1, 0) == 0) { - if (lsb_outputf(output, "%s", lua_tostring(lsb->lua, -1))) { + ret = lsb_outputf(ob, "%s", lua_tostring(lsb->lua, -1)); + if (ret) { lua_pop(lsb->lua, 1); // Remove the string table. - return 1; + return ret; } } else { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, @@ -267,43 +264,42 @@ serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *output) } lua_pop(lsb->lua, 2); // Remove the error message and the string // table. - return 1; + return LSB_ERR_LUA; } lua_pop(lsb->lua, 2); // Remove the pcall result and the string table. break; case LUA_TBOOLEAN: - if (lsb_outputf(output, "%s", lua_toboolean(lsb->lua, index) - ? "true" : "false")) { - return 1; - } + ret = lsb_outputf(ob, "%s", lua_toboolean(lsb->lua, index) ? "true" : + "false"); break; default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "serialize_data cannot preserve type '%s'", lua_typename(lsb->lua, lua_type(lsb->lua, index))); - return 1; + ret = LSB_ERR_LUA; } - return 0; + return ret; } -static int +static lsb_err_value serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) { - int kindex = -2, vindex = -1, result = 0; + lsb_err_value ret = NULL; lua_CFunction fp = NULL; + int kindex = -2, vindex = -1; if (ignore_value_type(lsb, data, vindex, &fp)) { - return 0; + return ret; } - if (serialize_data(lsb, kindex, &lsb->output)) { - return 1; + ret = serialize_data(lsb, kindex, &lsb->output); + if (ret) { + return ret; } size_t pos = data->keys.pos; - if (lsb_outputf(&data->keys, "%s[%s]", data->keys.buf + parent, - lsb->output.buf)) { - return 1; - } + ret = lsb_outputf(&data->keys, "%s[%s]", data->keys.buf + parent, + lsb->output.buf); + if (ret) return ret; if (lua_type(lsb->lua, vindex) == LUA_TTABLE) { const void *ptr = lua_topointer(lsb->lua, vindex); @@ -313,11 +309,11 @@ serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) if (seen != NULL) { data->keys.pos += 1; fprintf(data->fh, "%s = {}\n", data->keys.buf + pos); - result = serialize_table(lsb, data, pos); + ret = serialize_table(lsb, data, pos); } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve table out of memory"); - return 1; + "lsb_serialize preserve table out of memory"); + return LSB_ERR_UTIL_OOM; } } else { fprintf(data->fh, "%s = ", data->keys.buf + pos); @@ -333,21 +329,21 @@ serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) data->keys.pos += 1; lua_pushlightuserdata(lsb->lua, data->keys.buf + pos); lua_pushlightuserdata(lsb->lua, &lsb->output); - lsb_clear_output_buffer(&lsb->output); - result = fp(lsb->lua); + lsb->output.pos = 0; + int result = fp(lsb->lua); lua_pop(lsb->lua, 2); // remove the key and the output - if (result == 0) { + if (!result) { size_t n = fwrite(lsb->output.buf, 1, lsb->output.pos, data->fh); if (n != lsb->output.pos) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_serialize failed %s", data->keys.buf + pos); - return 1; + return LSB_ERR_LUA; } } } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_serialize out of memory %s", data->keys.buf + pos); - return 1; + return LSB_ERR_UTIL_OOM; } } else { fprintf(data->fh, "%s = ", data->keys.buf + pos); @@ -358,21 +354,22 @@ serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) } else { fprintf(data->fh, "%s = ", data->keys.buf + pos); data->keys.pos = pos; - result = serialize_data(lsb, vindex, &lsb->output); - if (result == 0) { + ret = serialize_data(lsb, vindex, &lsb->output); + if (!ret) { fprintf(data->fh, "%s\n", lsb->output.buf); } } - return result; + return ret; } -int lsb_preserve_global_data(lsb_lua_sandbox *lsb) +lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) { if (!lsb->lua || !lsb->state_file) { - return 0; + return NULL; } + lua_sethook(lsb->lua, NULL, 0, 0); // make sure the string library is loaded before we start lua_getglobal(lsb->lua, LUA_STRLIBNAME); @@ -380,11 +377,18 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) lua_getglobal(lsb->lua, "require"); if (!lua_iscfunction(lsb->lua, -1)) { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_preserve_global_data 'require' not found"); - return 1; + "preserve_global_data 'require' function not found"); + return LSB_ERR_LUA; } lua_pushstring(lsb->lua, LUA_STRLIBNAME); - lua_call(lsb->lua, 1, 1); + if (lua_pcall(lsb->lua, 1, 1, 0)) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, + "preserve_global_data failed loading 'string'"); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + return LSB_ERR_LUA; + } } lua_pop(lsb->lua, 1); @@ -393,15 +397,15 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) FILE *fh = fopen(lsb->state_file, "wb"); if (fh == NULL) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_preserve_global_data could not open: %s", + "preserve_global_data could not open: %s", lsb->state_file); if (len >= LSB_ERROR_SIZE || len < 0) { lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } - return 1; + return LSB_ERR_LUA;; } - int result = 0; + lsb_err_value ret = NULL; serialization_data data; data.fh = fh; @@ -412,7 +416,6 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) // size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif - lua_sethook(lsb->lua, NULL, 0, 0); // size_t cur_output_size = lsb->output.size; // size_t max_output_size = lsb->output.maxsize; lsb->output.maxsize = 0; @@ -423,21 +426,23 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) data.tables.array = malloc(data.tables.size * sizeof(table_ref)); if (data.tables.array == NULL || lsb_init_output_buffer(&data.keys, 0)) { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_preserve_global_data out of memory"); - result = 1; + "preserve_global_data out of memory"); + ret = LSB_ERR_UTIL_OOM; } else { fprintf(data.fh, "if %s and %s ~= %d then return end\n", preservation_version, preservation_version, get_preservation_version(lsb->lua)); - lsb_outputs(&data.keys, "_G", 2); - data.keys.pos += 1; - data.globals = lua_topointer(lsb->lua, -1); - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - result = serialize_kvp(lsb, &data, 0); - lua_pop(lsb->lua, 1); + ret = lsb_outputs(&data.keys, "_G", 2); + if (!ret) { + data.keys.pos += 1; // preserve the NUL in this use case + data.globals = lua_topointer(lsb->lua, -1); + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (!ret && lua_next(lsb->lua, -2) != 0) { + ret = serialize_kvp(lsb, &data, 0); + lua_pop(lsb->lua, 1); + } } lua_pop(lsb->lua, lua_gettop(lsb->lua)); // Wipe the entire Lua stack. Since incremental cleanup on failure @@ -446,9 +451,7 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) free(data.tables.array); lsb_free_output_buffer(&data.keys); fclose(fh); - if (result != 0) { - remove(lsb->state_file); - } + if (ret) remove(lsb->state_file); // Uncomment if we start preserving state when not destroying the sandbox // Note: serialization uses the output buffer, inprogress output can be @@ -474,8 +477,7 @@ int lsb_preserve_global_data(lsb_lua_sandbox *lsb) } // end restore */ - - return result; + return ret; } @@ -490,10 +492,13 @@ static int file_exists(const char *fn) } -int lsb_restore_global_data(lsb_lua_sandbox *lsb) +lsb_err_value restore_global_data(lsb_lua_sandbox *lsb) { - if (!lsb || !lsb->state_file || !file_exists(lsb->state_file)) { - return 0; + if (!lsb) { + return LSB_ERR_UTIL_NULL; + } + if (!lsb->state_file || !file_exists(lsb->state_file)) { + return NULL; } // Clear the sandbox limits during restoration. @@ -505,17 +510,17 @@ int lsb_restore_global_data(lsb_lua_sandbox *lsb) #endif lua_sethook(lsb->lua, NULL, 0, 0); - int err = 0; - if ((err = luaL_dofile(lsb->lua, lsb->state_file)) != 0) { + int err = luaL_dofile(lsb->lua, lsb->state_file); + if (err) { if (LUA_ERRFILE != err) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_restore_global_data %s", + "restore_global_data %s", lua_tostring(lsb->lua, -1)); if (len >= LSB_ERROR_SIZE || len < 0) { lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } lsb_terminate(lsb, NULL); - return 1; + return LSB_ERR_LUA; } } #ifdef LUA_JIT @@ -527,7 +532,7 @@ int lsb_restore_global_data(lsb_lua_sandbox *lsb) lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; #endif - return 0; + return NULL; } @@ -539,33 +544,34 @@ void lsb_add_serialize_function(lua_State *lua, lua_CFunction fp) } -int lsb_serialize_binary(lsb_output_buffer *ob, const void *src, size_t len) +lsb_err_value lsb_serialize_binary(lsb_output_buffer *ob, const void *src, size_t len) { + lsb_err_value ret = NULL; const char *uc = (const char *)src; - for (unsigned i = 0; i < len; ++i) { + for (unsigned i = 0; !ret && i < len; ++i) { switch (uc[i]) { case '\n': - if (lsb_outputs(ob, "\\n", 2)) return 1; + ret = lsb_outputs(ob, "\\n", 2); break; case '\r': - if (lsb_outputs(ob, "\\r", 2)) return 1; + ret = lsb_outputs(ob, "\\r", 2); break; case '"': - if (lsb_outputs(ob, "\\\"", 2)) return 1; + ret = lsb_outputs(ob, "\\\"", 2); break; case '\\': - if (lsb_outputs(ob, "\\\\", 2)) return 1; + ret = lsb_outputs(ob, "\\\\", 2); break; default: - if (lsb_outputc(ob, uc[i])) return 1; + ret = lsb_outputc(ob, uc[i]); break; } } - return 0; + return ret; } -int lsb_serialize_double(lsb_output_buffer *ob, double d) +lsb_err_value lsb_serialize_double(lsb_output_buffer *ob, double d) { if (isnan(d)) { return lsb_outputs(ob, "0/0", 3); diff --git a/src/test/mu_test.h b/src/test/mu_test.h index 206651b..e8d6bc0 100644 --- a/src/test/mu_test.h +++ b/src/test/mu_test.h @@ -15,7 +15,7 @@ #define snprintf _snprintf #endif -#ifdef _MSC_VER +#if defined(_MSC_VER) #define PRIuSIZE "Iu" #else #define PRIuSIZE "zu" diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 4983e12..3993d84 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -34,6 +34,7 @@ size_t written_data_len = 0; static const char *test_cfg = + "userflag = true\n" "memory_limit = 0\n" "instruction_limit = 0\n" "output_limit = 0\n" @@ -58,6 +59,17 @@ int file_exists(const char *fn) return 0; } +void dlog(const char *component, int level, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, + component ? component : "unnamed"); + vfprintf(stderr, fmt, args); + fwrite("\n", 1, 1, stderr); + va_end(args); +} + int process(lsb_lua_sandbox *lsb, double ts) { @@ -162,11 +174,74 @@ int write_output(lua_State *lua) } +static char* test_api_assertion() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", "", NULL); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + lsb_stop_sandbox(NULL); + mu_assert(lsb_destroy(NULL) == NULL, "not null"); + mu_assert(lsb_usage(NULL, 0, 0) == 0, "not 0"); + mu_assert(lsb_usage(sb, LSB_UT_MAX, 0) == 0, "not 0"); + mu_assert(lsb_usage(sb, 0, LSB_US_MAX) == 0, "not 0"); + mu_assert(strcmp(lsb_get_error(NULL), "") == 0, "not empty"); + lsb_set_error(NULL, "foo"); + mu_assert(lsb_get_lua(NULL) == NULL, "not null"); + mu_assert(lsb_get_lua_file(NULL) == NULL, "not null"); + mu_assert(lsb_get_parent(NULL) == NULL, "not null"); + mu_assert(lsb_get_state(NULL) == LSB_UNKNOWN, "not unknown"); + lsb_add_function(NULL, write_output, "foo"); + lsb_add_function(sb, NULL, "foo"); + lsb_add_function(sb, write_output, NULL); + mu_assert(lsb_pcall_setup(NULL, "foo") == LSB_ERR_UTIL_NULL, "not null"); + mu_assert(lsb_pcall_setup(sb, NULL) == LSB_ERR_UTIL_NULL, "not null"); + lsb_add_function(NULL, NULL, NULL); + lsb_pcall_teardown(NULL); + lsb_terminate(NULL, NULL); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_create() +{ + static char *cfg = "function foo() return 0 end\nt = {[true] = 1}\n"; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", cfg, dlog); + mu_assert(sb, "lsb_create() failed"); + lsb_destroy(sb); + sb = lsb_create(NULL, "lua/counter.lua", cfg, NULL); + mu_assert(sb, "lsb_create() failed"); + lsb_destroy(sb); + return NULL; +} + + static char* test_create_error() { lsb_lua_sandbox *sb = lsb_create(NULL, NULL, NULL, NULL); mu_assert(!sb, "lsb_create() null lua_file"); + sb = lsb_create(NULL, "lua/counter.lua", "output_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "instruction_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "path = 1", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "cpath = 1", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "test = {", dlog); + mu_assert(!sb, "lsb_create() invalid config"); + return NULL; } @@ -174,16 +249,15 @@ static char* test_create_error() static char* test_read_config() { const char *cfg = "memory_limit = 65765\n" - "instruction_limit = 1000\n" - "output_limit = 1024\n" - "array = {'foo', 99}\n" - "hash = {foo = 'bar', hash1 = {subfoo = 'subbar'}}\n" - MODULE_PATH; + "instruction_limit = 1000\n" + "output_limit = 1024\n" + "array = {'foo', 99}\n" + "hash = {foo = 'bar', hash1 = {subfoo = 'subbar'}}\n" + MODULE_PATH; lsb_lua_sandbox *sb = lsb_create(NULL, "lua/read_config.lua", cfg, NULL); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -195,15 +269,14 @@ static char* test_read_config() static char* test_init_error() { // null sandbox - int result = lsb_init(NULL, NULL); - mu_assert(result == 1, "lsb_init() null sandbox ptr"); + lsb_err_value ret = lsb_init(NULL, NULL); + mu_assert(ret == LSB_ERR_UTIL_NULL, "lsb_init() null sandbox ptr"); // load error lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple1.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, NULL); - mu_assert(result == 2, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, NULL); + mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); lsb_state s = lsb_get_state(sb); mu_assert(s == LSB_TERMINATED, "lsb_get_state() received: %d", s); e = lsb_destroy(sb); @@ -212,9 +285,8 @@ static char* test_init_error() // out of memory sb = lsb_create(NULL, "lua/simple.lua", "memory_limit = 6000", NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, NULL); - mu_assert(result == 2, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, NULL); + mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s\n", e); @@ -222,10 +294,10 @@ static char* test_init_error() mu_assert(sb, "lsb_create() received: NULL"); // disabled external modules - result = lsb_init(sb, NULL); - mu_assert(result == 2, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - const char *expected = "lua/lpeg_date_time.lua:7: module 'date_time' not found:"; + ret = lsb_init(sb, NULL); + mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); + const char *expected = "lua/lpeg_date_time.lua:7: module 'date_time' not " + "found:"; mu_assert(strcmp(lsb_get_error(sb), expected) == 0, "lsb_get_error() received: %s", lsb_get_error(sb)); @@ -238,16 +310,15 @@ static char* test_init_error() static char* test_destroy_error() { - const char *expected = "lsb_preserve_global_data could not open: " + const char *expected = "preserve_global_data could not open: " "invaliddir/simple.preserve"; e = lsb_destroy(NULL); mu_assert(!e, "lsb_destroy() received: %s", e); lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, "invaliddir/simple.preserve"); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, "invaliddir/simple.preserve"); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(e, "lsb_destroy() received NULL"); mu_assert(strcmp(e, expected) == 0, @@ -284,14 +355,21 @@ static char* test_usage_error() return NULL; } -static char* test_misc() -{ - lsb_state s = lsb_get_state(NULL); - mu_assert(s == LSB_UNKNOWN, "lsb_get_state() received: %d", s); - - const char *le = lsb_get_error(NULL); - mu_assert(strlen(le) == 0, "lsb_get_error() received: %s", le); +static char* test_stop() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_stop_sandbox(sb); + lua_getglobal(sb->lua, "process"); + lua_pushnumber(sb->lua, 0); + mu_assert_rv(2, lua_pcall(sb->lua, 1, 2, 0)); + const char *msg = lua_tostring(sb->lua, -1); + mu_assert(strcmp(LSB_SHUTTING_DOWN, msg) == 0, "received: %s", msg); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); return NULL; } @@ -304,9 +382,8 @@ static char* test_simple() "output_limit = 1024;", NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, "simple.preserve"); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, "simple.preserve"); + mu_assert(!ret, "lsb_init() received: %s", ret); size_t u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); mu_assert(u > 0, "Current memory usage received: %" PRIuSIZE, u); @@ -358,11 +435,10 @@ static char* test_simple_error() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); - result = process(sb, 1); + int result = process(sb, 1); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -398,13 +474,12 @@ static char* test_output() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, "circular_buffer.preserve"); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, "circular_buffer.preserve"); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); for (int x = 0; outputs[x]; ++x) { - result = process(sb, x); + int result = process(sb, x); mu_assert(!result, "process() test: %d failed: %d %s", x, result, lsb_get_error(sb)); if (outputs[x][0]) { @@ -437,12 +512,11 @@ static char* test_output_errors() MODULE_PATH "output_limit = 128", NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, i); + int result = process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); const char *le = lsb_get_error(sb); @@ -470,11 +544,11 @@ static char* test_cbuf_errors() MODULE_PATH "memory_limit = 32767", NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s %s", ret, lsb_get_error(sb)); - result = process(sb, i); + int result = process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); const char *le = lsb_get_error(sb); @@ -496,9 +570,8 @@ static char* test_cbuf_core() test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, ""); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, ""); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -521,12 +594,11 @@ static char* test_cbuf() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); - result = report(sb, 0); + int result = report(sb, 0); mu_assert(result == 0, "report() received: %d", result); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "error %s", lsb_get_error(sb)); @@ -594,9 +666,8 @@ static char* test_cbuf_delta() test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); process(sb, 0); @@ -605,7 +676,7 @@ static char* test_cbuf_delta() process(sb, 2e9); process(sb, 2e9); process(sb, 2e9); - result = report(sb, 0); + int result = report(sb, 0); mu_assert(result == 0, "report() received: %d", result); mu_assert(strcmp(outputs[0], written_data) == 0, "received: %s", written_data); @@ -636,11 +707,10 @@ static char* test_cjson() MODULE_PATH "output_limit = 64", NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -656,12 +726,11 @@ static char* test_cjson_unlimited() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cjson_unlimited.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -719,11 +788,10 @@ static char* test_errors() NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); - result = process(sb, i); + int result = process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); const char *le = lsb_get_error(sb); @@ -758,11 +826,10 @@ static char* test_lpeg() lsb_lua_sandbox *sb = lsb_create(NULL, tests[i], test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -780,11 +847,10 @@ static char* test_util() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/util_test.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -803,9 +869,8 @@ static char* test_serialize() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -830,11 +895,10 @@ static char* test_restore() remove(output_file); lsb_lua_sandbox *sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); mu_assert(strcmp("101", written_data) == 0, "test: initial load received: %s", @@ -845,9 +909,8 @@ static char* test_restore() // re-load to test the preserved data sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, @@ -862,9 +925,8 @@ static char* test_restore() // re-load to test the preserved data with a version change sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, @@ -888,9 +950,8 @@ static char* test_serialize_failure() "lua/serialize_failure.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(e, "lsb_destroy() received: no error"); mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); @@ -909,9 +970,8 @@ static char* test_bloom_filter_core() test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -933,14 +993,13 @@ static char* test_bloom_filter() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/bloom_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); int i = 0; for (; tests[i]; ++i) { - result = process(sb, i); + int result = process(sb, i); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); result = report(sb, 0); @@ -949,7 +1008,7 @@ static char* test_bloom_filter() written_data); } - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); result = report(sb, 0); @@ -964,9 +1023,8 @@ static char* test_bloom_filter() sb = lsb_create(NULL, "lua/bloom_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); report(sb, 0); @@ -1004,9 +1062,8 @@ static char* test_hyperloglog_core() test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -1021,19 +1078,18 @@ static char* test_hyperloglog() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); for (int i = 0; i < 100000; ++i) { - result = process(sb, i); + int result = process(sb, i); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); } - result = report(sb, 0); + int result = report(sb, 0); mu_assert(result == 0, "report() received: %d", result); mu_assert(strcmp("100070", written_data) == 0, "test: initial received: %s", written_data); // count should remain the same @@ -1050,9 +1106,8 @@ static char* test_hyperloglog() sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); result = report(sb, 0); @@ -1089,11 +1144,10 @@ static char* test_struct() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/struct.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); e = lsb_destroy(sb); @@ -1108,9 +1162,8 @@ static char* test_sandbox_config() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sandbox_config.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -1133,14 +1186,13 @@ static char* test_cuckoo_filter() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cuckoo_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); int i = 0; for (; tests[i]; ++i) { - result = process(sb, i); + int result = process(sb, i); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); result = report(sb, 0); @@ -1149,7 +1201,7 @@ static char* test_cuckoo_filter() written_data); } - result = process(sb, 0); + int result = process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); result = report(sb, 0); @@ -1164,9 +1216,8 @@ static char* test_cuckoo_filter() sb = lsb_create(NULL, "lua/cuckoo_filter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); report(sb, 0); @@ -1208,12 +1259,11 @@ static char* test_sax() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sax.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); - result = report(sb, 0); + int result = report(sb, 0); mu_assert(result == 0, "report() received: %d %s", result, lsb_get_error(sb)); mu_assert(strcmp("### CDC ##### #####", written_data) == 0, "received: %s", written_data); @@ -1230,9 +1280,8 @@ static char* test_sax() sb = lsb_create(NULL, "lua/sax.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); report(sb, 0); @@ -1251,9 +1300,8 @@ static char* benchmark_counter() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); clock_t t = clock(); for (int x = 0; x < iter; ++x) { process(sb, 0); @@ -1279,9 +1327,8 @@ static char* benchmark_serialize() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } @@ -1302,9 +1349,8 @@ static char* benchmark_deserialize() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, "output/serialize.data"); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, "output/serialize.data"); + mu_assert(!ret, "lsb_init() received: %s", ret); free(sb->state_file); sb->state_file = NULL; // poke the internals to prevent serialization e = lsb_destroy(sb); @@ -1325,9 +1371,8 @@ static char* benchmark_lua_types_output() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); clock_t t = clock(); @@ -1350,9 +1395,8 @@ static char* benchmark_cbuf_output() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); clock_t t = clock(); @@ -1376,9 +1420,8 @@ static char* benchmark_cbuf_add() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer_add.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); double ts = 0; clock_t t = clock(); @@ -1405,9 +1448,8 @@ static char* benchmark_bloom_filter_add() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/bloom_filter_benchmark.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); clock_t t = clock(); @@ -1434,9 +1476,8 @@ static char* benchmark_hyperloglog_add() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); clock_t t = clock(); @@ -1465,9 +1506,8 @@ static char* benchmark_cuckoo_filter_add() "lua/cuckoo_filter_benchmark.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); clock_t t = clock(); @@ -1494,9 +1534,8 @@ static char* benchmark_sax_add() lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sax_benchmark.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); clock_t t = clock(); @@ -1519,12 +1558,14 @@ static char* benchmark_sax_add() static char* all_tests() { - mu_run_test(test_read_config); + mu_run_test(test_api_assertion); + mu_run_test(test_create); mu_run_test(test_create_error); + mu_run_test(test_read_config); mu_run_test(test_init_error); mu_run_test(test_destroy_error); mu_run_test(test_usage_error); - mu_run_test(test_misc); + mu_run_test(test_stop); mu_run_test(test_simple); mu_run_test(test_simple_error); mu_run_test(test_output); diff --git a/src/util/heka_message.c b/src/util/heka_message.c index ee7cbd4..198eb5d 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -16,7 +16,6 @@ #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" - static size_t decode_header(char *buf, size_t len, size_t max_message_size) { if (*buf != 0x08) { @@ -204,9 +203,9 @@ bool lsb_decode_heka_message(lsb_heka_message *m, size_t len, lsb_logger logger) { - if (!buf || len == 0) { + if (!m || !buf || len == 0) { if (logger) { - logger(__FUNCTION__, 4, "invalid buffer"); + logger(__FUNCTION__, 4, LSB_ERR_UTIL_NULL); } return false; } @@ -323,6 +322,13 @@ bool lsb_find_heka_message(lsb_heka_message *m, size_t *discarded_bytes, lsb_logger logger) { + if (!m || !ib || !discarded_bytes) { + if (logger) { + logger(__FUNCTION__, 4, LSB_ERR_UTIL_NULL); + } + return false; + } + *discarded_bytes = 0; if (ib->readpos == ib->scanpos) { return false; // empty buffer @@ -408,21 +414,24 @@ bool lsb_find_heka_message(lsb_heka_message *m, } -int lsb_init_heka_message(lsb_heka_message *m, int num_fields) +lsb_err_value lsb_init_heka_message(lsb_heka_message *m, int num_fields) { - if (num_fields < 1) return 1; + if (!m) return LSB_ERR_UTIL_NULL; + if (num_fields < 1) return LSB_ERR_UTIL_PRANGE; m->fields = malloc(num_fields * sizeof(lsb_heka_field)); - if (!m->fields) return 2; + if (!m->fields) return LSB_ERR_UTIL_OOM; m->fields_size = num_fields; lsb_clear_heka_message(m); - return 0; + return NULL; } void lsb_clear_heka_message(lsb_heka_message *m) { + if (!m) return; + lsb_init_const_string(&m->raw); lsb_init_const_string(&m->uuid); lsb_init_const_string(&m->type); @@ -441,6 +450,8 @@ void lsb_clear_heka_message(lsb_heka_message *m) void lsb_free_heka_message(lsb_heka_message *m) { + if (!m) return; + free(m->fields); m->fields = NULL; m->fields_size = 0; @@ -449,11 +460,15 @@ void lsb_free_heka_message(lsb_heka_message *m) bool lsb_read_heka_field(lsb_heka_message *m, - lsb_const_string *name, - int fi, - int ai, - lsb_read_value *val) + lsb_const_string *name, + int fi, + int ai, + lsb_read_value *val) { + if (!m || !name || !val) { + return false; + } + int fcnt = 0; const char *p, *e; val->type = LSB_READ_NIL; @@ -488,13 +503,18 @@ bool lsb_read_heka_field(lsb_heka_message *m, } -int lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) +lsb_err_value +lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) { + if (!ob) { + return LSB_ERR_UTIL_NULL; + } + static const size_t needed = 18; - lsb_clear_output_buffer(ob); // writing a uuid will always clear the buffer - //since it is the start of a new message - int result = lsb_expand_output_buffer(ob, needed); - if (result) return result; + ob->pos = 0; // writing a uuid will always clear the buffer as it is the + // start of a new message + lsb_err_value ret = lsb_expand_output_buffer(ob, needed); + if (ret) return ret; ob->buf[ob->pos++] = 2 | (LSB_PB_UUID << 3); // write key ob->buf[ob->pos++] = LSB_UUID_SIZE; // write length @@ -535,5 +555,5 @@ int lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) ob->buf[8] = (ob->buf[8] & 0x0F) | 0x40; ob->buf[10] = (ob->buf[10] & 0x0F) | 0xA0; } - return 0; + return NULL; } diff --git a/src/util/input_buffer.c b/src/util/input_buffer.c index e9176fa..a409a91 100644 --- a/src/util/input_buffer.c +++ b/src/util/input_buffer.c @@ -16,10 +16,12 @@ #include "luasandbox/util/util.h" #include "luasandbox/util/heka_message.h" -int lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size) +lsb_err_value +lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size) { + if (!b) return LSB_ERR_UTIL_NULL; b->buf = NULL; - if (max_message_size == 0) return 1; + if (max_message_size == 0) return LSB_ERR_UTIL_PRANGE; max_message_size += LSB_MAX_HDR_SIZE; b->size = max_message_size < BUFSIZ ? max_message_size : BUFSIZ; b->maxsize = max_message_size; @@ -27,12 +29,14 @@ int lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size) b->scanpos = 0; b->msglen = 0; b->buf = malloc(b->size); - return b->buf ? 0 : 2; + return b->buf ? NULL : LSB_ERR_UTIL_OOM; } void lsb_free_input_buffer(lsb_input_buffer *b) { + if (!b) return; + free(b->buf); b->buf = NULL; b->size = 0; @@ -42,8 +46,10 @@ void lsb_free_input_buffer(lsb_input_buffer *b) } -int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) +lsb_err_value lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) { + if (!b) return LSB_ERR_UTIL_NULL; + if (b->scanpos != 0) { // shift the data to the beginning of the buffer if (b->scanpos == b->readpos) { b->scanpos = b->readpos = 0; @@ -56,7 +62,7 @@ int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) if (b->readpos + len > b->size) { size_t newsize = b->readpos + len; - if (newsize > b->maxsize) return 1; + if (newsize > b->maxsize) return LSB_ERR_UTIL_FULL; newsize = lsb_lp2(newsize); if (newsize > b->maxsize) newsize = b->maxsize;; @@ -65,8 +71,8 @@ int lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) b->buf = tmp; b->size = newsize; } else { - return 2; + return LSB_ERR_UTIL_OOM; } } - return 0; + return NULL; } diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index 1b35e09..bfd6e66 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -23,8 +23,10 @@ #pragma warning( disable : 4056 ) #endif -int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) +lsb_err_value +lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) { + if (!b) return LSB_ERR_UTIL_NULL; if (max_message_size && max_message_size < LSB_OUTPUT_SIZE) { b->size = max_message_size; } else { @@ -33,36 +35,28 @@ int lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) b->maxsize = max_message_size; b->pos = 0; b->buf = malloc(b->size); - b->err = b->buf ? 0 : 2; - return b->err; + return b->buf ? NULL : LSB_ERR_UTIL_OOM; } void lsb_free_output_buffer(lsb_output_buffer *b) { + if (!b) return; free(b->buf); b->buf = NULL; b->size = 0; - lsb_clear_output_buffer(b); -} - - -void lsb_clear_output_buffer(lsb_output_buffer *b) -{ b->pos = 0; - b->err = 0; } -int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) +lsb_err_value lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) { - if (b->err) return b->err; + if (!b) return LSB_ERR_UTIL_NULL; - if (needed <= b->size - b->pos) return 0; + if (needed <= b->size - b->pos) return NULL; if (b->maxsize && needed + b->pos > b->maxsize) { - b->err = 1; - return b->err; + return LSB_ERR_UTIL_FULL; } size_t newsize = lsb_lp2(b->pos + needed); @@ -72,29 +66,33 @@ int lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) void *ptr = realloc(b->buf, newsize); if (!ptr) { - b->err = 2; - return b->err; + return LSB_ERR_UTIL_OOM; } b->buf = ptr; b->size = newsize; - return 0; + return NULL; } -int lsb_outputc(lsb_output_buffer *b, char ch) +lsb_err_value lsb_outputc(lsb_output_buffer *b, char ch) { - int result = lsb_expand_output_buffer(b, 2); - if (result) return result; + if (!b) return LSB_ERR_UTIL_NULL; + lsb_err_value ret = lsb_expand_output_buffer(b, 2); + if (ret) return ret; b->buf[b->pos++] = ch; b->buf[b->pos] = 0; - return 0; + return NULL; } -int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) +lsb_err_value lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) { + if (!b || !fmt) { + return LSB_ERR_UTIL_NULL; + } + va_list args; int remaining = 0; char *ptr = NULL, *old_ptr = NULL; @@ -112,11 +110,9 @@ int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) needed = remaining; } if (needed >= remaining) { - if (b->maxsize - && (b->size >= b->maxsize - || b->pos + needed >= b->maxsize)) { - b->err = 1; - return b->err; // exceeded max + if (b->maxsize && (b->size >= b->maxsize + || b->pos + needed >= b->maxsize)) { + return LSB_ERR_UTIL_FULL; } size_t newsize = b->size * 2; while ((size_t)needed >= newsize - b->pos) { @@ -132,8 +128,7 @@ int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) b->buf = p; b->size = newsize; } else { - b->err = 2; - return b->err; // malloc failed + return LSB_ERR_UTIL_OOM; } } else { b->pos += needed; @@ -142,24 +137,26 @@ int lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) } while (1); free(old_ptr); - return 0; + return NULL; } -int lsb_outputs(lsb_output_buffer *b, const char *str, size_t len) +lsb_err_value lsb_outputs(lsb_output_buffer *b, const char *str, size_t len) { - int result = lsb_expand_output_buffer(b, len + 1); - if (result) return result; + if (!b) return LSB_ERR_UTIL_NULL; + lsb_err_value ret = lsb_expand_output_buffer(b, len + 1); + if (ret) return ret; - memcpy(b->buf + b->pos, str, len); + memcpy(b->buf + b->pos, str, len + 1); b->pos += len; - b->buf[b->pos] = 0; - return 0; + return ret; } -int lsb_outputd(lsb_output_buffer *b, double d) +lsb_err_value lsb_outputd(lsb_output_buffer *b, double d) { + if (!b) return LSB_ERR_UTIL_NULL; + if (isnan(d)) { return lsb_outputs(b, "nan", 3); } @@ -173,8 +170,10 @@ int lsb_outputd(lsb_output_buffer *b, double d) } -int lsb_outputfd(lsb_output_buffer *b, double d) +lsb_err_value lsb_outputfd(lsb_output_buffer *b, double d) { + if (!b) return LSB_ERR_UTIL_NULL; + if (d < INT_MIN || d > INT_MAX) { return lsb_outputf(b, "%0.17g", d); } @@ -227,18 +226,18 @@ int lsb_outputfd(lsb_output_buffer *b, double d) number /= 10; } while (number > 0); - int result = lsb_expand_output_buffer(b, (p - buffer) + negative); - if (result) return result; + lsb_err_value ret = lsb_expand_output_buffer(b, (p - buffer) + negative); + if (!ret) { + if (negative) { + b->buf[b->pos++] = '-'; + } - if (negative) { - b->buf[b->pos++] = '-'; - } + do { + --p; + b->buf[b->pos++] = *p; + } while (p != buffer); - do { - --p; - b->buf[b->pos++] = *p; - } while (p != buffer); - - b->buf[b->pos] = 0; - return 0; + b->buf[b->pos] = 0; + } + return ret; } diff --git a/src/util/protobuf.c b/src/util/protobuf.c index 1084ca4..94a56fa 100644 --- a/src/util/protobuf.c +++ b/src/util/protobuf.c @@ -12,27 +12,33 @@ #include #include -#define check_rv(fn) { int rv = fn; if (rv) return rv; }; - const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype) { + if (!p || !tag || !wiretype) return NULL; + *wiretype = 7 & (unsigned char)*p; *tag = (unsigned char)*p >> 3; return ++p; } -int lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, +lsb_err_value lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, unsigned char wiretype) { - check_rv(lsb_expand_output_buffer(ob, 1)); - ob->buf[ob->pos++] = wiretype | (tag << 3); - return 0; + lsb_err_value ret = lsb_expand_output_buffer(ob, 1); + if (!ret) { + ob->buf[ob->pos++] = wiretype | (tag << 3); + } + return ret; } const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi) { + if (!p || !e) { + return NULL; + } + *vi = 0; int i, shift = 0; for (i = 0; p != e && i < LSB_MAX_VARINT_BYTES; ++i, ++p) { @@ -50,6 +56,8 @@ const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi) int lsb_pb_output_varint(char *buf, unsigned long long i) { int pos = 0; + if (!buf) return pos; + if (i == 0) { buf[pos++] = 0; return pos; @@ -64,59 +72,68 @@ int lsb_pb_output_varint(char *buf, unsigned long long i) } -int lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i) +lsb_err_value lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i) { - check_rv(lsb_expand_output_buffer(ob, LSB_MAX_VARINT_BYTES)); - ob->pos += lsb_pb_output_varint(ob->buf + ob->pos, i); - return 0; + lsb_err_value ret = lsb_expand_output_buffer(ob, LSB_MAX_VARINT_BYTES); + if (!ret) { + ob->pos += lsb_pb_output_varint(ob->buf + ob->pos, i); + } + return ret; } -int lsb_pb_write_bool(lsb_output_buffer *ob, int i) +lsb_err_value lsb_pb_write_bool(lsb_output_buffer *ob, int i) { - check_rv(lsb_expand_output_buffer(ob, 1)); - if (i) { - ob->buf[ob->pos++] = 1; - } else { - ob->buf[ob->pos++] = 0; + lsb_err_value ret = lsb_expand_output_buffer(ob, 1); + if (!ret) { + if (i) { + ob->buf[ob->pos++] = 1; + } else { + ob->buf[ob->pos++] = 0; + } } - return 0; + return ret; } -int lsb_pb_write_double(lsb_output_buffer *ob, double i) +lsb_err_value lsb_pb_write_double(lsb_output_buffer *ob, double i) { static const size_t needed = sizeof(double); - check_rv(lsb_expand_output_buffer(ob, needed)); - // todo add big endian support if necessary - memcpy(&ob->buf[ob->pos], &i, needed); - ob->pos += needed; - return 0; + + lsb_err_value ret = lsb_expand_output_buffer(ob, needed); + if (!ret) { + // todo add big endian support if necessary + memcpy(&ob->buf[ob->pos], &i, needed); + ob->pos += needed; + } + return ret; } -int +lsb_err_value lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len) { - check_rv(lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH)); - check_rv(lsb_pb_write_varint(ob, len)); - check_rv(lsb_expand_output_buffer(ob, len)); - memcpy(&ob->buf[ob->pos], s, len); - ob->pos += len; - return 0; + lsb_err_value ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, len); + if (!ret) ret = lsb_expand_output_buffer(ob, len); + if (!ret) { + memcpy(&ob->buf[ob->pos], s, len); + ob->pos += len; + } + return ret; } -int lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos) +lsb_err_value lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos) { if (len_pos >= ob->pos) { - return 1; + return LSB_ERR_UTIL_PRANGE; } size_t len = ob->pos - len_pos - 1; if (len < 128) { ob->buf[len_pos] = (char)len; - return 0; + return NULL; } size_t l = len, cnt = 0; while (l) { @@ -124,9 +141,11 @@ int lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos) ++cnt; // compute the number of bytes needed for the varint length } size_t needed = cnt - 1; - check_rv(lsb_expand_output_buffer(ob, needed)); - ob->pos += needed; - memmove(&ob->buf[len_pos + cnt], &ob->buf[len_pos + 1], len); - lsb_pb_output_varint(ob->buf + len_pos, len); - return 0; + lsb_err_value ret = lsb_expand_output_buffer(ob, needed); + if (!ret) { + ob->pos += needed; + memmove(&ob->buf[len_pos + cnt], &ob->buf[len_pos + 1], len); + lsb_pb_output_varint(ob->buf + len_pos, len); + } + return ret; } diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index c34ea7a..490d24c 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -12,6 +12,7 @@ #include #include "../../test/mu_test.h" +#include "luasandbox/error.h" #include "luasandbox/util/heka_message.h" #define TEST_UUID "\x0a\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" @@ -50,6 +51,7 @@ static char* test_init() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 10), "failed"); lsb_free_heka_message(&m); + lsb_free_heka_message(NULL); return NULL; } @@ -89,6 +91,9 @@ static char* test_decode() bool ok = lsb_decode_heka_message(&m, tests[i].s, tests[i].len, logger); mu_assert(ok, "test: %d failed err: %s", i, lm.msg); } + mu_assert(!lsb_decode_heka_message(NULL, NULL, 0, NULL), "succeeded"); + mu_assert(!lsb_decode_heka_message(&m, NULL, 0, NULL), "succeeded"); + mu_assert(!lsb_decode_heka_message(&m, tests[0].s, 0, NULL), "succeeded"); lsb_free_heka_message(&m); return NULL; } @@ -184,6 +189,9 @@ static char* test_find_message() mu_assert(tests[i].b == b, "test: %u failed", i); mu_assert(tests[i].d == db, "test: %u failed expected: %" PRIuSIZE " received: %" PRIuSIZE, i, tests[i].d, db); } + mu_assert(!lsb_find_heka_message(NULL, NULL, true, NULL, NULL), "succeeded"); + mu_assert(!lsb_find_heka_message(&m, NULL, true, NULL, NULL), "succeeded"); + mu_assert(!lsb_find_heka_message(&m, &ib, true, NULL, NULL), "succeeded"); lsb_free_input_buffer(&ib); lsb_free_heka_message(&m); return NULL; @@ -256,6 +264,9 @@ static char* test_read_heka_field() mu_assert(v.type == LSB_READ_BOOL, "%d", v.type); mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + mu_assert(!lsb_read_heka_field(NULL, NULL, 0, 0, NULL), "succeeded"); + mu_assert(!lsb_read_heka_field(&m, NULL, 0, 0, NULL), "succeeded"); + mu_assert(!lsb_read_heka_field(&m, &cs, 0, 0, NULL), "succeeded"); lsb_free_heka_message(&m); return NULL; } @@ -263,46 +274,56 @@ static char* test_read_heka_field() static char* test_write_heka_uuid() { + lsb_err_value ret; lsb_output_buffer ob; lsb_init_output_buffer(&ob, LSB_UUID_SIZE + 2); const char header[2] = "\x0a\x10"; const char bin_uuid[LSB_UUID_SIZE] = { 0 }; - mu_assert_rv(0, lsb_write_heka_uuid(&ob, bin_uuid, LSB_UUID_SIZE)); + ret = lsb_write_heka_uuid(&ob, bin_uuid, LSB_UUID_SIZE); + mu_assert(!ret, "received %s", ret); mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); mu_assert(memcmp(ob.buf + 2, bin_uuid, LSB_UUID_SIZE) == 0, "invalid"); - const char str_uuid[LSB_UUID_STR_SIZE] = "00000000-0000-0000-0000-" + const char str_uuid[] = "00000000-0000-0000-0000-" "000000000000"; - mu_assert_rv(0, lsb_write_heka_uuid(&ob, str_uuid, LSB_UUID_STR_SIZE)); + ret = lsb_write_heka_uuid(&ob, str_uuid, LSB_UUID_STR_SIZE); + mu_assert(!ret, "received %s", ret); mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); mu_assert(memcmp(ob.buf + 2, bin_uuid, LSB_UUID_SIZE) == 0, "invalid"); - const char err_uuid[LSB_UUID_STR_SIZE] = "00000000+0000-0000-0000-" + const char err_uuid[] = "00000000+0000-0000-0000-" "000000000000"; - mu_assert_rv(0, lsb_write_heka_uuid(&ob, err_uuid, LSB_UUID_STR_SIZE)); + ret = lsb_write_heka_uuid(&ob, err_uuid, LSB_UUID_STR_SIZE); + mu_assert(!ret, "received %s", ret); mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); mu_assert(ob.buf[8] & 0x40, "invalid format should create a type 4 uuid"); - mu_assert_rv(0, lsb_write_heka_uuid(&ob, NULL, 0)); + ret = lsb_write_heka_uuid(&ob, NULL, 0); + mu_assert(!ret, "received %s", ret); mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(ob.buf[8] & 0x40, "null string should create a type 4 uuid"); lsb_free_output_buffer(&ob); - mu_assert_rv(0, lsb_write_heka_uuid(&ob, bin_uuid, 10)); + ret = lsb_write_heka_uuid(&ob, bin_uuid, 10); + mu_assert(!ret, "received %s", ret); mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(ob.buf[8] & 0x40, "unexpected length should create a type 4 uuid"); lsb_free_output_buffer(&ob); lsb_output_buffer sob; lsb_init_output_buffer(&sob, LSB_UUID_SIZE); - mu_assert_rv(1, lsb_write_heka_uuid(&sob, NULL, 0)); + ret = lsb_write_heka_uuid(&sob, NULL, 0); + mu_assert(ret, "received "); mu_assert(sob.pos == 0, "received: %" PRIuSIZE, sob.pos); lsb_free_output_buffer(&sob); + ret = lsb_write_heka_uuid(NULL, bin_uuid, LSB_UUID_SIZE); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received %s", lsb_err_string(ret)); + return NULL; } diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c index 7220577..6dc6cec 100644 --- a/src/util/test/test_input_buffer.c +++ b/src/util/test/test_input_buffer.c @@ -10,6 +10,7 @@ #include #include "../../test/mu_test.h" +#include "luasandbox/error.h" #include "luasandbox/util/input_buffer.h" #include "luasandbox/util/heka_message.h" @@ -22,12 +23,17 @@ static char* test_init_small_buf() { size_t size = 100; lsb_input_buffer b; + + lsb_err_value ret = lsb_init_input_buffer(NULL, size); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); mu_assert(b.size == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, b.size); mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, b.size); lsb_free_input_buffer(&b); + lsb_free_input_buffer(NULL); return NULL; } @@ -48,7 +54,8 @@ static char* test_init_large_buf() static char* test_init_zero_buf() { lsb_input_buffer b; - mu_assert(lsb_init_input_buffer(&b, 0), "init succeeded"); + lsb_err_value ret = lsb_init_input_buffer(&b, 0); + mu_assert(ret == LSB_ERR_UTIL_PRANGE, "received: %s", lsb_err_string(ret)); lsb_free_input_buffer(&b); return NULL; } @@ -62,6 +69,9 @@ static char* test_expand_buf() mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); mu_assert(b.size == BUFSIZ, "received: %" PRIuSIZE, b.size); + lsb_err_value ret = lsb_expand_input_buffer(NULL, 0); + mu_assert(LSB_ERR_UTIL_NULL == ret, "received: %s", lsb_err_string(ret)); + mu_assert(!lsb_expand_input_buffer(&b, 1024 * 9), "expand failed"); mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, @@ -82,7 +92,8 @@ static char* test_expand_failure() size_t size = 1024; lsb_input_buffer b; mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); - mu_assert_rv(1, lsb_expand_input_buffer(&b, size + LSB_MAX_HDR_SIZE + 1)); + lsb_err_value ret = lsb_expand_input_buffer(&b, size + LSB_MAX_HDR_SIZE + 1); + mu_assert(LSB_ERR_UTIL_FULL == ret, "received: %s", lsb_err_string(ret)); lsb_free_input_buffer(&b); return NULL; } diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index 5f6271a..0489657 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -29,10 +29,15 @@ static char* test_init_small_buf() { size_t size = 512; lsb_output_buffer b; + + lsb_err_value ret = lsb_init_output_buffer(NULL, size); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); mu_assert(b.size == size, "received: %" PRIuSIZE, b.size); mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); lsb_free_output_buffer(&b); + lsb_free_output_buffer(NULL); return NULL; } @@ -68,6 +73,9 @@ static char* test_expand_buf() mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + lsb_err_value ret = lsb_expand_output_buffer(NULL, 0); + mu_assert(LSB_ERR_UTIL_NULL == ret, "received: %s", lsb_err_string(ret)); + mu_assert(!lsb_expand_output_buffer(&b, 1024 * 9), "expand failed"); mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); @@ -107,12 +115,29 @@ static char* test_outputc() static char* test_outputf() { + lsb_err_value ret; lsb_output_buffer b; - mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + mu_assert(!lsb_init_output_buffer(&b, 10), "init failed"); lsb_outputf(&b, "%s", "foo"); mu_assert(strcmp("foo", b.buf) == 0, "received: %s", b.buf); lsb_outputf(&b, " %s", "bar"); mu_assert(strcmp("foo bar", b.buf) == 0, "received: %s", b.buf); + ret = lsb_outputf(&b, " %s", "exceed the buffer"); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received: %s", lsb_err_string(ret)); + ret = lsb_outputf(NULL, "%s", "bar"); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + ret = lsb_outputf(&b, NULL, "bar"); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + lsb_free_output_buffer(&b); + + size_t len = 2000; + mu_assert(!lsb_init_output_buffer(&b, len), "init failed"); + for (size_t i = 0; i < len - 1; ++i) { + ret = lsb_outputf(&b, "%c", 'a'); + mu_assert(!ret, "received: %s", ret); + } + ret = lsb_outputf(&b, "%c", 'a'); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received: %s", lsb_err_string(ret)); lsb_free_output_buffer(&b); return NULL; } diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c index b89fb4f..f7a9e2c 100644 --- a/src/util/test/test_protobuf.c +++ b/src/util/test/test_protobuf.c @@ -9,6 +9,7 @@ #include #include +#include "luasandbox/error.h" #include "luasandbox/util/heka_message.h" #include "luasandbox/util/protobuf.h" @@ -28,6 +29,12 @@ static char* test_lsb_pb_read_key() mu_assert(p = input + 1, "received: %p", p); mu_assert(tag == 1, "received: %d", tag); mu_assert(wt == 3, "received: %d", wt); + p = lsb_pb_read_key(NULL, &tag, &wt); + mu_assert(!p, "not null"); + p = lsb_pb_read_key(input, NULL, &wt); + mu_assert(!p, "not null"); + p = lsb_pb_read_key(input, &tag, NULL); + mu_assert(!p, "not null"); return NULL; } @@ -37,12 +44,13 @@ static char* test_lsb_pb_write_key() lsb_output_buffer ob; lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); - mu_assert_rv(0, lsb_pb_write_key(&ob, 1, 3)); + lsb_err_value ret = lsb_pb_write_key(&ob, 1, 3); + mu_assert(!ret, "received %s", ret); mu_assert(memcmp(ob.buf, "\x0b", 1) == 0, "received: %02hhx", ob.buf[0]); ob.pos = ob.maxsize; - mu_assert_rv(1, lsb_pb_write_key(&ob, 1, 3)); - + ret = lsb_pb_write_key(&ob, 1, 3); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); lsb_free_output_buffer(&ob); return NULL; } @@ -63,6 +71,7 @@ static char* test_varint() { 5000000000LL, "\x80\xe4\x97\xd0\x12" } }; + lsb_err_value ret; long long vi; const char *p; for (unsigned i = 0; i < sizeof tests / sizeof tests[0]; ++i){ @@ -76,7 +85,8 @@ static char* test_varint() lsb_output_buffer ob; lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); - mu_assert_rv(0, lsb_pb_write_varint(&ob, v)); + ret = lsb_pb_write_varint(&ob, v); + mu_assert(!ret, "received %s", ret); if (strncmp(s, ob.buf, len) != 0) { char expected[LSB_MAX_VARINT_BYTES + 1] = { 0 }; char received[LSB_MAX_VARINT_BYTES + 1] = { 0 }; @@ -100,7 +110,8 @@ static char* test_varint() lsb_output_buffer ob; lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); ob.pos = ob.maxsize; - mu_assert_rv(1, lsb_pb_write_varint(&ob, 1)); + ret = lsb_pb_write_varint(&ob, 1); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); lsb_free_output_buffer(&ob); return NULL; @@ -111,13 +122,14 @@ static char* test_lsb_pb_write_bool() { lsb_output_buffer ob; lsb_init_output_buffer(&ob, 1); - mu_assert_rv(0, lsb_pb_write_bool(&ob, 7)); + lsb_err_value ret = lsb_pb_write_bool(&ob, 7); + mu_assert(!ret, "received %s", ret); mu_assert(ob.buf[0] == 1, "received: %02hhx", ob.buf[0]); mu_assert(lsb_pb_write_bool(&ob, 7), "buffer should be full"); - ob.err = 0; ob.pos = 0; - mu_assert_rv(0, lsb_pb_write_bool(&ob, 0)); + ret = lsb_pb_write_bool(&ob, 0); + mu_assert(!ret, "received %s", ret); mu_assert(ob.buf[0] == 0, "received: %02hhx", ob.buf[0]); lsb_free_output_buffer(&ob); @@ -130,10 +142,12 @@ static char* test_lsb_pb_write_double() double d = 7.13; lsb_output_buffer ob; lsb_init_output_buffer(&ob, sizeof d); - mu_assert_rv(0, lsb_pb_write_double(&ob, d)); + lsb_err_value ret = lsb_pb_write_double(&ob, d); + mu_assert(!ret, "received %s", ret); mu_assert(memcmp(ob.buf, &d, sizeof d) == 0, "received: %g", *((double *)ob.buf)); - mu_assert_rv(1, lsb_pb_write_double(&ob, d)); + ret = lsb_pb_write_double(&ob, d); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); lsb_free_output_buffer(&ob); return NULL; } @@ -148,18 +162,22 @@ static char* test_lsb_pb_write_string() // failure writing the key ob.pos = ob.maxsize; - mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); + lsb_err_value ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); // failure writing the len ob.pos = ob.maxsize - 1; - mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); + ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); // failure writing the string ob.pos = ob.maxsize - 3; - mu_assert_rv(1, lsb_pb_write_string(&ob, 1, foo, len)); + ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); - lsb_clear_output_buffer(&ob); - mu_assert_rv(0, lsb_pb_write_string(&ob, 1, foo, len)); + ob.pos = 0; + ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(!ret, "received %s", ret); mu_assert(ob.pos = len + 2, "received: %" PRIuSIZE, ob.pos); mu_assert(memcmp("\x0a\x03" "foo", ob.buf, 5) == 0, "received: " "%02hhx%02hhx%02hhx%02hhx2%hhx", ob.buf[0], ob.buf[1], ob.buf[2], @@ -178,23 +196,26 @@ static char* test_lsb_pb_update_field_length() ob.buf[0] = 'a'; ob.buf[2] = 'b'; ob.pos = 3; - mu_assert_rv(0, lsb_pb_update_field_length(&ob, 1)); + lsb_err_value ret = lsb_pb_update_field_length(&ob, 1); + mu_assert(!ret, "received %s", ret); mu_assert(ob.buf[1] == 1, "received: %d", ob.buf[1]); mu_assert(ob.buf[2] == 'b', "received: %02hhx", ob.buf[2]); mu_assert(ob.pos == 3, "received: %" PRIuSIZE, ob.pos); // buffer full ob.pos = 1024; - mu_assert_rv(1, lsb_pb_update_field_length(&ob, 1)); + ret = lsb_pb_update_field_length(&ob, 1); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); // position is out of bounds ob.pos = 512; - mu_assert_rv(1, lsb_pb_update_field_length(&ob, 512)); + ret = lsb_pb_update_field_length(&ob, 512); + mu_assert(ret == LSB_ERR_UTIL_PRANGE, "received %s", lsb_err_string(ret)); - ob.err = 0; ob.buf[301] = 'x'; ob.pos = 302; - mu_assert_rv(0, lsb_pb_update_field_length(&ob, 1)); + ret = lsb_pb_update_field_length(&ob, 1); + mu_assert(!ret, "received %s", ret); mu_assert((unsigned char)ob.buf[1] == 0xac, "received: %02hhx", ob.buf[1]); mu_assert(ob.buf[2] == 0x02, "received: %02hhx", ob.buf[2]); mu_assert(ob.buf[3] == 'b', "received: %02hhx", ob.buf[3]); diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c index 6b74bf0..2b528cc 100644 --- a/src/util/test/test_string_matcher.c +++ b/src/util/test/test_string_matcher.c @@ -48,9 +48,14 @@ static char* test_success() , "[" , "%[" , "]" , "%]" , "]" , "]" - , "(" , "%(" - , ")" , "%)" - , ")" , ")" + , "(" , "%(" + , ")" , "%)" + , ")" , ")" + , "abc" , "^%a-$" + , "a$c" , "a$c" + , "a" , "%a?" + , "abc" , "%a*" + , "123ab" , "%f[%a]" , NULL }; diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index 2a664c2..6ec16db 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -7,9 +7,11 @@ /** @brief lsb_util unit tests @file */ #include +#include #include #include "../../test/mu_test.h" +#include "luasandbox/error.h" #include "luasandbox/util/util.h" static char* test_stub() @@ -34,6 +36,20 @@ static char* test_lsb_lp2() } +static char* test_lsb_read_file() +{ + char *s = lsb_read_file("Makefile"); + mu_assert(s, "read file failed"); + free(s); + + s = lsb_read_file("_foo_bar_"); + free(s); // if it succeeded don't leak + mu_assert(!s, "read file succeeded"); + return NULL; +} + + + static char* benchmark_lsb_get_time() { int iter = 1000000; @@ -66,6 +82,7 @@ static char* all_tests() { mu_run_test(test_stub); mu_run_test(test_lsb_lp2); + mu_run_test(test_lsb_read_file); mu_run_test(benchmark_lsb_get_time); return NULL; diff --git a/src/util/util.c b/src/util/util.c index 7082a4a..7e26b75 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -25,6 +25,11 @@ #include "luasandbox/util/util.h" +lsb_err_id LSB_ERR_UTIL_NULL = "pointer is NULL"; +lsb_err_id LSB_ERR_UTIL_OOM = "memory allocation failed"; +lsb_err_id LSB_ERR_UTIL_FULL = "buffer full"; +lsb_err_id LSB_ERR_UTIL_PRANGE = "parameter out of range"; + size_t lsb_lp2(unsigned long long x) { if (x == 0) return 0; From a91243f14966c51285970ac2506c5c71181dbd93 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 4 Feb 2016 08:36:37 -0800 Subject: [PATCH 103/235] Add the missing error.h header --- include/luasandbox/error.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 include/luasandbox/error.h diff --git a/include/luasandbox/error.h b/include/luasandbox/error.h new file mode 100644 index 0000000..fc4501b --- /dev/null +++ b/include/luasandbox/error.h @@ -0,0 +1,23 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Error handling and logging @file */ + +#ifndef luasandbox_error_h_ +#define luasandbox_error_h_ + +// See Identify your Errors better with char[] +// http://accu.org/index.php/journals/2184 +typedef char const lsb_err_id[]; +typedef char const *lsb_err_value; +#define lsb_err_string(s) s ? s : "" + +typedef void (*lsb_logger)(const char *component, + int level, + const char *fmt, + ...); + +#endif From 026897e53b436f33758a6119f169cfdb1c60283e Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Thu, 4 Feb 2016 16:52:27 -0800 Subject: [PATCH 104/235] Correct compilation errors on Windows --- CMakeLists.txt | 4 +++- include/luasandbox.h | 8 ++++---- include/luasandbox/error.h | 4 ++-- include/luasandbox/heka/sandbox.h | 4 ++-- include/luasandbox/util/util.h | 8 ++++---- src/CMakeLists.txt | 2 -- src/heka/CMakeLists.txt | 2 +- src/heka/message.c | 2 +- src/heka/sandbox.c | 5 ++++- src/luasandbox.c | 7 +++---- src/util/CMakeLists.txt | 2 +- src/util/output_buffer.c | 8 ++++++-- src/util/util.c | 4 ++-- 13 files changed, 33 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe56424..f127c4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,7 +42,6 @@ set(LIB_INSTALL_DIR lib/) set(INCLUDE_INSTALL_DIR include/) include_directories("${EP_BASE}/include") -include_directories("${CMAKE_SOURCE_DIR}/include") install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${LIB_INSTALL_DIR}${PROJECT_NAME}/modules COMPONENT core) install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION share/doc/${PROJECT_NAME} COMPONENT core) @@ -63,3 +62,6 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake ${CMAKE_CURRENT DESTINATION ${INSTALL_CMAKE_DIR} COMPONENT core) add_subdirectory(src) +add_custom_target(copy_includes +COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/include) +add_dependencies(luasandboxutil copy_includes) diff --git a/include/luasandbox.h b/include/luasandbox.h index 94cfe80..fe14687 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -60,15 +60,15 @@ typedef enum { typedef struct lsb_lua_sandbox lsb_lua_sandbox; -LSB_EXPORT extern lsb_err_id LSB_ERR_INIT; -LSB_EXPORT extern lsb_err_id LSB_ERR_LUA; -LSB_EXPORT extern lsb_err_id LSB_ERR_TERMINATED; - #ifdef __cplusplus extern "C" { #endif +LSB_EXPORT extern lsb_err_id LSB_ERR_INIT; +LSB_EXPORT extern lsb_err_id LSB_ERR_LUA; +LSB_EXPORT extern lsb_err_id LSB_ERR_TERMINATED; + /** * Allocates and initializes the structure around the Lua sandbox allowing * full specification of the sandbox configuration using a Lua configuration diff --git a/include/luasandbox/error.h b/include/luasandbox/error.h index fc4501b..39b1a60 100644 --- a/include/luasandbox/error.h +++ b/include/luasandbox/error.h @@ -11,8 +11,8 @@ // See Identify your Errors better with char[] // http://accu.org/index.php/journals/2184 -typedef char const lsb_err_id[]; -typedef char const *lsb_err_value; +typedef const char lsb_err_id[]; +typedef const char *lsb_err_value; #define lsb_err_string(s) s ? s : "" typedef void (*lsb_logger)(const char *component, diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index c90205e..b287dca 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -33,8 +33,6 @@ #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" -LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; - enum lsb_heka_pm_rv { LSB_HEKA_PM_SENT = 0, LSB_HEKA_PM_FAIL = -1, @@ -66,6 +64,8 @@ extern "C" { #endif +LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; + /** * inject_message callback function provided by the host. Only one (or neither) * of the checkpoint values will be set in a call. Numeric checkpoints can have diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index d040570..e17b62e 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -27,15 +27,15 @@ #endif #endif +#ifdef __cplusplus +extern "C" { +#endif + LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_NULL; LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_OOM; LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_FULL; LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_PRANGE; -#ifdef __cplusplus -extern "C" { -#endif - /** * Hacker's Delight - Henry S. Warren, Jr. page 48 * diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bcaf288..f81ef80 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,9 +11,7 @@ luasandbox_serialize.c set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) -add_custom_target(copy_includes COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/include) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) -add_dependencies(luasandbox copy_includes) add_dependencies(lua_bloom_filter luasandbox) add_dependencies(lua_circular_buffer luasandbox) diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index 983d8a5..522cea8 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -11,7 +11,7 @@ stream_reader.c add_library(luasandboxheka SHARED ${HEKA_SRC}) set_target_properties(luasandboxheka PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) -target_compile_definitions(luasandboxheka PUBLIC -Dluasandboxheka_EXPORTS) +target_compile_definitions(luasandboxheka PRIVATE -Dluasandboxheka_EXPORTS) target_link_libraries(luasandboxheka luasandbox) if(WIN32) target_link_libraries(luasandboxheka ws2_32) diff --git a/src/heka/message.c b/src/heka/message.c index 7f769f5..dd743a2 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -330,7 +330,7 @@ static lsb_err_value encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, const char *representation, int value_type) { - lsb_err_value ret; + lsb_err_value ret = NULL; int first = (int)lua_objlen(lsb->lua, -1); int multiple = first > 1 ? first : 0; size_t len_pos = 0; diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index c2629c0..6aec594 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -23,6 +23,7 @@ #ifdef _WIN32 #include +#define snprintf _snprintf #else #include #endif @@ -499,7 +500,9 @@ int lsb_heka_pm_input(lsb_heka_sandbox *hsb, const char *cp_string, bool profile) { - if (!hsb || hsb->type != 'i') return 1; + if (!hsb || hsb->type != 'i') { + return 1; + } lsb_err_value ret = lsb_pcall_setup(hsb->lsb, pm_func_name); if (ret) { diff --git a/src/luasandbox.c b/src/luasandbox.c index 8fad3df..1abe7b5 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -23,10 +23,9 @@ #include "luasandbox_impl.h" #include "luasandbox_serialize.h" - -lsb_err_id LSB_ERR_INIT = "already initialized"; -lsb_err_id LSB_ERR_LUA = "lua error"; // use lsb_get_error for more detail -lsb_err_id LSB_ERR_TERMINATED = "sandbox already terminated"; +lsb_err_id LSB_ERR_INIT = "already initialized"; +lsb_err_id LSB_ERR_LUA = "lua error"; // use lsb_get_error for details +lsb_err_id LSB_ERR_TERMINATED = "sandbox already terminated"; static jmp_buf g_jbuf; diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 518f63b..c04b7a1 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -15,7 +15,7 @@ util.c add_library(luasandboxutil SHARED ${UTIL_SRC}) set_target_properties(luasandboxutil PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) -target_compile_definitions(luasandboxutil PUBLIC -Dluasandboxutil_EXPORTS) +target_compile_definitions(luasandboxutil PRIVATE -Dluasandboxutil_EXPORTS) if(LIBM_LIBRARY) target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index bfd6e66..54f56a8 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -77,7 +77,9 @@ lsb_err_value lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) lsb_err_value lsb_outputc(lsb_output_buffer *b, char ch) { - if (!b) return LSB_ERR_UTIL_NULL; + if (!b) { + return LSB_ERR_UTIL_NULL; + } lsb_err_value ret = lsb_expand_output_buffer(b, 2); if (ret) return ret; @@ -143,7 +145,9 @@ lsb_err_value lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) lsb_err_value lsb_outputs(lsb_output_buffer *b, const char *str, size_t len) { - if (!b) return LSB_ERR_UTIL_NULL; + if (!b) { + return LSB_ERR_UTIL_NULL; + } lsb_err_value ret = lsb_expand_output_buffer(b, len + 1); if (ret) return ret; diff --git a/src/util/util.c b/src/util/util.c index 7e26b75..c97cc30 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -6,6 +6,8 @@ /** General purpose utility functions @file */ +#include "luasandbox/util/util.h" + #include #include #include @@ -23,8 +25,6 @@ #include #endif -#include "luasandbox/util/util.h" - lsb_err_id LSB_ERR_UTIL_NULL = "pointer is NULL"; lsb_err_id LSB_ERR_UTIL_OOM = "memory allocation failed"; lsb_err_id LSB_ERR_UTIL_FULL = "buffer full"; From da2dc21b9dd856279b626e5dd97606199b0001c4 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 19 Feb 2016 14:50:17 -0800 Subject: [PATCH 105/235] Added a JSON module for JSON-schema validation --- CMakeLists.txt | 10 +- covfn.txt | 337 ++++++------- docs/heka/heka_json.md | 202 ++++++++ docs/heka/index.md | 16 + include/luasandbox.h | 3 +- include/luasandbox/heka/sandbox.h | 5 +- include/luasandbox/util/util.h | 15 + include/luasandbox_output.h | 3 +- include/luasandbox_serialize.h | 3 +- src/heka/CMakeLists.txt | 1 + src/heka/message.c | 14 +- src/heka/rapidjson.cpp | 676 +++++++++++++++++++++++++++ src/heka/rapidjson_impl.h | 30 ++ src/heka/sandbox.c | 16 + src/heka/stream_reader.c | 9 +- src/heka/stream_reader_impl.h | 3 + src/heka/test/lua/encode_message.lua | 9 + src/heka/test/lua/heka_json.lua | 298 ++++++++++++ src/heka/test/lua/iim.lua | 18 +- src/heka/test/test_sandbox.c | 18 +- src/util/CMakeLists.txt | 4 + src/util/util.c | 69 ++- 22 files changed, 1582 insertions(+), 177 deletions(-) create mode 100644 docs/heka/heka_json.md create mode 100644 src/heka/rapidjson.cpp create mode 100644 src/heka/rapidjson_impl.h create mode 100644 src/heka/test/lua/heka_json.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index f127c4f..e91d730 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,10 +3,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox C) +project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 13) +set(CPACK_PACKAGE_VERSION_MINOR 14) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") @@ -24,6 +24,12 @@ endif() find_library(LIBM_LIBRARY m) +find_package(ZLIB) + +if (ZLIB_FOUND) + add_definitions(-DHAVE_ZLIB) +endif() + find_package(Git REQUIRED) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") diff --git a/covfn.txt b/covfn.txt index 986d544..d5d2e5f 100644 --- a/covfn.txt +++ b/covfn.txt @@ -1,156 +1,181 @@ -Function Source Line FnCov C/D Coverage ------------------------------ ------------------ ----- --------------------- -add_table_ref...void*,size_t) ...rialize.c 170 1 / 1 1 / 4 = 25% -restore_globa...lua_sandbox*) ...rialize.c 495 1 / 1 7 / 18 = 38% -lsb_create(vo...*,lsb_logger) ...sandbox.c 379 1 / 1 11 / 26 = 42% -serialize_dat...tput_buffer*) ...rialize.c 229 1 / 1 8 / 18 = 44% -hsr_new(lua_State*) ..._reader.c 21 1 / 1 4 / 8 = 50% -lsb_create_me...,const char*) ...matcher.c 443 1 / 1 3 / 6 = 50% -lsb_outputc(l...buffer*,char) ..._buffer.c 78 1 / 1 2 / 4 = 50% -instruction_m...*,lua_Debug*) ...sandbox.c 115 1 / 1 1 / 2 = 50% -read_config(lua_State*) ...sandbox.c 139 1 / 1 1 / 2 = 50% -set_tz() ...sandbox.c 336 1 / 1 1 / 2 = 50% -heka_encode_m...sandbox*,int) ...message.c 871 1 / 1 14 / 24 = 58% -set_random_seed() ...sandbox.c 351 1 / 1 7 / 12 = 58% -lsb_read_file(const char*) ...il/util.c 47 1 / 1 6 / 10 = 60% -lsb_heka_crea...eka_im_input) ...sandbox.c 341 1 / 1 11 / 18 = 61% -lsb_heka_crea..._im_analysis) ...sandbox.c 529 1 / 1 11 / 18 = 61% -lsb_heka_crea...e_checkpoint) ...sandbox.c 612 1 / 1 11 / 18 = 61% -min_expand(Ma...,const char*) ...matcher.c 171 1 / 1 5 / 8 = 62% -preserve_glob...lua_sandbox*) ...rialize.c 366 1 / 1 27 / 42 = 64% -lsb_init_heka...message*,int) ...message.c 417 1 / 1 4 / 6 = 66% -process_field...,const char*) ...message.c 93 1 / 1 55 / 82 = 67% -encode_field_...st char*,int) ...message.c 330 1 / 1 19 / 28 = 67% -lsb_heka_pm_i... char*,_Bool) ...sandbox.c 497 1 / 1 11 / 16 = 68% -read_string(l...,const char*) ...message.c 52 1 / 1 7 / 10 = 70% -set_restricti...eka_sandbox*) ...sandbox.c 251 1 / 1 19 / 27 = 70% -check_int(lua...st char*,int) ...sandbox.c 192 1 / 1 5 / 7 = 71% -encode_field_...st char*,int) ...message.c 403 1 / 1 78 / 105 = 74% -process_messa...e*,int,_Bool) ...sandbox.c 412 1 / 1 29 / 39 = 74% -encode_fields...st char*,int) ...message.c 608 1 / 1 27 / 36 = 75% -serialize_kvp...data*,size_t) ...rialize.c 286 1 / 1 18 / 24 = 75% -decode_header...ize_t,size_t) ...message.c 19 1 / 1 12 / 16 = 75% -read_message(lua_State*) ...sandbox.c 35 1 / 1 6 / 8 = 75% -hsr_decode_me...e(lua_State*) ..._reader.c 77 1 / 1 6 / 8 = 75% -process_varin...,const char*) ...message.c 72 1 / 1 3 / 4 = 75% -encode_int(ls...st char*,int) ...message.c 287 1 / 1 3 / 4 = 75% -load_op_node(...,match_node*) ...matcher.c 321 1 / 1 3 / 4 = 75% -lsb_destroy_m...tch_builder*) ...matcher.c 480 1 / 1 3 / 4 = 75% -output(lua_State*) ...sandbox.c 123 1 / 1 3 / 4 = 75% -lsb_stop_sand...lua_sandbox*) ...sandbox.c 561 1 / 1 3 / 4 = 75% -lsb_get_outpu...box*,size_t*) ..._output.c 110 1 / 1 3 / 4 = 75% -lsb_clear_hek...eka_message*) ...message.c 431 1 / 1 3 / 4 = 75% -lsb_outputs(l...char*,size_t) ..._buffer.c 144 1 / 1 3 / 4 = 75% -lsb_outputfd(...ffer*,double) ..._buffer.c 173 1 / 1 31 / 40 = 77% -lsb_init(lsb_...,const char*) ...sandbox.c 461 1 / 1 28 / 36 = 77% -lsb_pcall_set...,const char*) ...sandbox.c 676 1 / 1 14 / 18 = 77% -hsr_find_message(lua_State*) ..._reader.c 106 1 / 1 25 / 32 = 78% -read_string(i...onst_string*) ...message.c 39 1 / 1 8 / 10 = 80% -read_string_v..._read_value*) ...message.c 57 1 / 1 8 / 10 = 80% -read_integer_..._read_value*) ...message.c 78 1 / 1 8 / 10 = 80% -check_string(...,const char*) ...sandbox.c 170 1 / 1 4 / 5 = 80% -lsb_heka_time...time_t,_Bool) ...sandbox.c 725 1 / 1 13 / 16 = 81% -matchbalance(...,const char*) ...matcher.c 137 1 / 1 13 / 16 = 81% -lsb_serialize...void*,size_t) ...rialize.c 547 1 / 1 9 / 11 = 81% -lsb_outputf(l...st char*,...) ..._buffer.c 90 1 / 1 23 / 28 = 82% -lsb_create_me...,const char*) ...matcher.c 493 1 / 1 24 / 29 = 82% -heka_decode_m...e(lua_State*) ...message.c 683 1 / 1 54 / 65 = 83% -heka_read_mes...eka_message*) ...message.c 912 1 / 1 45 / 54 = 83% -set_missing_h...eka_sandbox*) ...message.c 32 1 / 1 10 / 12 = 83% -lsb_heka_pm_a...ssage*,_Bool) ...sandbox.c 593 1 / 1 10 / 12 = 83% -ignore_value_...a_CFunction*) ...rialize.c 114 1 / 1 10 / 12 = 83% -inject_messag...s(lua_State*) ...sandbox.c 130 1 / 1 5 / 6 = 83% -lsb_destroy(lsb_lua_sandbox*) ...sandbox.c 574 1 / 1 5 / 6 = 83% -lsb_pb_output...ed long long) ...rotobuf.c 56 1 / 1 5 / 6 = 83% -lsb_output(ls...,int,int,int) ..._output.c 42 1 / 1 27 / 32 = 84% -heka_encode_m...e(lua_State*) ...message.c 810 1 / 1 11 / 13 = 84% -match_class(int,int) ...matcher.c 68 1 / 1 11 / 13 = 84% -match(MatchSt...,const char*) ...matcher.c 183 1 / 1 39 / 46 = 84% -lsb_pb_read_v...*,long long*) ...rotobuf.c 36 1 / 1 17 / 20 = 85% -inject_messag...t(lua_State*) ...sandbox.c 51 1 / 1 18 / 21 = 85% -lsb_heka_pm_o...,void*,_Bool) ...sandbox.c 699 1 / 1 12 / 14 = 85% -lsb_expand_in...ffer*,size_t) ..._buffer.c 49 1 / 1 12 / 14 = 85% -numeric_test(...node*,double) ...matcher.c 213 1 / 1 6 / 7 = 85% -load_sandbox_...*,lsb_logger) ...sandbox.c 223 1 / 1 19 / 22 = 86% -matchbracketc...,const char*) ...matcher.c 99 1 / 1 19 / 22 = 86% -eval_tree(mat...eka_message*) ...matcher.c 297 1 / 1 14 / 16 = 87% -update_checkpoint(lua_State*) ...sandbox.c 218 1 / 1 7 / 8 = 87% -lsb_init_inpu...ffer*,size_t) ..._buffer.c 20 1 / 1 7 / 8 = 87% -lsb_outputd(l...ffer*,double) ..._buffer.c 156 1 / 1 7 / 8 = 87% -process_field...,const char*) ...message.c 121 1 / 1 50 / 57 = 87% -copy_table(lu...*,lsb_logger) ...sandbox.c 263 1 / 1 22 / 25 = 88% -load_expressi...,match_node*) ...matcher.c 335 1 / 1 42 / 47 = 89% -lsb_decode_he...t,lsb_logger) ...message.c 201 1 / 1 51 / 57 = 89% -inject_payload(lua_State*) ...sandbox.c 160 1 / 1 18 / 20 = 90% -lsb_add_funct...,const char*) ...sandbox.c 664 1 / 1 9 / 10 = 90% -lsb_init_outp...ffer*,size_t) ..._buffer.c 27 1 / 1 9 / 10 = 90% -lsb_find_heka...*,lsb_logger) ...message.c 319 1 / 1 29 / 32 = 90% -eval_node(mat...eka_message*) ...matcher.c 235 1 / 1 31 / 34 = 91% -memory_manage...ize_t,size_t) ...sandbox.c 78 1 / 1 11 / 12 = 91% -lsb_read_heka..._read_value*) ...message.c 462 1 / 1 24 / 26 = 92% -classend(const char*) ...matcher.c 43 1 / 1 16 / 17 = 94% -lsb_expand_ou...ffer*,size_t) ..._buffer.c 52 1 / 1 17 / 18 = 94% -string_test(m...onst_string*) ...matcher.c 167 1 / 1 18 / 19 = 94% -encode_string...st char*,int) ...message.c 258 1 / 1 2 / 2 = 100% -encode_field_...tput_buffer*) ...message.c 381 1 / 1 4 / 4 = 100% -lsb_destroy_m...age_matcher*) ...matcher.c 559 1 / 1 6 / 6 = 100% -lsb_eval_mess...eka_message*) ...matcher.c 578 1 / 1 0 / 0 -lsb_heka_stop...eka_sandbox*) ...sandbox.c 675 1 / 1 0 / 0 -lsb_heka_term...,const char*) ...sandbox.c 681 1 / 1 0 / 0 -lsb_heka_dest...eka_sandbox*) ...sandbox.c 687 1 / 1 2 / 2 = 100% -lsb_heka_get_...eka_sandbox*) ...sandbox.c 766 1 / 1 2 / 2 = 100% -lsb_heka_get_...eka_sandbox*) ...sandbox.c 772 1 / 1 2 / 2 = 100% -lsb_heka_get_...eka_sandbox*) ...sandbox.c 778 1 / 1 2 / 2 = 100% -lsb_heka_is_r...eka_sandbox*) ...sandbox.c 799 1 / 1 4 / 4 = 100% -check_hsr(lua_State*,int) ..._reader.c 68 1 / 1 0 / 0 -hsr_read_message(lua_State*) ..._reader.c 196 1 / 1 6 / 6 = 100% -hsr_gc(lua_State*) ..._reader.c 208 1 / 1 0 / 0 -luaopen_heka_...r(lua_State*) ..._reader.c 230 1 / 1 0 / 0 -libsize(const luaL_Reg*) ...sandbox.c 44 1 / 1 2 / 2 = 100% -preload_modules(lua_State*) ...sandbox.c 51 1 / 1 2 / 2 = 100% -instruction_u...lua_sandbox*) ...sandbox.c 109 1 / 1 0 / 0 -unprotected_panic(lua_State*) ...sandbox.c 153 1 / 1 0 / 0 -get_usage_con...,const char*) ...sandbox.c 161 1 / 1 0 / 0 -stop_hook(lua...*,lua_Debug*) ...sandbox.c 553 1 / 1 0 / 0 -lsb_usage(lsb...b_usage_stat) ...sandbox.c 597 1 / 1 10 / 10 = 100% -lsb_get_error...lua_sandbox*) ...sandbox.c 621 1 / 1 2 / 2 = 100% -lsb_set_error...,const char*) ...sandbox.c 627 1 / 1 4 / 4 = 100% -lsb_get_lua(lsb_lua_sandbox*) ...sandbox.c 640 1 / 1 2 / 2 = 100% -lsb_get_lua_f...lua_sandbox*) ...sandbox.c 646 1 / 1 2 / 2 = 100% -lsb_get_paren...lua_sandbox*) ...sandbox.c 652 1 / 1 2 / 2 = 100% -lsb_get_state...lua_sandbox*) ...sandbox.c 658 1 / 1 2 / 2 = 100% -lsb_pcall_tea...lua_sandbox*) ...sandbox.c 700 1 / 1 4 / 4 = 100% -lsb_terminate...,const char*) ...sandbox.c 714 1 / 1 6 / 6 = 100% -lsb_add_outpu...ua_CFunction) ..._output.c 22 1 / 1 0 / 0 -lsb_get_outpu...a_State*,int) ..._output.c 30 1 / 1 0 / 0 -get_serialize...a_State*,int) ...rialize.c 92 1 / 1 0 / 0 -find_table_re...,const void*) ...rialize.c 149 1 / 1 4 / 4 = 100% -get_preservat...n(lua_State*) ...rialize.c 195 1 / 1 4 / 4 = 100% -serialize_tab...data*,size_t) ...rialize.c 214 1 / 1 6 / 6 = 100% -file_exists(const char*) ...rialize.c 484 1 / 1 2 / 2 = 100% -lsb_add_seria...ua_CFunction) ...rialize.c 539 1 / 1 0 / 0 -lsb_serialize...ffer*,double) ...rialize.c 574 1 / 1 6 / 6 = 100% -read_double_v..._read_value*) ...message.c 97 1 / 1 2 / 2 = 100% -process_varin...*,long long*) ...message.c 110 1 / 1 4 / 4 = 100% -lsb_free_heka...eka_message*) ...message.c 451 1 / 1 2 / 2 = 100% -lsb_write_hek...char*,size_t) ...message.c 507 1 / 1 22 / 22 = 100% -lsb_free_inpu...nput_buffer*) ..._buffer.c 36 1 / 1 2 / 2 = 100% -lsb_free_outp...tput_buffer*) ..._buffer.c 42 1 / 1 2 / 2 = 100% -lsb_pb_read_k...r*,int*,int*) ...rotobuf.c 15 1 / 1 8 / 8 = 100% -lsb_pb_write_...nsigned char) ...rotobuf.c 25 1 / 1 2 / 2 = 100% -lsb_pb_write_...ed long long) ...rotobuf.c 75 1 / 1 2 / 2 = 100% -lsb_pb_write_..._buffer*,int) ...rotobuf.c 85 1 / 1 4 / 4 = 100% -lsb_pb_write_...ffer*,double) ...rotobuf.c 99 1 / 1 2 / 2 = 100% -lsb_pb_write_...char*,size_t) ...rotobuf.c 114 1 / 1 6 / 6 = 100% -lsb_pb_update...ffer*,size_t) ...rotobuf.c 127 1 / 1 8 / 8 = 100% -lsb_init_runn...nning_stats*) ...g_stats.c 13 1 / 1 0 / 0 -lsb_update_ru...tats*,double) ...g_stats.c 21 1 / 1 4 / 4 = 100% -lsb_sd_runnin...nning_stats*) ...g_stats.c 37 1 / 1 2 / 2 = 100% -lsb_init_cons...onst_string*) .../string.c 11 1 / 1 0 / 0 -singlematch(i...,const char*) ...matcher.c 119 1 / 1 4 / 4 = 100% -max_expand(Ma...,const char*) ...matcher.c 156 1 / 1 10 / 10 = 100% -lsb_string_ma...,const char*) ...matcher.c 261 1 / 1 10 / 10 = 100% -lsb_lp2(unsigned long long) ...il/util.c 33 1 / 1 2 / 2 = 100% -lsb_get_time() ...il/util.c 73 1 / 1 0 / 0 ------------------------------ ------------------ ----- --------------------- -Total 100% 1567 / 1958 = 80% +Function Source Line FnCov C/D Coverage +-------------------------------------------------------------------------------------------------- ------------------------------ ----- --------------------- +add_table_ref(table_ref_array*,const void*,size_t) ...uasandbox_serialize.c 170 1 / 1 1 / 4 = 25% +restore_global_data(lsb_lua_sandbox*) ...uasandbox_serialize.c 495 1 / 1 7 / 18 = 38% +lsb_create(void*,const char*,const char*,lsb_logger) ../src/luasandbox.c 378 1 / 1 11 / 26 = 42% +serialize_data(lsb_lua_sandbox*,int,lsb_output_buffer*) ...uasandbox_serialize.c 229 1 / 1 8 / 18 = 44% +lsb_ungzip(const char*,size_t,size_t*) ../src/util/util.c 113 1 / 1 10 / 22 = 45% +hsr_new(lua_State*) .../heka/stream_reader.c 21 1 / 1 4 / 8 = 50% +lsb_create_message_match_builder(const char*,const char*) ...eka/message_matcher.c 443 1 / 1 3 / 6 = 50% +instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 114 1 / 1 1 / 2 = 50% +read_config(lua_State*) ../src/luasandbox.c 138 1 / 1 1 / 2 = 50% +set_tz() ../src/luasandbox.c 335 1 / 1 1 / 2 = 50% +heka_encode_message_table(lsb_lua_sandbox*,int) ../src/heka/message.c 881 1 / 1 14 / 24 = 58% +set_random_seed() ../src/luasandbox.c 350 1 / 1 7 / 12 = 58% +lsb_read_file(const char*) ../src/util/util.c 51 1 / 1 6 / 10 = 60% +lsb_heka_create_input(void*,const char*,const char*,const char*,lsb_logger,lsb_heka_im_input) ../src/heka/sandbox.c 343 1 / 1 11 / 18 = 61% +lsb_heka_create_analysis(void*,const char*,const...r*,const char*,lsb_logger,lsb_heka_im_analysis) ../src/heka/sandbox.c 539 1 / 1 11 / 18 = 61% +lsb_heka_create_output(void*,const char*,const c...st char*,lsb_logger,lsb_heka_update_checkpoint) ../src/heka/sandbox.c 622 1 / 1 11 / 18 = 61% +hj_parse(lua_State*) ...rc/heka/rapidjson.cpp 126 1 / 1 5 / 8 = 62% +hj_remove(lua_State*) ...rc/heka/rapidjson.cpp 578 1 / 1 5 / 8 = 62% +min_expand(MatchState*,const char*,const char*,const char*) ...util/string_matcher.c 171 1 / 1 5 / 8 = 62% +preserve_global_data(lsb_lua_sandbox*) ...uasandbox_serialize.c 366 1 / 1 27 / 42 = 64% +output_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 559 1 / 1 8 / 12 = 66% +hj_iter(lua_State*) ...rc/heka/rapidjson.cpp 388 1 / 1 6 / 9 = 66% +lsb_init_heka_message(lsb_heka_message*,int) ...c/util/heka_message.c 417 1 / 1 4 / 6 = 66% +process_fields(lua_State*,const char*,const char*) ../src/heka/message.c 93 1 / 1 55 / 82 = 67% +encode_field_array(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 330 1 / 1 19 / 28 = 67% +lsb_heka_pm_input(lsb_heka_sandbox*,double,const char*,_Bool) ../src/heka/sandbox.c 505 1 / 1 11 / 16 = 68% +read_string(lua_State*,int,const char*,const char*) ../src/heka/message.c 52 1 / 1 7 / 10 = 70% +set_restrictions(lua_State*,lsb_heka_sandbox*) ../src/heka/sandbox.c 253 1 / 1 19 / 27 = 70% +check_int(lua_State*,int,const char*,int) ../src/luasandbox.c 191 1 / 1 5 / 7 = 71% +hj_parse_message(lua_State*) ...rc/heka/rapidjson.cpp 454 1 / 1 25 / 34 = 73% +process_message(lsb_heka_sandbox*,lsb_heka_message*,lua_State*,int,_Bool) ../src/heka/sandbox.c 420 1 / 1 29 / 39 = 74% +encode_fields(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 618 1 / 1 27 / 36 = 75% +serialize_kvp(lsb_lua_sandbox*,serialization_data*,size_t) ...uasandbox_serialize.c 286 1 / 1 18 / 24 = 75% +decode_header(char*,size_t,size_t) ...c/util/heka_message.c 19 1 / 1 12 / 16 = 75% +read_message(lua_State*) ../src/heka/sandbox.c 37 1 / 1 6 / 8 = 75% +hsr_decode_message(lua_State*) .../heka/stream_reader.c 77 1 / 1 6 / 8 = 75% +process_varint(lua_State*,const char*,int,int,const char*,const char*) ../src/heka/message.c 72 1 / 1 3 / 4 = 75% +encode_int(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 287 1 / 1 3 / 4 = 75% +load_op_node(lua_State*,match_node*) ...eka/message_matcher.c 321 1 / 1 3 / 4 = 75% +lsb_destroy_message_match_builder(lsb_message_match_builder*) ...eka/message_matcher.c 480 1 / 1 3 / 4 = 75% +output(lua_State*) ../src/luasandbox.c 122 1 / 1 3 / 4 = 75% +lsb_stop_sandbox(lsb_lua_sandbox*) ../src/luasandbox.c 560 1 / 1 3 / 4 = 75% +lsb_get_output(lsb_lua_sandbox*,size_t*) ...c/luasandbox_output.c 110 1 / 1 3 / 4 = 75% +lsb_clear_heka_message(lsb_heka_message*) ...c/util/heka_message.c 431 1 / 1 3 / 4 = 75% +lsb_outputc(lsb_output_buffer*,char) .../util/output_buffer.c 78 1 / 1 3 / 4 = 75% +lsb_outputs(lsb_output_buffer*,const char*,size_t) .../util/output_buffer.c 146 1 / 1 3 / 4 = 75% +encode_field_value(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 403 1 / 1 84 / 110 = 76% +lsb_outputfd(lsb_output_buffer*,double) .../util/output_buffer.c 177 1 / 1 31 / 40 = 77% +lsb_init(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 460 1 / 1 28 / 36 = 77% +lsb_pcall_setup(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 675 1 / 1 14 / 18 = 77% +hsr_find_message(lua_State*) .../heka/stream_reader.c 106 1 / 1 25 / 32 = 78% +read_string(int,const char*,const char*,lsb_const_string*) ...c/util/heka_message.c 39 1 / 1 8 / 10 = 80% +read_string_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 57 1 / 1 8 / 10 = 80% +read_integer_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 78 1 / 1 8 / 10 = 80% +check_string(lua_State*,int,const char*,const char*) ../src/luasandbox.c 169 1 / 1 4 / 5 = 80% +lsb_heka_timer_event(lsb_heka_sandbox*,time_t,_Bool) ../src/heka/sandbox.c 744 1 / 1 13 / 16 = 81% +matchbalance(MatchState*,const char*,const char*) ...util/string_matcher.c 137 1 / 1 13 / 16 = 81% +lsb_serialize_binary(lsb_output_buffer*,const void*,size_t) ...uasandbox_serialize.c 547 1 / 1 9 / 11 = 81% +lsb_outputf(lsb_output_buffer*,const char*,...) .../util/output_buffer.c 92 1 / 1 23 / 28 = 82% +lsb_create_message_matcher(const lsb_message_match_builder*,const char*) ...eka/message_matcher.c 493 1 / 1 24 / 29 = 82% +heka_decode_message(lua_State*) ../src/heka/message.c 693 1 / 1 54 / 65 = 83% +heka_read_message(lua_State*,lsb_heka_message*) ../src/heka/message.c 922 1 / 1 45 / 54 = 83% +set_missing_headers(lua_State*,int,lsb_heka_sandbox*) ../src/heka/message.c 32 1 / 1 10 / 12 = 83% +lsb_heka_pm_analysis(lsb_heka_sandbox*,lsb_heka_message*,_Bool) ../src/heka/sandbox.c 603 1 / 1 10 / 12 = 83% +ignore_value_type(lsb_lua_sandbox*,serialization_data*,int,lua_CFunction*) ...uasandbox_serialize.c 114 1 / 1 10 / 12 = 83% +hj_parse_schema(lua_State*) ...rc/heka/rapidjson.cpp 100 1 / 1 5 / 6 = 83% +inject_message_analysis(lua_State*) ../src/heka/sandbox.c 132 1 / 1 5 / 6 = 83% +lsb_destroy(lsb_lua_sandbox*) ../src/luasandbox.c 573 1 / 1 5 / 6 = 83% +lsb_pb_output_varint(char*,unsigned long long) ../src/util/protobuf.c 56 1 / 1 5 / 6 = 83% +lsb_output(lsb_lua_sandbox*,int,int,int) ...c/luasandbox_output.c 42 1 / 1 27 / 32 = 84% +heka_encode_message(lua_State*) ../src/heka/message.c 820 1 / 1 11 / 13 = 84% +match_class(int,int) ...util/string_matcher.c 68 1 / 1 11 / 13 = 84% +match(MatchState*,const char*,const char*) ...util/string_matcher.c 183 1 / 1 39 / 46 = 84% +lsb_pb_read_varint(const char*,const char*,long long*) ../src/util/protobuf.c 36 1 / 1 17 / 20 = 85% +inject_message_input(lua_State*) ../src/heka/sandbox.c 53 1 / 1 18 / 21 = 85% +lsb_heka_pm_output(lsb_heka_sandbox*,lsb_heka_message*,void*,_Bool) ../src/heka/sandbox.c 718 1 / 1 12 / 14 = 85% +lsb_expand_input_buffer(lsb_input_buffer*,size_t) ...c/util/input_buffer.c 49 1 / 1 12 / 14 = 85% +numeric_test(match_node*,double) ...eka/message_matcher.c 213 1 / 1 6 / 7 = 85% +load_sandbox_config(const char*,lsb_logger) ../src/luasandbox.c 222 1 / 1 19 / 22 = 86% +matchbracketclass(int,const char*,const char*) ...util/string_matcher.c 99 1 / 1 19 / 22 = 86% +eval_tree(match_node*,lsb_heka_message*) ...eka/message_matcher.c 297 1 / 1 14 / 16 = 87% +hj_type(lua_State*) ...rc/heka/rapidjson.cpp 237 1 / 1 7 / 8 = 87% +hj_size(lua_State*) ...rc/heka/rapidjson.cpp 266 1 / 1 7 / 8 = 87% +update_checkpoint(lua_State*) ../src/heka/sandbox.c 220 1 / 1 7 / 8 = 87% +lsb_init_input_buffer(lsb_input_buffer*,size_t) ...c/util/input_buffer.c 20 1 / 1 7 / 8 = 87% +lsb_outputd(lsb_output_buffer*,double) .../util/output_buffer.c 160 1 / 1 7 / 8 = 87% +process_fields(lsb_heka_field*,const char*,const char*) ...c/util/heka_message.c 121 1 / 1 50 / 57 = 87% +copy_table(lua_State*,lua_State*,lsb_logger) ../src/luasandbox.c 262 1 / 1 22 / 25 = 88% +load_expression_node(lua_State*,match_node*) ...eka/message_matcher.c 335 1 / 1 42 / 47 = 89% +lsb_decode_heka_message(lsb_heka_message*,const char*,size_t,lsb_logger) ...c/util/heka_message.c 201 1 / 1 51 / 57 = 89% +inject_payload(lua_State*) ../src/heka/sandbox.c 162 1 / 1 18 / 20 = 90% +lsb_add_function(lsb_lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 663 1 / 1 9 / 10 = 90% +lsb_init_output_buffer(lsb_output_buffer*,size_t) .../util/output_buffer.c 27 1 / 1 9 / 10 = 90% +lsb_find_heka_message(lsb_heka_message*,lsb_input_buffer*,_Bool,size_t*,lsb_logger) ...c/util/heka_message.c 319 1 / 1 29 / 32 = 90% +eval_node(match_node*,lsb_heka_message*) ...eka/message_matcher.c 235 1 / 1 31 / 34 = 91% +memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 77 1 / 1 11 / 12 = 91% +lsb_read_heka_field(lsb_heka_message*,lsb_const_string*,int,int,lsb_read_value*) ...c/util/heka_message.c 462 1 / 1 24 / 26 = 92% +classend(const char*) ...util/string_matcher.c 43 1 / 1 16 / 17 = 94% +lsb_expand_output_buffer(lsb_output_buffer*,size_t) .../util/output_buffer.c 52 1 / 1 17 / 18 = 94% +string_test(match_node*,lsb_const_string*) ...eka/message_matcher.c 167 1 / 1 18 / 19 = 94% +encode_string(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 258 1 / 1 2 / 2 = 100% +encode_field_object(lsb_lua_sandbox*,lsb_output_buffer*) ../src/heka/message.c 381 1 / 1 4 / 4 = 100% +lsb_destroy_message_matcher(lsb_message_matcher*) ...eka/message_matcher.c 559 1 / 1 6 / 6 = 100% +lsb_eval_message_matcher(lsb_message_matcher*,lsb_heka_message*) ...eka/message_matcher.c 578 1 / 1 0 / 0 +check_value(lua_State*) ...rc/heka/rapidjson.cpp 54 1 / 1 6 / 6 = 100% +schema_gc(lua_State*) ...rc/heka/rapidjson.cpp 71 1 / 1 0 / 0 +iter_gc(lua_State*) ...rc/heka/rapidjson.cpp 80 1 / 1 0 / 0 +hj_gc(lua_State*) ...rc/heka/rapidjson.cpp 89 1 / 1 0 / 0 +hj_validate(lua_State*) ...rc/heka/rapidjson.cpp 152 1 / 1 4 / 4 = 100% +hj_find(lua_State*) ...rc/heka/rapidjson.cpp 181 1 / 1 19 / 19 = 100% +hj_make_field(lua_State*) ...rc/heka/rapidjson.cpp 292 1 / 1 0 / 0 +hj_object_iter(lua_State*) ...rc/heka/rapidjson.cpp 305 1 / 1 4 / 4 = 100% +hj_array_iter(lua_State*) ...rc/heka/rapidjson.cpp 331 1 / 1 4 / 4 = 100% +hj_value(lua_State*) ...rc/heka/rapidjson.cpp 359 1 / 1 7 / 7 = 100% +read_message(lua_State*,int,lsb_heka_message*) ...rc/heka/rapidjson.cpp 428 1 / 1 14 / 14 = 100% +OutputBufferWrapper::OutputBufferWrapper(lsb_output_buffer*) ...rc/heka/rapidjson.cpp 532 1 / 1 0 / 0 +OutputBufferWrapper::Put(Ch) ...rc/heka/rapidjson.cpp 544 1 / 1 2 / 2 = 100% +OutputBufferWrapper::Flush() ...rc/heka/rapidjson.cpp 549 1 / 1 0 / 0 +OutputBufferWrapper::GetError() ...rc/heka/rapidjson.cpp 550 1 / 1 0 / 0 +luaopen_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 646 1 / 1 0 / 0 +lsb_heka_stop_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 694 1 / 1 0 / 0 +lsb_heka_terminate_sandbox(lsb_heka_sandbox*,const char*) ../src/heka/sandbox.c 700 1 / 1 0 / 0 +lsb_heka_destroy_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 706 1 / 1 2 / 2 = 100% +lsb_heka_get_error(lsb_heka_sandbox*) ../src/heka/sandbox.c 785 1 / 1 2 / 2 = 100% +lsb_heka_get_lua_file(lsb_heka_sandbox*) ../src/heka/sandbox.c 791 1 / 1 2 / 2 = 100% +lsb_heka_get_stats(lsb_heka_sandbox*) ../src/heka/sandbox.c 797 1 / 1 2 / 2 = 100% +lsb_heka_is_running(lsb_heka_sandbox*) ../src/heka/sandbox.c 818 1 / 1 4 / 4 = 100% +check_hsr(lua_State*,int) .../heka/stream_reader.c 68 1 / 1 0 / 0 +hsr_read_message(lua_State*) .../heka/stream_reader.c 196 1 / 1 6 / 6 = 100% +hsr_gc(lua_State*) .../heka/stream_reader.c 208 1 / 1 0 / 0 +luaopen_heka_stream_reader(lua_State*) .../heka/stream_reader.c 235 1 / 1 0 / 0 +libsize(const luaL_Reg*) ../src/luasandbox.c 43 1 / 1 2 / 2 = 100% +preload_modules(lua_State*) ../src/luasandbox.c 50 1 / 1 2 / 2 = 100% +instruction_usage(lsb_lua_sandbox*) ../src/luasandbox.c 108 1 / 1 0 / 0 +unprotected_panic(lua_State*) ../src/luasandbox.c 152 1 / 1 0 / 0 +get_usage_config(lua_State*,int,const char*) ../src/luasandbox.c 160 1 / 1 0 / 0 +stop_hook(lua_State*,lua_Debug*) ../src/luasandbox.c 552 1 / 1 0 / 0 +lsb_usage(lsb_lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 596 1 / 1 10 / 10 = 100% +lsb_get_error(lsb_lua_sandbox*) ../src/luasandbox.c 620 1 / 1 2 / 2 = 100% +lsb_set_error(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 626 1 / 1 4 / 4 = 100% +lsb_get_lua(lsb_lua_sandbox*) ../src/luasandbox.c 639 1 / 1 2 / 2 = 100% +lsb_get_lua_file(lsb_lua_sandbox*) ../src/luasandbox.c 645 1 / 1 2 / 2 = 100% +lsb_get_parent(lsb_lua_sandbox*) ../src/luasandbox.c 651 1 / 1 2 / 2 = 100% +lsb_get_state(lsb_lua_sandbox*) ../src/luasandbox.c 657 1 / 1 2 / 2 = 100% +lsb_pcall_teardown(lsb_lua_sandbox*) ../src/luasandbox.c 699 1 / 1 4 / 4 = 100% +lsb_terminate(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 713 1 / 1 6 / 6 = 100% +lsb_add_output_function(lua_State*,lua_CFunction) ...c/luasandbox_output.c 22 1 / 1 0 / 0 +lsb_get_output_function(lua_State*,int) ...c/luasandbox_output.c 30 1 / 1 0 / 0 +get_serialize_function(lua_State*,int) ...uasandbox_serialize.c 92 1 / 1 0 / 0 +find_table_ref(table_ref_array*,const void*) ...uasandbox_serialize.c 149 1 / 1 4 / 4 = 100% +get_preservation_version(lua_State*) ...uasandbox_serialize.c 195 1 / 1 4 / 4 = 100% +serialize_table(lsb_lua_sandbox*,serialization_data*,size_t) ...uasandbox_serialize.c 214 1 / 1 6 / 6 = 100% +file_exists(const char*) ...uasandbox_serialize.c 484 1 / 1 2 / 2 = 100% +lsb_add_serialize_function(lua_State*,lua_CFunction) ...uasandbox_serialize.c 539 1 / 1 0 / 0 +lsb_serialize_double(lsb_output_buffer*,double) ...uasandbox_serialize.c 574 1 / 1 6 / 6 = 100% +read_double_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 97 1 / 1 2 / 2 = 100% +process_varint(int,const char*,const char*,long long*) ...c/util/heka_message.c 110 1 / 1 4 / 4 = 100% +lsb_free_heka_message(lsb_heka_message*) ...c/util/heka_message.c 451 1 / 1 2 / 2 = 100% +lsb_write_heka_uuid(lsb_output_buffer*,const char*,size_t) ...c/util/heka_message.c 507 1 / 1 22 / 22 = 100% +lsb_free_input_buffer(lsb_input_buffer*) ...c/util/input_buffer.c 36 1 / 1 2 / 2 = 100% +lsb_free_output_buffer(lsb_output_buffer*) .../util/output_buffer.c 42 1 / 1 2 / 2 = 100% +lsb_pb_read_key(const char*,int*,int*) ../src/util/protobuf.c 15 1 / 1 8 / 8 = 100% +lsb_pb_write_key(lsb_output_buffer*,unsigned char,unsigned char) ../src/util/protobuf.c 25 1 / 1 2 / 2 = 100% +lsb_pb_write_varint(lsb_output_buffer*,unsigned long long) ../src/util/protobuf.c 75 1 / 1 2 / 2 = 100% +lsb_pb_write_bool(lsb_output_buffer*,int) ../src/util/protobuf.c 85 1 / 1 4 / 4 = 100% +lsb_pb_write_double(lsb_output_buffer*,double) ../src/util/protobuf.c 99 1 / 1 2 / 2 = 100% +lsb_pb_write_string(lsb_output_buffer*,char,const char*,size_t) ../src/util/protobuf.c 114 1 / 1 6 / 6 = 100% +lsb_pb_update_field_length(lsb_output_buffer*,size_t) ../src/util/protobuf.c 127 1 / 1 8 / 8 = 100% +lsb_init_running_stats(lsb_running_stats*) .../util/running_stats.c 13 1 / 1 0 / 0 +lsb_update_running_stats(lsb_running_stats*,double) .../util/running_stats.c 21 1 / 1 4 / 4 = 100% +lsb_sd_running_stats(lsb_running_stats*) .../util/running_stats.c 37 1 / 1 2 / 2 = 100% +lsb_init_const_string(lsb_const_string*) ../src/util/string.c 11 1 / 1 0 / 0 +singlematch(int,const char*,const char*) ...util/string_matcher.c 119 1 / 1 4 / 4 = 100% +max_expand(MatchState*,const char*,const char*,const char*) ...util/string_matcher.c 156 1 / 1 10 / 10 = 100% +lsb_string_match(const char*,size_t,const char*) ...util/string_matcher.c 261 1 / 1 10 / 10 = 100% +lsb_lp2(unsigned long long) ../src/util/util.c 37 1 / 1 2 / 2 = 100% +lsb_get_time() ../src/util/util.c 77 1 / 1 0 / 0 +-------------------------------------------------------------------------------------------------- ------------------------------ ----- --------------------- +Total 100% 1712 / 2138 = 80% diff --git a/docs/heka/heka_json.md b/docs/heka/heka_json.md new file mode 100644 index 0000000..2b083fd --- /dev/null +++ b/docs/heka/heka_json.md @@ -0,0 +1,202 @@ +## Heka JSON Module + +Allows for JSON-Schema validation and more efficient manipulation of large JSON +structures where only a small amount of the data is consumed by the plugin. +Schema pattern matching is restricted to a subset of regex decribed in the +Regular Expression section of the +[RapidJSON Schema Documentation](http://rapidjson.org/md_doc_schema.html). + +### Availability + +Input/Output plugins only. + +### API + +#### parse + +Creates a Heka JSON Document from a string. + +```lua +local ok, doc = pcall(heka_json.parse, '{"foo":"bar"}') +assert(ok, doc) + +``` +*Arguments* +* JSON (string) - JSON string to parse + +*Return* +* doc (userdata) - Heka JSON document or an error is thrown + +#### parse_schema + +Creates a Heka JSON Schema. + +```lua +local ok, doc = pcall(heka_json.parse_schema, '{"type":"array","minItems": 1,"oneOf": [{"items": {"type":"number"}}]}') +assert(ok, doc) + +``` +*Arguments* +* JSON (string) - JSON schema string to parse + +*Return* +* schema (userdata) - Heka JSON schema or an error is thrown + +#### parse_message + +Creates a Heka JSON Document from a message variable. + +```lua +local ok, doc = pcall(heka_json.parse_message, "Fields[myjson]") +assert(ok, doc) + +``` +*Arguments* +* heka_stream_reader (userdata) - require only for Input plugins since there is + no active message available. +* variableName (string) + * Payload + * Fields[*name*] +* fieldIndex (unsigned) - optional, only used in combination with the Fields + variableName use to retrieve a specific instance of a repeated field name; + zero indexed +* arrayIndex (unsigned) - optional, only used in combination with the Fields + variableName use to retrieve a specific element out of a field containing an + array; zero indexed + +*Return* +* doc (userdata) - Heka JSON document or an error is thrown + +### Heka JSON Document API Methods + +#### validate + +Checks that the JSON document conforms to the specified schema. + +```lua +local ok, err = doc:validate(schema) +assert(ok, err) + +``` +*Arguments* +* heka_schema (userdata) - a compiled schema to validate against + +*Return* +* ok (bool) - true if valid +* err (string) - error message on failure + +#### type + +Returns the type of the value in the JSON structure. + +```lua +local t = doc:type() +assert(t == "object", t) + +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document + +*Return* +* type (string) - "string", "number", "boolean", "object", "array" or "null" + +#### find + +Searches for and returns a value in the JSON structure. + +```lua +local v = doc:find("obj", "arr", 0, "foo") +assert(v, "not found") + +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document +* key (string, number) - object key, or array index +* keyN (string, number) - final object key, or array index + +*Return* +* value (lightuserdata) - handle to be passed to other methods, nil if not found + +#### value + +Returns the primitive value of the JSON element. + +```lua +local v = doc:find("obj", "arr", 0, "foo") +local str = doc:value(v) +assert("bar" == str, tostring(str)) + +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document + +*Return* +* primitive - string, number, bool, nil or throws an error if not convertable (object, array) + +#### iter + +Retrieves an interator function for an object/array. + +```lua +local v = doc:find("obj", "arr") +for i,v in doc:iter(v) do +-- ... +end +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document + +*Return* +* iter (function) - iterator function returning an index/value for arrays or a key/value for + objects. Throws an error on primitive types. + +#### size + +Returns the size of the value. +```lua +local v = doc:find("obj", "arr") +local n = doc:size(v) + +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document + +*Return* +* size (number) - Number of element in an array/object or the length of the string. + Throws an error on numeric, boolean and null types. + +#### make_field + +Helper function to wrap the lightuserdata so it can be used in an inject_message field. + +```lua +local msg = {Fields = {}} +local v = doc:find("obj", "arr") +msg.Fields.array = doc:make_field(v) -- set array to the JSON string representation of "arr" +inject_message(msg) + +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document + +*Return* +* table - shorthand for `{value = v, userdata = doc}` + +#### remove + +Searches for and NULL's out the resulting value in the JSON structure returning the +extracted Heka JSON document. + +```lua +local rv = doc:remove("obj", "arr") +assert(rv, "not found") +rv:size() -- size of the extracted array + +``` +*Arguments* +* value (lightuserdata) - optional, when not specified the function is applied to document +* key (string, number) - object key, or array index +* keyN (string, number) - final object key, or array index + +*Return* +* doc (userdata) - the object removed from the original JSON or nil diff --git a/docs/heka/index.md b/docs/heka/index.md index 2651da2..1fd5193 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -14,33 +14,49 @@ of that infrastructure has been created to replace it called * [Message Matcher](message_matcher.md) * [Input Sandbox](input.md) * [Heka Stream Reader](heka_stream_reader.md) + * [Heka JSON](heka_json.md) * [Analysis Sandbox](analysis.md) * [Output Sandbox](output.md) + * [Heka JSON](heka_json.md) ## Sandbox API Changes from the Go Heka Sandbox There are a few intentional changes between tho original Heka sandbox and this version. ### Changes + 1. `read_message` now has a `framed` parameter to retrive the raw message with stream framing. + #### Input Sandbox + 1. `inject_message` accepts a numeric or string checkpoint as the second argument 1. `process_message` receives the checkpoint value as the first argument (if it was provided by `inject_message`). + #### Output Sandbox + 1. `process_message` receives a sequence_id as the first argument (if it was provided by `update_checkpoint`). Extended return codes have been added to support skipping, retrying, batching, and asynchronous output. + #### Analysis/Output Sandbox + 1. `timer_event` has a second parameter `shutdown` that is set to true when the sandbox is exiting. ### Additions + #### Input Sandbox + 1. A [Heka Stream Reader](heka_stream_reader.md) Lua module was added. +1. A [Heka JSON](heka_json.md) Lua module was added. + #### Output Sandbox + 1. [update_checkpoint](output.md#update_checkpoint) was added for batch and asynchronous processing. ### Removals + 1. The `write_message` API was removed; messages are immutable and this API broke that rule. 1. The `read_next_field` API was removed; instead the raw message should be decoded and the Lua table iterated. ### Notes + 1. The `read_config` API in unchanged but now has access to the entire sandbox configuration. diff --git a/include/luasandbox.h b/include/luasandbox.h index fe14687..335f912 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -9,7 +9,6 @@ #ifndef luasandbox_h_ #define luasandbox_h_ -#include "luasandbox/lua.h" #include "luasandbox/error.h" #ifdef _WIN32 @@ -65,6 +64,8 @@ extern "C" { #endif +#include "luasandbox/lua.h" + LSB_EXPORT extern lsb_err_id LSB_ERR_INIT; LSB_EXPORT extern lsb_err_id LSB_ERR_LUA; LSB_EXPORT extern lsb_err_id LSB_ERR_TERMINATED; diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index b287dca..da8292b 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -13,8 +13,6 @@ #include #include "../error.h" -#include "../lauxlib.h" -#include "../lua.h" #include "../util/heka_message.h" #ifdef _WIN32 @@ -64,6 +62,9 @@ extern "C" { #endif +#include "../lauxlib.h" +#include "../lua.h" + LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; /** diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index e17b62e..8858cd1 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -62,6 +62,21 @@ LSB_UTIL_EXPORT char* lsb_read_file(const char *fn); */ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); +#ifdef HAVE_ZLIB + +/** + * Ungzip the provided string into a new string + * + * @param s String to ungzip + * @param s_len Length of the string to ungzip + * @param r_len Length of the returned string + * + * @return char* Returned string (MUST be freed by the caller), NULL on failure + */ +LSB_UTIL_EXPORT char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len); + +#endif + #ifdef __cplusplus } #endif diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index 85a4400..52e4029 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -12,13 +12,14 @@ #include #include "luasandbox.h" -#include "luasandbox/lua.h" #include "luasandbox/util/output_buffer.h" #ifdef __cplusplus extern "C" { #endif +#include "luasandbox/lua.h" + /** * Add a output function to the environment table. The environment table must be * on the top of the stack. This function will receive the userdata and diff --git a/include/luasandbox_serialize.h b/include/luasandbox_serialize.h index 40dfda4..bb9d95d 100644 --- a/include/luasandbox_serialize.h +++ b/include/luasandbox_serialize.h @@ -11,13 +11,14 @@ #include -#include "luasandbox/lua.h" #include "luasandbox_output.h" #ifdef __cplusplus extern "C" { #endif +#include "luasandbox/lua.h" + /** * Add a serialization function to the environment table. The environment table * must be on the top of the stack. This function will receive the userdata, diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index 522cea8..f82f3bc 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -5,6 +5,7 @@ set(HEKA_SRC message.c message_matcher.c +rapidjson.cpp sandbox.c stream_reader.c ) diff --git a/src/heka/message.c b/src/heka/message.c index dd743a2..bf1bd41 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -540,13 +540,22 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, } break; + case LUA_TLIGHTUSERDATA: + lua_getfield(lsb->lua, -4, "userdata"); + if (lua_type(lsb->lua, -1) != LUA_TUSERDATA) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "a lightuserdata output must also specify a userdata value"); + return LSB_ERR_LUA; + } + // fall thru + case LUA_TUSERDATA: { lua_CFunction fp = lsb_get_output_function(lsb->lua, -1); size_t len_pos = 0; if (!fp) { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "user data object does not implement lsb_output"); + "userdata object does not implement lsb_output"); return LSB_ERR_LUA; } if (first) { @@ -576,11 +585,12 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, lua_pop(lsb->lua, 1); // remove output function if (result) { snprintf(lsb->error_message, LSB_ERROR_SIZE, - "user data output callback failed: %d", result); + "userdata output callback failed: %d", result); return LSB_ERR_LUA; } ret = lsb_pb_update_field_length(ob, len_pos); } + if (t == LUA_TLIGHTUSERDATA) lua_pop(lsb->lua, 1); // remove the userdata break; default: diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp new file mode 100644 index 0000000..0b34637 --- /dev/null +++ b/src/heka/rapidjson.cpp @@ -0,0 +1,676 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua RapidJSON wrapper implementation @file */ + +#include "rapidjson_impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "luasandbox_output.h" +#include "sandbox_impl.h" +#include "stream_reader_impl.h" + +namespace rj = rapidjson; + +typedef struct heka_json +{ + rj::Document *doc; + rj::Value *val; + std::set *refs; +} heka_json; + + +typedef struct heka_schema +{ + rj::SchemaDocument *doc; +} heka_schema; + + +typedef struct heka_object_iter +{ + rj::Value::MemberIterator *it; + rj::Value::MemberIterator *end; +} heka_object_iterator; + + +const char *mozsvc_heka_json = "mozsvc.heka_json"; +const char *mozsvc_heka_json_table = "heka_json"; + +static const char *mozsvc_heka_schema = "mozsvc.heka_schema"; +static const char *mozsvc_heka_object_iter = "mozsvc.heka_object_iter"; + + +static rj::Value* check_value(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n >= 1 && n <= 2, 0, "invalid number of arguments"); + heka_json *hj = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_json)); + rj::Value *v = static_cast(lua_touserdata(lua, 2)); + if (!v) { + luaL_checktype(lua, 2, LUA_TNONE); // only default it when it is not set + v = hj->doc ? hj->doc : hj->val; + } else if (hj->refs->find(v) == hj->refs->end()) { + luaL_error(lua, "invalid value"); + } + return v; +} + + +static int schema_gc(lua_State *lua) +{ + heka_schema *hs = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_schema)); + delete(hs->doc); + return 0; +} + + +static int iter_gc(lua_State *lua) +{ + heka_object_iter *hoi = static_cast(lua_touserdata(lua, 1)); + delete(hoi->it); + delete(hoi->end); + return 0; +} + + +static int hj_gc(lua_State *lua) +{ + heka_json *hj = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_json)); + delete(hj->refs); + delete(hj->doc); + delete(hj->val); + return 0; +} + + +static int hj_parse_schema(lua_State *lua) +{ + const char *json = luaL_checkstring(lua, 1); + heka_schema *hs = static_cast(lua_newuserdata(lua, sizeof*hs)); + hs->doc = NULL; + luaL_getmetatable(lua, mozsvc_heka_schema); + lua_setmetatable(lua, -2); + + { // allows doc to be destroyed before the longjmp + rj::Document doc; + if (doc.Parse(json).HasParseError()) { + lua_pushfstring(lua, "failed to parse offset:%f %s", + (lua_Number)doc.GetErrorOffset(), + rj::GetParseError_En(doc.GetParseError())); + } else { + hs->doc = new rj::SchemaDocument(doc); + if (!hs->doc) { + lua_pushstring(lua, "memory allocation failed"); + } + } + } + if (!hs->doc) return lua_error(lua); + return 1; +} + + +static int hj_parse(lua_State *lua) +{ + const char *json = luaL_checkstring(lua, 1); + heka_json *hj = static_cast(lua_newuserdata(lua, sizeof*hj)); + hj->doc = new rj::Document; + hj->val = NULL; + hj->refs = new std::set; + luaL_getmetatable(lua, mozsvc_heka_json); + lua_setmetatable(lua, -2); + + if (!hj->doc || !hj->refs) { + lua_pushstring(lua, "memory allocation failed"); + return lua_error(lua); + } else { + if (hj->doc->Parse(json).HasParseError()) { + lua_pushfstring(lua, "failed to parse offset:%f %s", + (lua_Number)hj->doc->GetErrorOffset(), + rj::GetParseError_En(hj->doc->GetParseError())); + return lua_error(lua); + } + } + hj->refs->insert(hj->doc); + return 1; +} + + +static int hj_validate(lua_State *lua) +{ + heka_json *hj = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_json)); + heka_schema *hs = static_cast + (luaL_checkudata(lua, 2, mozsvc_heka_schema)); + + rj::SchemaValidator validator(*hs->doc); + rj::Value *v = hj->doc ? hj->doc : hj->val; + if (!v->Accept(validator)) { + lua_pushboolean(lua, false); + luaL_Buffer b; + luaL_buffinit(lua, &b); + rj::StringBuffer sb; + validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); + luaL_addstring(&b, "SchemaURI: "); + luaL_addstring(&b, sb.GetString()); + luaL_addstring(&b, " Keyword: "); + luaL_addstring(&b, validator.GetInvalidSchemaKeyword()); + sb.Clear(); + validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); + luaL_addstring(&b, " DocumentURI: "); + luaL_addstring(&b, sb.GetString()); + luaL_pushresult(&b); + } + return 2; // ok, err +} + + +static int hj_find(lua_State *lua) +{ + heka_json *hj = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_json)); + + int start = 3; + rj::Value *v = static_cast(lua_touserdata(lua, 2)); + if (!v) { + v = hj->doc ? hj->doc : hj->val; + start = 2; + } else if (hj->refs->find(v) == hj->refs->end()) { + return luaL_error(lua, "invalid value"); + } + + int n = lua_gettop(lua); + for (int i = start; i <= n; ++i) { + switch (lua_type(lua, i)) { + case LUA_TSTRING: + { + if (!v->IsObject()) { + lua_pushnil(lua); + return 1; + } + rj::Value::MemberIterator itr = v->FindMember(lua_tostring(lua, i)); + if (itr == v->MemberEnd()) { + lua_pushnil(lua); + return 1; + } + v = &itr->value; + } + break; + case LUA_TNUMBER: + { + if (!v->IsArray()) { + lua_pushnil(lua); + return 1; + } + rj::SizeType idx = static_cast(lua_tonumber(lua, i)); + if (idx >= v->Size()) { + lua_pushnil(lua); + return 1; + } + v = &(*v)[idx]; + } + break; + default: + lua_pushnil(lua); + return 1; + } + } + hj->refs->insert(v); + lua_pushlightuserdata(lua, v); + return 1; +} + + +static int hj_type(lua_State *lua) +{ + rj::Value *v = check_value(lua); + + switch (v->GetType()) { + case rj::kStringType: + lua_pushstring(lua, "string"); + break; + case rj::kNumberType: + lua_pushstring(lua, "number"); + break; + case rj::kFalseType: + case rj::kTrueType: + lua_pushstring(lua, "boolean"); + break; + case rj::kObjectType: + lua_pushstring(lua, "object"); + break; + case rj::kArrayType: + lua_pushstring(lua, "array"); + break; + case rj::kNullType: + lua_pushstring(lua, "null"); + break; + } + return 1; +} + + +static int hj_size(lua_State *lua) +{ + rj::Value *v = check_value(lua); + + switch (v->GetType()) { + case rj::kStringType: + lua_pushnumber(lua, (lua_Number)v->GetStringLength()); + break; + case rj::kNumberType: + return luaL_error(lua, "attempt to get length of a number"); + case rj::kFalseType: + case rj::kTrueType: + return luaL_error(lua, "attempt to get length of a boolean"); + case rj::kObjectType: + lua_pushnumber(lua, (lua_Number)v->MemberCount()); + break; + case rj::kArrayType: + lua_pushnumber(lua, (lua_Number)v->Size()); + break; + case rj::kNullType: + return luaL_error(lua, "attempt to get length of a NULL"); + } + return 1; +} + + +static int hj_make_field(lua_State *lua) +{ + rj::Value *v = check_value(lua); + + lua_createtable(lua, 0, 2); + lua_pushlightuserdata(lua, (void *)v); + lua_setfield(lua, -2, "value"); + lua_pushvalue(lua, 1); + lua_setfield(lua, -2, "userdata"); + return 1; +} + + +static int hj_object_iter(lua_State *lua) +{ + heka_object_iter *hoi = static_cast + (lua_touserdata(lua, lua_upvalueindex(1))); + rj::Value *v = (rj::Value *)lua_touserdata(lua, lua_upvalueindex(2)); + heka_json *hj = (heka_json *)lua_touserdata(lua, lua_upvalueindex(3)); + + if (hj->refs->find(v) == hj->refs->end()) { + return luaL_error(lua, "iterator has been invalidated"); + } + + if (*hoi->it != *hoi->end) { + rj::Value *next = &(*hoi->it)->value; + hj->refs->insert(next); + lua_pushlstring(lua, (*hoi->it)->name.GetString(), + (lua_Number)(*hoi->it)->name.GetStringLength()); + lua_pushlightuserdata(lua, next); + ++*hoi->it; + } else { + lua_pushnil(lua); + lua_pushnil(lua); + } + return 2; +} + + +static int hj_array_iter(lua_State *lua) +{ + rj::SizeType it = (rj::SizeType)lua_tonumber(lua, lua_upvalueindex(1)); + rj::SizeType end = (rj::SizeType)lua_tonumber(lua, lua_upvalueindex(2)); + rj::Value *v = (rj::Value *)lua_touserdata(lua, lua_upvalueindex(3)); + heka_json *hj = (heka_json *)lua_touserdata(lua, lua_upvalueindex(4)); + + if (hj->refs->find(v) == hj->refs->end()) { + return luaL_error(lua, "iterator has been invalidated"); + } + + if (it != end) { + rj::Value *next = &(*v)[it]; + hj->refs->insert(next); + lua_pushnumber(lua, (lua_Number)it); + lua_pushlightuserdata(lua, next); + + ++it; + lua_pushnumber(lua, (lua_Number)it); + lua_replace(lua, lua_upvalueindex(1)); + } else { + lua_pushnil(lua); + lua_pushnil(lua); + } + return 2; +} + + +static int hj_value(lua_State *lua) +{ + rj::Value *v = check_value(lua); + + switch (v->GetType()) { + case rj::kStringType: + lua_pushlstring(lua, v->GetString(), (size_t)v->GetStringLength()); + break; + case rj::kNumberType: + lua_pushnumber(lua, (lua_Number)v->GetDouble()); + break; + case rj::kFalseType: + case rj::kTrueType: + lua_pushboolean(lua, v->GetBool()); + break; + case rj::kObjectType: + return luaL_error(lua, "value() not allowed on an object"); + break; + case rj::kArrayType: + return luaL_error(lua, "value() not allowed on an array"); + break; + default: + lua_pushnil(lua); + break; + } + return 1; +} + + +static int hj_iter(lua_State *lua) +{ + rj::Value *v = check_value(lua); + + switch (v->GetType()) { + case rj::kObjectType: + { + heka_object_iter *hoi = static_cast + (lua_newuserdata(lua, sizeof*hoi)); + hoi->it = new rj::Value::MemberIterator; + hoi->end = new rj::Value::MemberIterator; + luaL_getmetatable(lua, mozsvc_heka_object_iter); + lua_setmetatable(lua, -2); + if (!hoi->it || !hoi->end) { + return luaL_error(lua, "memary allocation failure"); + } + *hoi->it = v->MemberBegin(); + *hoi->end = v->MemberEnd(); + lua_pushlightuserdata(lua, (void *)v); + lua_pushvalue(lua, 1); + lua_pushcclosure(lua, hj_object_iter, 3); + } + break; + case rj::kArrayType: + { + lua_pushnumber(lua, 0); + lua_pushnumber(lua, (lua_Number)v->Size()); + lua_pushlightuserdata(lua, (void *)v); + lua_pushvalue(lua, 1); + lua_pushcclosure(lua, hj_array_iter, 4); + } + break; + default: + return luaL_error(lua, "iter() not allowed on a primitive type"); + break; + } + return 1; +} + + +lsb_const_string read_message(lua_State *lua, int idx, lsb_heka_message *m) +{ + lsb_const_string ret = { NULL, 0 }; + size_t field_len; + const char *field = luaL_checklstring(lua, idx, &field_len); + int fi = (int)luaL_optinteger(lua, idx + 1, 0); + luaL_argcheck(lua, fi >= 0, idx + 1, "field index must be >= 0"); + int ai = (int)luaL_optinteger(lua, idx + 2, 0); + luaL_argcheck(lua, ai >= 0, idx + 2, "array index must be >= 0"); + + if (strcmp(field, LSB_PAYLOAD) == 0) { + if (m->payload.s) ret = m->payload; + } else { + if (field_len >= 8 + && memcmp(field, LSB_FIELDS "[", 7) == 0 + && field[field_len - 1] == ']') { + lsb_read_value v; + lsb_const_string f = { field + 7, field_len - 8 }; + lsb_read_heka_field(m, &f, fi, ai, &v); + if (v.type == LSB_READ_STRING) ret = v.u.s; + } + } + return ret; +} + + +static int hj_parse_message(lua_State *lua) +{ + lsb_lua_sandbox *lsb = static_cast + (lua_touserdata(lua, lua_upvalueindex(1))); + if (!lsb) { + return luaL_error(lua, "%s() invalid lightuserdata", __FUNCTION__); + } + int n = lua_gettop(lua); + int idx = 1; + + lsb_heka_message *msg = NULL; + lsb_heka_sandbox *hsb = static_cast(lsb_get_parent(lsb)); + if (hsb->type == 'i') { + luaL_argcheck(lua, n >= 2 && n <= 4, 0, "invalid number of arguments"); + heka_stream_reader *hsr = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_stream_reader)); + msg = &hsr->msg; + idx = 2; + } else { + luaL_argcheck(lua, n >= 1 && n <= 3, 0, "invalid number of arguments"); + if (!hsb->msg || !hsb->msg->raw.s) { + return luaL_error(lua, "no active message"); + } + msg = hsb->msg; + } + + lsb_const_string json = read_message(lua, idx, msg); + if (!json.s) return luaL_error(lua, "field not found"); + +#ifdef HAVE_ZLIB + char *inflated = NULL; + // automatically handle gzipped strings (optimization for Mozilla telemetry + // messages) + if (json.len > 2) { + if (json.s[0] == 0x1f && (unsigned char)json.s[1] == 0x8b) { + inflated = lsb_ungzip(json.s, json.len, &json.len); + if (!inflated) return luaL_error(lua, "lsb_ungzip failed"); + json.s = inflated; + } + } +#endif + + heka_json *hj = static_cast(lua_newuserdata(lua, sizeof*hj)); + hj->doc = new rj::Document; + hj->val = NULL; + hj->refs = new std::set; + luaL_getmetatable(lua, mozsvc_heka_json); + lua_setmetatable(lua, -2); + + if (!hj->doc || !hj->refs) { + lua_pushstring(lua, "memory allocation failed"); + return lua_error(lua); + } + + bool err = false; + { // allows ms to be destroyed before the longjmp + rj::MemoryStream ms(json.s, json.len); + if (hj->doc->ParseStream<0, rj::UTF8<> >(ms).HasParseError()) { + err = true; + lua_pushfstring(lua, "failed to parse offset:%f %s", + (lua_Number)hj->doc->GetErrorOffset(), + rj::GetParseError_En(hj->doc->GetParseError())); + } + } + +#ifdef HAVE_ZLIB + if (inflated) free(inflated); +#endif + + if (err) return lua_error(lua); + hj->refs->insert(hj->doc); + return 1; +} + + +class OutputBufferWrapper { +public: + typedef char Ch; + OutputBufferWrapper(lsb_output_buffer *ob) : ob_(ob), err_(NULL) { } +#if _BullseyeCoverage + #pragma BullseyeCoverage off +#endif + Ch Peek() const { assert(false); return '\0'; } + Ch Take() { assert(false); return '\0'; } + size_t Tell() const { return 0; } + Ch* PutBegin() { assert(false); return 0; } + size_t PutEnd(Ch *) { assert(false); return 0; } +#if _BullseyeCoverage + #pragma BullseyeCoverage on +#endif + void Put(Ch c) + { + const char *err = lsb_outputc(ob_, c); + if (err) err_ = err; + } + void Flush() { return; } + const char* GetError() { return err_; } +private: + OutputBufferWrapper(const OutputBufferWrapper&); + OutputBufferWrapper& operator=(const OutputBufferWrapper&); + lsb_output_buffer *ob_; + const char *err_; +}; + + +static int output_heka_json(lua_State *lua) +{ + lsb_output_buffer *ob = static_cast + (lua_touserdata(lua, -1)); + heka_json *hj = static_cast(lua_touserdata(lua, -2)); + rj::Value *v = static_cast(lua_touserdata(lua, -3)); + if (!(ob && hj && v)) { + return 1; + } + if (hj->refs->find(v) == hj->refs->end()) { + return 1; + } + OutputBufferWrapper obw(ob); + rapidjson::Writer writer(obw); + v->Accept(writer); + return obw.GetError() == NULL ? 0 : 1; +} + + +static int hj_remove(lua_State *lua) +{ + hj_find(lua); + rj::Value *v = static_cast(lua_touserdata(lua, -1)); + if (!v) { + lua_pushnil(lua); + return 1; + } + + heka_json *hj = static_cast + (luaL_checkudata(lua, 1, mozsvc_heka_json)); + + heka_json *nv = static_cast(lua_newuserdata(lua, sizeof*nv)); + nv->doc = NULL; + nv->val = new rj::Value; + nv->refs = new std::set; + luaL_getmetatable(lua, mozsvc_heka_json); + lua_setmetatable(lua, -2); + + if (!nv->val || !nv->refs) { + lua_pushstring(lua, "memory allocation failed"); + return lua_error(lua); + } + + *nv->val = *v; // move the value out replacing the original with NULL + hj->refs->erase(v); + nv->refs->insert(nv->val); + return 1; +} + + +static const struct luaL_reg schemalib_m[] = +{ + { "__gc", schema_gc }, + { NULL, NULL } +}; + + +static const struct luaL_reg iterlib_m[] = +{ + { "__gc", iter_gc }, + { NULL, NULL } +}; + + +static const struct luaL_reg hjlib_f[] = +{ + { "parse_schema", hj_parse_schema }, + { "parse", hj_parse }, + { NULL, NULL } +}; + + +static const struct luaL_reg hjlib_m[] = +{ + { "validate", hj_validate }, + { "type", hj_type }, + { "find", hj_find }, + { "value", hj_value }, + { "iter", hj_iter }, + { "size", hj_size }, + { "make_field", hj_make_field }, + { "remove", hj_remove }, + { "__gc", hj_gc }, + { NULL, NULL } +}; + + +int luaopen_heka_json(lua_State *lua) +{ + lua_newtable(lua); + lsb_add_output_function(lua, output_heka_json); + lua_replace(lua, LUA_ENVIRONINDEX); + + luaL_newmetatable(lua, mozsvc_heka_schema); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, schemalib_m); + lua_pop(lua, 1); + + luaL_newmetatable(lua, mozsvc_heka_object_iter); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, iterlib_m); + lua_pop(lua, 1); + + luaL_newmetatable(lua, mozsvc_heka_json); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, hjlib_m); + luaL_register(lua, mozsvc_heka_json_table, hjlib_f); + // special case parse_message since it needs access to the sandbox + lsb_lua_sandbox *lsb = static_cast + (lua_touserdata(lua, lua_upvalueindex(1))); + lua_pushlightuserdata(lua, static_cast(lsb)); + lua_pushcclosure(lua, hj_parse_message, 1); + lua_setfield(lua, -2, "parse_message"); + return 1; +} diff --git a/src/heka/rapidjson_impl.h b/src/heka/rapidjson_impl.h new file mode 100644 index 0000000..10f8cf8 --- /dev/null +++ b/src/heka/rapidjson_impl.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight Heka JSON structures @file */ + +#ifndef luasandbox_heka_json_h_ +#define luasandbox_heka_json_h_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "lua.h" +#include "luasandbox/util/heka_message.h" +#include "lauxlib.h" + +extern const char *mozsvc_heka_json; +extern const char *mozsvc_heka_json_table; + +int luaopen_heka_json(lua_State *lua); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 6aec594..e246205 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -20,6 +20,7 @@ #include "message_impl.h" #include "sandbox_impl.h" #include "stream_reader_impl.h" +#include "rapidjson_impl.h" #ifdef _WIN32 #include @@ -391,6 +392,12 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lua_pushstring(lua, mozsvc_heka_stream_reader_table); lua_pushcfunction(lua, luaopen_heka_stream_reader); lua_rawset(lua, -3); +// preload Heka JSON with a pointer back to the sandbox + lua_pushstring(lua, mozsvc_heka_json_table); + lua_pushlightuserdata(lua, (void *)hsb->lsb); + lua_pushcclosure(lua, luaopen_heka_json, 1); + lua_rawset(lua, -3); + lua_pop(lua, 1); // remove the preloaded table if (lsb_init(hsb->lsb, state_file)) { @@ -658,6 +665,15 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); lsb_add_function(hsb->lsb, update_checkpoint, "update_checkpoint"); +// preload Heka JSON with a pointer back to the sandbox + luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); + lua_pushstring(lua, mozsvc_heka_json_table); + lua_pushlightuserdata(lua, (void *)hsb->lsb); + lua_pushcclosure(lua, luaopen_heka_json, 1); + lua_rawset(lua, -3); + + lua_pop(lua, 1); // remove the preloaded table + if (lsb_init(hsb->lsb, state_file)) { if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); lsb_destroy(hsb->lsb); diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c index 4d32c34..b5f258b 100644 --- a/src/heka/stream_reader.c +++ b/src/heka/stream_reader.c @@ -217,13 +217,18 @@ static int hsr_gc(lua_State *lua) static const struct luaL_reg heka_stream_readerlib_f[] = { - { "new", hsr_new }, { NULL, NULL } + { "new", hsr_new }, + { NULL, NULL } }; static const struct luaL_reg heka_stream_readerlib_m[] = { - { "find_message", hsr_find_message }, { "decode_message", hsr_decode_message }, { "read_message", hsr_read_message }, { "__gc", hsr_gc }, { NULL, NULL } + { "find_message", hsr_find_message }, + { "decode_message", hsr_decode_message }, + { "read_message", hsr_read_message }, + { "__gc", hsr_gc }, + { NULL, NULL } }; diff --git a/src/heka/stream_reader_impl.h b/src/heka/stream_reader_impl.h index 6512fba..abc2cf5 100644 --- a/src/heka/stream_reader_impl.h +++ b/src/heka/stream_reader_impl.h @@ -17,6 +17,9 @@ extern const char *mozsvc_heka_stream_reader; extern const char *mozsvc_heka_stream_reader_table; +extern const char *mozsvc_heka_json; +extern const char *mozsvc_heka_json_table; + typedef struct heka_stream_reader { char *name; diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua index bdc3211..bb22422 100644 --- a/src/heka/test/lua/encode_message.lua +++ b/src/heka/test/lua/encode_message.lua @@ -5,8 +5,11 @@ require "string" require "table" require "circular_buffer" +require "heka_json" local cb = circular_buffer.new(2,1,1) +local doc = heka_json.parse([[{"foo":"bar"}]]) +assert(doc) local msgs = { { @@ -73,6 +76,9 @@ local msgs = { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = cb}}, -- userdata rv = '\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\141\1\10\3\107\101\121\16\1\42\131\1\123\34\116\105\109\101\34\58\48\44\34\114\111\119\115\34\58\50\44\34\99\111\108\117\109\110\115\34\58\49\44\34\115\101\99\111\110\100\115\95\112\101\114\95\114\111\119\34\58\49\44\34\99\111\108\117\109\110\95\105\110\102\111\34\58\91\123\34\110\97\109\101\34\58\34\67\111\108\117\109\110\95\49\34\44\34\117\110\105\116\34\58\34\99\111\117\110\116\34\44\34\97\103\103\114\101\103\97\116\105\111\110\34\58\34\115\117\109\34\125\93\125\10\110\97\110\10\110\97\110\10' }, + { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {json = doc:make_field()}}, + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\23\10\4\106\115\111\110\16\1\42\13\123\34\102\111\111\34\58\34\98\97\114\34\125" + }, } for i, v in ipairs(msgs) do @@ -107,3 +113,6 @@ assert("encode_message() failed: array has mixed types" == err, string.format("r ok, err = pcall(encode_message, {Fields = { {noname = "foo", value = 1}} }) assert(not ok) assert("encode_message() failed: field name must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(heka_json.parse_message, "Payload") +assert("no active message" == err, err) diff --git a/src/heka/test/lua/heka_json.lua b/src/heka/test/lua/heka_json.lua new file mode 100644 index 0000000..2b6f418 --- /dev/null +++ b/src/heka/test/lua/heka_json.lua @@ -0,0 +1,298 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "heka_json" +require "heka_stream_reader" + +schema_json = [[{ + "type":"object", + "required":["Timestamp"], + "properties":{ + "Timestamp":{ + "type":"integer", + "minimum":0 + }, + "Type":{ + "type":"string" + }, + "Logger":{ + "type":"string" + }, + "Hostname":{ + "type":"string", + "format":"hostname" + }, + "EnvVersion":{ + "type":"string", + "pattern":"^[0-9]+(\\.[0-9]+){0,2}" + }, + "Severity":{ + "type":"integer", + "minimum":0, + "maximum":7 + }, + "Pid":{ + "type":"integer", + "minimum":0 + }, + "Fields":{ + "type":"object", + "minProperties":1, + "additionalProperties":{ + "anyOf": [ + { "$ref": "#/definitions/field_value"}, + { "$ref": "#/definitions/field_array"}, + { "$ref": "#/definitions/field_object"} + ] + } + } + }, + "definitions":{ + "field_value":{ + "type":["string", "number", "boolean"] + }, + "field_array":{ + "type":"array", + "minItems": 1, + "oneOf": [ + {"items": {"type":"string"}}, + {"items": {"type":"number"}}, + {"items": {"type":"boolean"}} + ] + }, + "field_object":{ + "type":"object", + "required":["value"], + "properties":{ + "value":{ + "oneOf": [ + { "$ref": "#/definitions/field_value" }, + { "$ref": "#/definitions/field_array" } + ] + }, + "representation":{"type":"string"} + } + } + } +}]] + +json = [[{"Timestamp":0, "Type":"foo", "Logger":"bar"}]] + +schema = heka_json.parse_schema(schema_json) +assert(schema) + +ok, err = pcall(heka_json.parse_schema, "{") +assert(not ok) +assert(err == "failed to parse offset:1 Missing a name for object member.", err) +ok, err = pcall(heka_json.parse, "{") +assert(not ok) +assert(err == "failed to parse offset:1 Missing a name for object member.", err) + +doc = heka_json.parse(json) +assert(doc) +assert(doc:validate(schema)) +doc1 = heka_json.parse(json) + +json = [[{"Timestamp":-1, "Type":"foo", "Logger":"bar"}]] +doc = heka_json.parse(json) +assert(doc) +assert(not doc:validate(schema)) + +json = [[{"Timestamp":0, "EnvVersion":"unknown"}]] +doc = heka_json.parse(json) +assert(doc) +assert(not doc:validate(schema)) + +json = [[{"array":[1,"string",true,false,[3,4],{"foo":"bar"},null]}]] +doc = heka_json.parse(json) +assert(doc) +assert(doc:type() == "object") +a = doc:find("array"); +assert(doc:type(a) == "array") +str = doc:find(a, 1) +assert("string" == doc:type(str), doc:type(str)) +tmp = doc:find("array", 3); +assert("boolean" == doc:type(tmp), doc:type(tmp)) +ok, err = pcall(doc.iter, doc, tmp) +assert("iter() not allowed on a primitive type", err, err) +ok, err = pcall(doc.find, doc, doc1, "array") +assert("invalid value" == err, err) +tmp = doc:find("array", "key") +assert(not tmp) +tmp = doc:find("array", 5, 0) +assert(not tmp) +tmp = doc:find("array", 7) +assert(not tmp) +tmp = doc:find(true) +assert(not tmp) +tmp = doc:find("array", 5, "missing") +assert(not tmp) + +ra = { + {value = 1, type = "number"}, + {value = "string", type = "string", len = 6}, + {value = true, type = "boolean"}, + {value = false, type = "boolean"}, + {type = "array", len = 2}, + {type = "object", len = 1}, + {type = "null"}, +} + +for i,v in doc:iter(a) do + assert(ra[i+1].type == doc:type(v)) + if ra[i+1].value ~= nil then + assert(ra[i+1].value == doc:value(v)) + else + ok, err = pcall(doc.value, doc, v) + if ra[i+1].type == "null" then + assert(ok) + else + assert(not ok) + end + end + if ra[i+1].len then + assert(ra[i+1].len == doc:size(v)) + else + ok, err = pcall(doc.size, doc, v) + assert(not ok) + end +end + +a0 = doc:find("array", 0); +assert(a0) +assert(doc:value(a0) == 1) + +a5 = doc:find("array", 5); +assert(a5) +assert(doc:type(a5) == "object") +assert(doc:size(a5) == 1) +for k,v in doc:iter(a5) do + assert(doc:type(v) == "string") + assert(doc:value(v) == "bar") + assert(k == "foo", tostring(k)) +end + +a5x = doc:find("array", 5); +assert(a5 == a5x) + +ok, doc = pcall(heka_json.parse_message, 1) +assert("bad argument #0 to '?' (invalid number of arguments)" == doc, doc) + +ok, doc = pcall(heka_json.parse_message, "", "Fields[json]") +assert("bad argument #1 to '?' (mozsvc.heka_stream_reader expected, got string)" == doc, doc) + +hsr = heka_stream_reader.new("test") +hsr:decode_message("\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\4\82\23\10\4\106\115\111\110\16\1\42\13\123\34\102\111\111\34\58\34\98\97\114\34\125") +ok, doc = pcall(heka_json.parse_message, hsr, "Fields[missing]") +assert("field not found" == doc, doc) + +ok, doc = pcall(heka_json.parse_message, hsr, "Fields[missing]", "") +assert("bad argument #3 to '?' (number expected, got string)" == doc, doc) + +ok, doc = pcall(heka_json.parse_message, hsr, "Fields[missing]", 0, "") +assert("bad argument #4 to '?' (number expected, got string)" == doc, doc) + +ok, err = pcall(heka_json.parse_message, hsr, "Fields[json") +assert("field not found" == err, err) + +ok, err = pcall(heka_json.parse_message, hsr, "Fieldsjson]") +assert("field not found" == err, err) + +ok, err = pcall(heka_json.parse_message, hsr, "foo") +assert("field not found" == err, err) + +ok, doc = pcall(heka_json.parse_message, hsr, "Fields[json]") +assert(ok, doc) + +foo = doc:find("foo") +assert(foo) +assert("bar" == doc:value(foo), tostring(doc:value(foo))) +ok, err = pcall(doc.value, doc, (doc1:find())) +assert(not ok) +assert("invalid value" == err, err) + +assert("object" == doc:type(), doc:type()); +rdoc = doc:remove() +assert("null" == doc:type(), doc:type()); +foo = rdoc:find("foo") +assert("bar" == rdoc:value(foo), tostring(doc:value(foo))) + +assert("object" == rdoc:type(), rdoc:type()); +nrdoc = rdoc:remove("foo") +assert("object" == rdoc:type(), rdoc:type()); + +nested = '{"main":{"m1":1}, "payload":{"values":[1,2,3]}}' +json = heka_json.parse(nested) +assert(json) +assert(json:size() == 2, json:size()) +values = json:find("payload", "values") +assert(values) +vit = json:iter(values) +i,v = vit() +assert(i == 0, tostring(i)) +assert(json:value(v) == 1, tostring(json:value(v))) +i,v = vit() +json:remove("payload") +assert(i == 1, tostring(i)) +assert(json:value(v) == 2, tostring(json:value(v))) +i,v = vit() +assert(i == 2, tostring(i)) +assert(json:value(v) == 3, tostring(json:value(v))) +i,v = vit() +assert(i == nil, tostring(i)) + +json = heka_json.parse(nested) +assert(json) +object = json:find("main") +vit = json:iter(object) +json:remove("main") +ok, err = pcall(vit) +assert(err == "iterator has been invalidated") + +json = heka_json.parse(nested) +assert(json) +values = json:find("payload", "values") +vit = json:iter(values) +rvalues = json:remove("payload", "values") +ok, err = pcall(vit) +assert(err == "iterator has been invalidated") +schema_array = [[{ +"type":"array", +"minItems": 1, +"oneOf": [{"items": {"type":"number"}}] +}]] +sa = heka_json.parse_schema(schema_array) +assert(sa) +assert(rvalues:validate(sa)) + +gz_nested = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139\8\0\0\0\0\0\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" +hsr:decode_message(gz_nested) +ok, json = pcall(heka_json.parse_message, hsr, "Payload") +assert(ok, json) +values = json:find("payload", "values") +assert(values) + +gz_corrupt = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139foobar\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" +hsr:decode_message(gz_corrupt) +ok, json = pcall(heka_json.parse_message, hsr, "Payload") +assert("lsb_ungzip failed" == json, json) + +minimal = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0" +hsr:decode_message(minimal) +ok, err = pcall(heka_json.parse_message, hsr, "Payload") +assert("field not found" == err, err) + +invalid_json = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\50\001{" +hsr:decode_message(invalid_json) +ok, err = pcall(heka_json.parse_message, hsr, "Payload") +assert("failed to parse offset:1 Missing a name for object member." == err, err) + +short_json = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\50\1\57" +hsr:decode_message(short_json) +ok, doc = pcall(heka_json.parse_message, hsr, "Payload") +assert(ok, doc) +assert("number" == doc:type(), doc:type()) +assert(9 == doc:value(), tostring(doc:value())) +rv = doc:remove("foo") +assert(not rv) diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index 3496926..5a300e1 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -4,14 +4,24 @@ require "string" require "bloom_filter"; +require "heka_json" local bf = bloom_filter.new(10, 0.01) +local json = [[{"foo":"bar"}]] +local doc = heka_json.parse(json) +local doc1 = heka_json.parse(json) +assert(doc) +local large_json = string.format('{"foo":"%s"}', string.rep("x", 66000)) +local large_doc = heka_json.parse(large_json) +assert(large_doc) + -- Table tests local msgs = { {{Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, nil}, {{Timestamp = 1, Uuid = "00000000-0000-0000-0000-000000000000"}, nil}, {{Timestamp = 2, Uuid = "00000000-0000-0000-0000-000000000000", Logger = "logger", Hostname = "hostname", Type = "type", Payload = "payload", EnvVersion = "envversion", Pid = 99, Severity = 5}, 99}, {{Timestamp = 3, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}}, "foo.log:123"}, + {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc:make_field()}}, nil}, } local err_msgs = { @@ -29,7 +39,10 @@ local err_msgs = { {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=1}}}, err = "inject_message() failed: invalid boolean value_type: 1"}, {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, - {msg = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}}, err = "inject_message() failed: user data object does not implement lsb_output"}, + {msg = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}}, err = "inject_message() failed: userdata object does not implement lsb_output"}, + {msg = {Timestamp = 1e9, Fields = {json = {value = doc:find()}}}, err = "inject_message() failed: a lightuserdata output must also specify a userdata value"}, + {msg = {Timestamp = 1e9, Fields = {json = {value = doc1:find(), userdata = doc}}}, err = "inject_message() failed: userdata output callback failed: 1"}, + {msg = {Timestamp = 1e9, Fields = {json = large_doc:make_field()}}, err = "inject_message() failed: userdata output callback failed: 1"}, } for i, v in ipairs(msgs) do @@ -111,6 +124,7 @@ ok, err = pcall(hsr.read_message, 1, 2) assert(not ok) assert("bad argument #1 to '?' (mozsvc.heka_stream_reader expected, got number)" == err, string.format("received: %s", err)) + -- String tests inject_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\005") @@ -123,5 +137,3 @@ ok, err = pcall(inject_message, {}) if ok then error(string.format("test should have failed")) end eerr = "inject_message() failed: rejected by the callback" assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) - - diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 979efc0..8b1d206 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -50,6 +50,7 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 156, .cp_numeric = NAN, .cp_string = "foo.log:123" }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, }; @@ -526,8 +527,8 @@ static char* test_im_input() "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); - mu_assert(305 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(7 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(350 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -597,6 +598,18 @@ static char* test_read_message() } +static char* test_heka_json() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/heka_json.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + static char* benchmark_decode_message() { int iter = 100000; @@ -641,6 +654,7 @@ static char* all_tests() mu_run_test(test_encode_message); mu_run_test(test_decode_message); mu_run_test(test_read_message); + mu_run_test(test_heka_json); mu_run_test(benchmark_decode_message); return NULL; diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index c04b7a1..ae96542 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -21,6 +21,10 @@ if(LIBM_LIBRARY) target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) endif() +if(ZLIB_FOUND) + target_link_libraries(luasandboxutil ${ZLIB_LIBRARIES}) +endif() + install(TARGETS luasandboxutil DESTINATION ${LIB_DIR} COMPONENT core) add_subdirectory(test) diff --git a/src/util/util.c b/src/util/util.c index c97cc30..12b7578 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -13,6 +13,10 @@ #include #include +#ifdef HAVE_ZLIB +#include +#endif + #ifdef _WIN32 #include #include @@ -80,7 +84,7 @@ unsigned long long lsb_get_time() static unsigned long long convert = 0; if (convert == 0) { mach_timebase_info_data_t tbi; - (void) mach_timebase_info(&tbi); + (void)mach_timebase_info(&tbi); convert = tbi.numer / tbi.denom; } return mach_absolute_time() * convert; @@ -89,12 +93,12 @@ unsigned long long lsb_get_time() static_assert(sizeof(LARGE_INTEGER) == sizeof qpf, "size mismatch"); unsigned long long t; - if (qpf == ULLONG_MAX) QueryPerformanceFrequency((LARGE_INTEGER*)&qpf); - if (qpf){ - QueryPerformanceCounter((LARGE_INTEGER*)&t); + if (qpf == ULLONG_MAX) QueryPerformanceFrequency((LARGE_INTEGER *)&qpf); + if (qpf) { + QueryPerformanceCounter((LARGE_INTEGER *)&t); return (t / qpf * 1000000000ULL) + ((t % qpf) * 1000000000ULL / qpf); } else { - GetSystemTimeAsFileTime((FILETIME*)&t); + GetSystemTimeAsFileTime((FILETIME *)&t); return t * 100ULL; } #else @@ -103,3 +107,58 @@ unsigned long long lsb_get_time() return tv.tv_sec * 1000000000ULL + tv.tv_usec * 1000ULL; #endif } + + +#ifdef HAVE_ZLIB +char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len) +{ + if (!s || !r_len) { + return NULL; + } + *r_len = 0; + size_t buf_len = 2 * s_len; + unsigned char *buf = malloc(buf_len); + if (!buf) { + return NULL; + } + + z_stream strm; + strm.zalloc = Z_NULL; + strm.zfree = Z_NULL; + strm.opaque = Z_NULL; + strm.avail_in = s_len; + strm.next_in = (unsigned char *)s; + strm.avail_out = buf_len; + strm.next_out = buf; + + int ret = inflateInit2(&strm, 16 + MAX_WBITS); + if (ret != Z_OK) { + free(buf); + return NULL; + } + + do { + if (ret == Z_BUF_ERROR) { + buf_len *= 2; + unsigned char *tmp = realloc(buf, buf_len); + if (!tmp) { + free(buf); + return NULL; + } else { + buf = tmp; + strm.avail_out = buf_len - strm.total_out; + strm.next_out = buf + strm.total_out; + } + } + ret = inflate(&strm, Z_FINISH); + } while (ret == Z_BUF_ERROR && strm.avail_in > 0); + + inflateEnd(&strm); + if (ret != Z_STREAM_END) { + free(buf); + return NULL; + } + *r_len = strm.total_out; + return (char *)buf; +} +#endif From 1eaf94decad3f24da4fc50f15f9dd9384939a55b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 24 Feb 2016 14:24:24 -0800 Subject: [PATCH 106/235] Fix issues #113 #114 - Refine the heka_json API and update the documentation - Add rapidjson to the build --- cmake/externals.cmake | 9 ++++ docs/heka/heka_json.md | 78 +++++++++++++++++-------------- docs/heka/index.md | 1 + include/luasandbox/util/util.h | 2 +- src/heka/rapidjson.cpp | 68 +++++++++++++++++++++------ src/heka/stream_reader.c | 5 +- src/heka/test/lua/heka_json.lua | 7 +++ src/heka/test/lua/iim.lua | 12 ++++- src/heka/test/test_sandbox.c | 6 ++- src/util/heka_message.c | 2 +- src/util/input_buffer.c | 2 +- src/util/test/test_heka_message.c | 2 + src/util/util.c | 5 +- 13 files changed, 136 insertions(+), 63 deletions(-) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index c64e13b..7206318 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -151,3 +151,12 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) + +externalproject_add( + rapidjson + GIT_REPOSITORY https://github.com/miloyip/rapidjson.git + GIT_TAG ed7efe6289cfa1fafc354aaa8a29fcd31c1607fd + BUILD_COMMAND ${CMAKE_COMMAND} -E echo "no build" + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "no install" +) +include_directories("${EP_BASE}/Source/rapidjson/include") diff --git a/docs/heka/heka_json.md b/docs/heka/heka_json.md index 2b083fd..9536e87 100644 --- a/docs/heka/heka_json.md +++ b/docs/heka/heka_json.md @@ -85,28 +85,32 @@ assert(ok, err) * ok (bool) - true if valid * err (string) - error message on failure -#### type +#### find -Returns the type of the value in the JSON structure. +Searches for and returns a value in the JSON structure. ```lua -local t = doc:type() -assert(t == "object", t) +local v = doc:find("obj", "arr", 0, "foo") +assert(v, "not found") ``` *Arguments* * value (lightuserdata) - optional, when not specified the function is applied to document +* key (string, number) - object key, or array index +* keyN (string, number) - final object key, or array index *Return* -* type (string) - "string", "number", "boolean", "object", "array" or "null" +* value (lightuserdata) - handle to be passed to other methods, nil if not found -#### find +#### remove -Searches for and returns a value in the JSON structure. +Searches for and NULL's out the resulting value in the JSON structure returning the +extracted Heka JSON document. ```lua -local v = doc:find("obj", "arr", 0, "foo") -assert(v, "not found") +local rv = doc:remove("obj", "arr") +assert(rv, "not found") +rv:size() -- number of elements in the extracted array ``` *Arguments* @@ -115,7 +119,7 @@ assert(v, "not found") * keyN (string, number) - final object key, or array index *Return* -* value (lightuserdata) - handle to be passed to other methods, nil if not found +* doc (userdata) - the object removed from the original JSON or nil #### value @@ -128,11 +132,29 @@ assert("bar" == str, tostring(str)) ``` *Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document +* value (lightuserdata, nil) - optional, when not specified the function is applied to document + (accepts nil for easier nesting without having to test the inner expression) + e.g., str = doc:value(doc:find("foo")) or "my default" *Return* * primitive - string, number, bool, nil or throws an error if not convertable (object, array) +#### type + +Returns the type of the value in the JSON structure. + +```lua +local t = doc:type() +assert(t == "object", t) + +``` +*Arguments* +* value (lightuserdata, nil) - optional, when not specified the function is applied to document + (accepts nil for easier nesting without having to test the inner expression) + +*Return* +* type (string, nil) - "string", "number", "boolean", "object", "array" or "null" + #### iter Retrieves an interator function for an object/array. @@ -144,10 +166,11 @@ for i,v in doc:iter(v) do end ``` *Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document +* value (lightuserdata, nil) - optional, when not specified the function is applied to document + (accepts nil for API consistency) *Return* -* iter (function) - iterator function returning an index/value for arrays or a key/value for +* iter (function, nil) - iterator function returning an index/value for arrays or a key/value for objects. Throws an error on primitive types. #### size @@ -159,10 +182,11 @@ local n = doc:size(v) ``` *Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document +* value (lightuserdata, nil) - optional, when not specified the function is applied to document + (accepts nil for easier nesting without having to test the inner expression) *Return* -* size (number) - Number of element in an array/object or the length of the string. +* size (number, nil) - Number of element in an array/object or the length of the string. Throws an error on numeric, boolean and null types. #### make_field @@ -177,26 +201,8 @@ inject_message(msg) ``` *Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document - -*Return* -* table - shorthand for `{value = v, userdata = doc}` - -#### remove - -Searches for and NULL's out the resulting value in the JSON structure returning the -extracted Heka JSON document. - -```lua -local rv = doc:remove("obj", "arr") -assert(rv, "not found") -rv:size() -- size of the extracted array - -``` -*Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document -* key (string, number) - object key, or array index -* keyN (string, number) - final object key, or array index +* value (lightuserdata, nil) - optional, when not specified the function is applied to document + (accepts nil for easier nesting without having to test the inner expression) *Return* -* doc (userdata) - the object removed from the original JSON or nil +* field (table, nil) - i.e., `{value = v, userdata = doc}` diff --git a/docs/heka/index.md b/docs/heka/index.md index 1fd5193..dae722b 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -51,6 +51,7 @@ There are a few intentional changes between tho original Heka sandbox and this v #### Output Sandbox 1. [update_checkpoint](output.md#update_checkpoint) was added for batch and asynchronous processing. +1. A [Heka JSON](heka_json.md) Lua module was added. ### Removals diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 8858cd1..957c6bb 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -69,7 +69,7 @@ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); * * @param s String to ungzip * @param s_len Length of the string to ungzip - * @param r_len Length of the returned string + * @param r_len (optional) Length of the returned string * * @return char* Returned string (MUST be freed by the caller), NULL on failure */ diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp index 0b34637..6ebb0c0 100644 --- a/src/heka/rapidjson.cpp +++ b/src/heka/rapidjson.cpp @@ -27,6 +27,7 @@ typedef struct heka_json { rj::Document *doc; rj::Value *val; + char *insitu; std::set *refs; } heka_json; @@ -59,8 +60,14 @@ static rj::Value* check_value(lua_State *lua) (luaL_checkudata(lua, 1, mozsvc_heka_json)); rj::Value *v = static_cast(lua_touserdata(lua, 2)); if (!v) { - luaL_checktype(lua, 2, LUA_TNONE); // only default it when it is not set - v = hj->doc ? hj->doc : hj->val; + int t = lua_type(lua, 2); + if (t == LUA_TNONE) { + v = hj->doc ? hj->doc : hj->val; + } else if (t == LUA_TNIL) { + v = NULL; + } else { + luaL_checktype(lua, 2, LUA_TLIGHTUSERDATA); + } } else if (hj->refs->find(v) == hj->refs->end()) { luaL_error(lua, "invalid value"); } @@ -93,6 +100,7 @@ static int hj_gc(lua_State *lua) delete(hj->refs); delete(hj->doc); delete(hj->val); + free(hj->insitu); return 0; } @@ -129,6 +137,7 @@ static int hj_parse(lua_State *lua) heka_json *hj = static_cast(lua_newuserdata(lua, sizeof*hj)); hj->doc = new rj::Document; hj->val = NULL; + hj->insitu = NULL; hj->refs = new std::set; luaL_getmetatable(lua, mozsvc_heka_json); lua_setmetatable(lua, -2); @@ -237,6 +246,10 @@ static int hj_find(lua_State *lua) static int hj_type(lua_State *lua) { rj::Value *v = check_value(lua); + if (!v) { + lua_pushnil(lua); + return 1; + } switch (v->GetType()) { case rj::kStringType: @@ -266,6 +279,10 @@ static int hj_type(lua_State *lua) static int hj_size(lua_State *lua) { rj::Value *v = check_value(lua); + if (!v) { + lua_pushnil(lua); + return 1; + } switch (v->GetType()) { case rj::kStringType: @@ -292,6 +309,10 @@ static int hj_size(lua_State *lua) static int hj_make_field(lua_State *lua) { rj::Value *v = check_value(lua); + if (!v) { + lua_pushnil(lua); + return 1; + } lua_createtable(lua, 0, 2); lua_pushlightuserdata(lua, (void *)v); @@ -359,6 +380,10 @@ static int hj_array_iter(lua_State *lua) static int hj_value(lua_State *lua) { rj::Value *v = check_value(lua); + if (!v) { + lua_pushnil(lua); + return 1; + } switch (v->GetType()) { case rj::kStringType: @@ -388,6 +413,10 @@ static int hj_value(lua_State *lua) static int hj_iter(lua_State *lua) { rj::Value *v = check_value(lua); + if (!v) { + lua_pushnil(lua); + return 1; + } switch (v->GetType()) { case rj::kObjectType: @@ -480,15 +509,14 @@ static int hj_parse_message(lua_State *lua) lsb_const_string json = read_message(lua, idx, msg); if (!json.s) return luaL_error(lua, "field not found"); -#ifdef HAVE_ZLIB char *inflated = NULL; +#ifdef HAVE_ZLIB // automatically handle gzipped strings (optimization for Mozilla telemetry // messages) if (json.len > 2) { if (json.s[0] == 0x1f && (unsigned char)json.s[1] == 0x8b) { - inflated = lsb_ungzip(json.s, json.len, &json.len); + inflated = lsb_ungzip(json.s, json.len, NULL); if (!inflated) return luaL_error(lua, "lsb_ungzip failed"); - json.s = inflated; } } #endif @@ -496,6 +524,7 @@ static int hj_parse_message(lua_State *lua) heka_json *hj = static_cast(lua_newuserdata(lua, sizeof*hj)); hj->doc = new rj::Document; hj->val = NULL; + hj->insitu = inflated; hj->refs = new std::set; luaL_getmetatable(lua, mozsvc_heka_json); lua_setmetatable(lua, -2); @@ -506,7 +535,15 @@ static int hj_parse_message(lua_State *lua) } bool err = false; - { // allows ms to be destroyed before the longjmp + if (hj->insitu) { + if (hj->doc->ParseInsitu(hj->insitu) + .HasParseError()) { + err = true; + lua_pushfstring(lua, "failed to parse offset:%f %s", + (lua_Number)hj->doc->GetErrorOffset(), + rj::GetParseError_En(hj->doc->GetParseError())); + } + } else { rj::MemoryStream ms(json.s, json.len); if (hj->doc->ParseStream<0, rj::UTF8<> >(ms).HasParseError()) { err = true; @@ -516,10 +553,6 @@ static int hj_parse_message(lua_State *lua) } } -#ifdef HAVE_ZLIB - if (inflated) free(inflated); -#endif - if (err) return lua_error(lua); hj->refs->insert(hj->doc); return 1; @@ -531,7 +564,7 @@ class OutputBufferWrapper { typedef char Ch; OutputBufferWrapper(lsb_output_buffer *ob) : ob_(ob), err_(NULL) { } #if _BullseyeCoverage - #pragma BullseyeCoverage off +#pragma BullseyeCoverage off #endif Ch Peek() const { assert(false); return '\0'; } Ch Take() { assert(false); return '\0'; } @@ -539,7 +572,7 @@ class OutputBufferWrapper { Ch* PutBegin() { assert(false); return 0; } size_t PutEnd(Ch *) { assert(false); return 0; } #if _BullseyeCoverage - #pragma BullseyeCoverage on +#pragma BullseyeCoverage on #endif void Put(Ch c) { @@ -562,11 +595,15 @@ static int output_heka_json(lua_State *lua) (lua_touserdata(lua, -1)); heka_json *hj = static_cast(lua_touserdata(lua, -2)); rj::Value *v = static_cast(lua_touserdata(lua, -3)); - if (!(ob && hj && v)) { + if (!(ob && hj)) { return 1; } - if (hj->refs->find(v) == hj->refs->end()) { - return 1; + if (!v) { + v = hj->doc ? hj->doc : hj->val; + } else { + if (hj->refs->find(v) == hj->refs->end()) { + return 1; + } } OutputBufferWrapper obw(ob); rapidjson::Writer writer(obw); @@ -590,6 +627,7 @@ static int hj_remove(lua_State *lua) heka_json *nv = static_cast(lua_newuserdata(lua, sizeof*nv)); nv->doc = NULL; nv->val = new rj::Value; + nv->insitu = NULL; nv->refs = new std::set; luaL_getmetatable(lua, mozsvc_heka_json); lua_setmetatable(lua, -2); diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c index b5f258b..d288801 100644 --- a/src/heka/stream_reader.c +++ b/src/heka/stream_reader.c @@ -165,7 +165,7 @@ static int hsr_find_message(lua_State *lua) } if (b->msglen) { need = b->msglen + (size_t)b->buf[b->scanpos + 1] + LSB_HDR_FRAME_SIZE - - b->readpos; + (b->readpos - b->scanpos); } else { need = b->scanpos + b->size - b->readpos; } @@ -176,7 +176,8 @@ static int hsr_find_message(lua_State *lua) lua_pushinteger(lua, 0); } else { if (lsb_expand_input_buffer(&hsr->buf, need)) { - luaL_error(lua, "%s buffer reallocation failed", hsr->name); + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); } size_t nread = fread(hsr->buf.buf + hsr->buf.readpos, 1, diff --git a/src/heka/test/lua/heka_json.lua b/src/heka/test/lua/heka_json.lua index 2b6f418..6b7d8dd 100644 --- a/src/heka/test/lua/heka_json.lua +++ b/src/heka/test/lua/heka_json.lua @@ -296,3 +296,10 @@ assert("number" == doc:type(), doc:type()) assert(9 == doc:value(), tostring(doc:value())) rv = doc:remove("foo") assert(not rv) + +assert(nil == doc:find(nil)) +assert(nil == doc:find(nil)) +assert(nil == doc:size(nil)) +assert(nil == doc:type(nil)) +assert(nil == doc:make_field(nil)) +assert(nil == doc:iter(nil)) diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index 5a300e1..6044361 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -9,8 +9,9 @@ local bf = bloom_filter.new(10, 0.01) local json = [[{"foo":"bar"}]] local doc = heka_json.parse(json) -local doc1 = heka_json.parse(json) assert(doc) +local doc1 = heka_json.parse(json) +local doc2 = heka_json.parse(json) local large_json = string.format('{"foo":"%s"}', string.rep("x", 66000)) local large_doc = heka_json.parse(large_json) assert(large_doc) @@ -21,7 +22,9 @@ local msgs = { {{Timestamp = 1, Uuid = "00000000-0000-0000-0000-000000000000"}, nil}, {{Timestamp = 2, Uuid = "00000000-0000-0000-0000-000000000000", Logger = "logger", Hostname = "hostname", Type = "type", Payload = "payload", EnvVersion = "envversion", Pid = 99, Severity = 5}, 99}, {{Timestamp = 3, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}}, "foo.log:123"}, - {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc:make_field()}}, nil}, + {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc:make_field()}}, nil}, -- use the lightuserdata + {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc}}, nil}, -- use the full userdata + {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc2:remove()}}, nil}, -- use the removed item } local err_msgs = { @@ -93,6 +96,11 @@ assert(true == found) assert(25 == consumed, string.format("expected: 25 received %d", consumed)) assert(0 < need, string.format("expected: >0 received: %d", need)) +found, consumed, need = hsr:find_message("\0\0\0\0\0\030\002\008\020\031") +assert(false == found) +assert(5 == consumed, string.format("expected: 5 received %d", consumed)) +assert(20 == need, string.format("expected: 20 received: %d", need)) + local found, consumed, need = hsr:find_message("") assert(not found) diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 8b1d206..4e799c1 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -51,6 +51,8 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 156, .cp_numeric = NAN, .cp_string = "foo.log:123" }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, }; @@ -527,8 +529,8 @@ static char* test_im_input() "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(7 == stats.im_cnt, "received %llu", stats.im_cnt); - mu_assert(350 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(9 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(440 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 198eb5d..5c2b999 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -346,7 +346,7 @@ bool lsb_find_heka_message(lsb_heka_message *m, return false; // header length is not buf } - size_t hlen = ib->buf[ib->scanpos + 1]; + size_t hlen = (unsigned char)ib->buf[ib->scanpos + 1]; size_t hend = ib->scanpos + hlen + 3; if (hend > ib->readpos) { return false; // header is not in buf diff --git a/src/util/input_buffer.c b/src/util/input_buffer.c index a409a91..c17ecbc 100644 --- a/src/util/input_buffer.c +++ b/src/util/input_buffer.c @@ -65,7 +65,7 @@ lsb_err_value lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) if (newsize > b->maxsize) return LSB_ERR_UTIL_FULL; newsize = lsb_lp2(newsize); - if (newsize > b->maxsize) newsize = b->maxsize;; + if (newsize > b->maxsize) newsize = b->maxsize; char *tmp = realloc(b->buf, newsize); if (tmp) { b->buf = tmp; diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 490d24c..efe7551 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -157,9 +157,11 @@ static char* test_find_message() size_t d; }; +#define ONE_TWENTY_SIX "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" #define add_input(str, result, discard) {.s = {.s = str, .len = sizeof str - 1}, .b = result, .d = discard}, struct find_message tests[] = { add_input("\x1e\x02\x08\x14\x1f" TEST_UUID TEST_NS, true, 0) // full message + add_input("\x1e\x80\x08\x14" ONE_TWENTY_SIX "\x1f" TEST_UUID TEST_NS, true, 0) // large header add_input("\x1e", false, 0) add_input("\x02", false, 0) add_input("\x08", false, 0) diff --git a/src/util/util.c b/src/util/util.c index 12b7578..a4a6e6c 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -112,10 +112,9 @@ unsigned long long lsb_get_time() #ifdef HAVE_ZLIB char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len) { - if (!s || !r_len) { + if (!s) { return NULL; } - *r_len = 0; size_t buf_len = 2 * s_len; unsigned char *buf = malloc(buf_len); if (!buf) { @@ -158,7 +157,7 @@ char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len) free(buf); return NULL; } - *r_len = strm.total_out; + if (r_len) *r_len = strm.total_out; return (char *)buf; } #endif From a94369598140f88e6c8f745d1ce1209e8520730c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 26 Feb 2016 06:49:10 -0800 Subject: [PATCH 107/235] Add an optional limit on gzip inflation - Always max_messages_size for the Heka sandbox --- include/luasandbox/util/util.h | 6 ++++-- src/heka/rapidjson.cpp | 13 ++++++++++--- src/heka/rapidjson_impl.h | 4 ++-- src/heka/test/lua/heka_json.lua | 5 +++++ src/heka/test/test_sandbox.c | 1 + src/util/util.c | 14 ++++++++++++-- 6 files changed, 34 insertions(+), 9 deletions(-) diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 957c6bb..f55fcfe 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -69,11 +69,13 @@ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); * * @param s String to ungzip * @param s_len Length of the string to ungzip + * @param max_len Maximum length the buffer is allowed to grow to (0 for + * unlimited) * @param r_len (optional) Length of the returned string - * * @return char* Returned string (MUST be freed by the caller), NULL on failure */ -LSB_UTIL_EXPORT char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len); +LSB_UTIL_EXPORT char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, + size_t *r_len); #endif diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp index 6ebb0c0..edf2434 100644 --- a/src/heka/rapidjson.cpp +++ b/src/heka/rapidjson.cpp @@ -515,7 +515,8 @@ static int hj_parse_message(lua_State *lua) // messages) if (json.len > 2) { if (json.s[0] == 0x1f && (unsigned char)json.s[1] == 0x8b) { - inflated = lsb_ungzip(json.s, json.len, NULL); + size_t mms = (size_t)lua_tointeger(lua, lua_upvalueindex(2)); + inflated = lsb_ungzip(json.s, json.len, mms, NULL); if (!inflated) return luaL_error(lua, "lsb_ungzip failed"); } } @@ -705,10 +706,16 @@ int luaopen_heka_json(lua_State *lua) luaL_register(lua, NULL, hjlib_m); luaL_register(lua, mozsvc_heka_json_table, hjlib_f); // special case parse_message since it needs access to the sandbox + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) != LUA_TTABLE) { + return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); + } lsb_lua_sandbox *lsb = static_cast (lua_touserdata(lua, lua_upvalueindex(1))); lua_pushlightuserdata(lua, static_cast(lsb)); - lua_pushcclosure(lua, hj_parse_message, 1); - lua_setfield(lua, -2, "parse_message"); + lua_getfield(lua, -2, LSB_HEKA_MAX_MESSAGE_SIZE); + lua_pushcclosure(lua, hj_parse_message, 2); + lua_setfield(lua, -3, "parse_message"); + lua_pop(lua, 1); // remove LSB_CONFIG_TABLE return 1; } diff --git a/src/heka/rapidjson_impl.h b/src/heka/rapidjson_impl.h index 10f8cf8..3e9003f 100644 --- a/src/heka/rapidjson_impl.h +++ b/src/heka/rapidjson_impl.h @@ -14,9 +14,9 @@ extern "C" { #endif -#include "lua.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" #include "luasandbox/util/heka_message.h" -#include "lauxlib.h" extern const char *mozsvc_heka_json; extern const char *mozsvc_heka_json_table; diff --git a/src/heka/test/lua/heka_json.lua b/src/heka/test/lua/heka_json.lua index 6b7d8dd..8ee1367 100644 --- a/src/heka/test/lua/heka_json.lua +++ b/src/heka/test/lua/heka_json.lua @@ -273,6 +273,11 @@ assert(ok, json) values = json:find("payload", "values") assert(values) +gz_too_large = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\51\31\139\8\0\0\0\0\0\0\3\237\193\49\13\0\0\8\3\176\31\25\147\129\178\201\71\7\73\219\52\155\2\0\0\0\0\0\0\0\0\255\100\14\116\172\116\102\0\36\0\0" +hsr:decode_message(gz_too_large) +ok, json = pcall(heka_json.parse_message, hsr, "Payload") +assert("lsb_ungzip failed" == json, json) + gz_corrupt = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139foobar\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" hsr:decode_message(gz_corrupt) ok, json = pcall(heka_json.parse_message, hsr, "Payload") diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 4e799c1..1210fda 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -604,6 +604,7 @@ static char* test_heka_json() { lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/heka_json.lua", NULL, + "max_message_size = 8196\n" "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); diff --git a/src/util/util.c b/src/util/util.c index a4a6e6c..3375043 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -110,12 +110,15 @@ unsigned long long lsb_get_time() #ifdef HAVE_ZLIB -char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len) +char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len) { - if (!s) { + if (!s || (max_len && s_len > max_len)) { return NULL; } size_t buf_len = 2 * s_len; + if (max_len && buf_len > max_len) { + buf_len = max_len; + } unsigned char *buf = malloc(buf_len); if (!buf) { return NULL; @@ -138,7 +141,14 @@ char* lsb_ungzip(const char *s, size_t s_len, size_t *r_len) do { if (ret == Z_BUF_ERROR) { + if (max_len && buf_len == max_len) { + free(buf); + return NULL; + } buf_len *= 2; + if (max_len && buf_len > max_len) { + buf_len = max_len; + } unsigned char *tmp = realloc(buf, buf_len); if (!tmp) { free(buf); From 251bad95a53dd2f63b113bb61f04c8553fcf1709 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Fri, 26 Feb 2016 10:09:00 -0800 Subject: [PATCH 108/235] Update schema_validation to work on Windows --- cmake/mozsvc.cmake | 2 +- src/heka/rapidjson.cpp | 2 +- src/heka/stream_reader_impl.h | 10 +++++++++- src/heka/test/lua/heka_json.lua | 34 +++++++++++++++++---------------- src/heka/test/test_sandbox.c | 3 +++ src/util/util.c | 8 ++++---- 6 files changed, 36 insertions(+), 23 deletions(-) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 42137fb..7da0435 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -10,7 +10,7 @@ if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") # enable C++ exception handling - set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /EHsc") + set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /EHs") # debug multi threaded dll runtime, complete debugging info, runtime error checking set(CMAKE_C_FLAGS_DEBUG "/MDd /Zi /RTC1") diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp index edf2434..80f10f5 100644 --- a/src/heka/rapidjson.cpp +++ b/src/heka/rapidjson.cpp @@ -338,7 +338,7 @@ static int hj_object_iter(lua_State *lua) rj::Value *next = &(*hoi->it)->value; hj->refs->insert(next); lua_pushlstring(lua, (*hoi->it)->name.GetString(), - (lua_Number)(*hoi->it)->name.GetStringLength()); + (size_t)(*hoi->it)->name.GetStringLength()); lua_pushlightuserdata(lua, next); ++*hoi->it; } else { diff --git a/src/heka/stream_reader_impl.h b/src/heka/stream_reader_impl.h index abc2cf5..f368435 100644 --- a/src/heka/stream_reader_impl.h +++ b/src/heka/stream_reader_impl.h @@ -9,8 +9,12 @@ #ifndef luasandbox_heka_stream_reader_h_ #define luasandbox_heka_stream_reader_h_ -#include "luasandbox/lua.h" +#ifdef __cplusplus +extern "C" +{ +#endif +#include "luasandbox/lua.h" #include "luasandbox/util/input_buffer.h" #include "luasandbox/util/heka_message.h" @@ -29,4 +33,8 @@ typedef struct heka_stream_reader int luaopen_heka_stream_reader(lua_State *lua); +#ifdef __cplusplus +} +#endif + #endif diff --git a/src/heka/test/lua/heka_json.lua b/src/heka/test/lua/heka_json.lua index 8ee1367..e50f8c2 100644 --- a/src/heka/test/lua/heka_json.lua +++ b/src/heka/test/lua/heka_json.lua @@ -266,22 +266,24 @@ sa = heka_json.parse_schema(schema_array) assert(sa) assert(rvalues:validate(sa)) -gz_nested = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139\8\0\0\0\0\0\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" -hsr:decode_message(gz_nested) -ok, json = pcall(heka_json.parse_message, hsr, "Payload") -assert(ok, json) -values = json:find("payload", "values") -assert(values) - -gz_too_large = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\51\31\139\8\0\0\0\0\0\0\3\237\193\49\13\0\0\8\3\176\31\25\147\129\178\201\71\7\73\219\52\155\2\0\0\0\0\0\0\0\0\255\100\14\116\172\116\102\0\36\0\0" -hsr:decode_message(gz_too_large) -ok, json = pcall(heka_json.parse_message, hsr, "Payload") -assert("lsb_ungzip failed" == json, json) - -gz_corrupt = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139foobar\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" -hsr:decode_message(gz_corrupt) -ok, json = pcall(heka_json.parse_message, hsr, "Payload") -assert("lsb_ungzip failed" == json, json) +if read_config("have_zlib") then + gz_nested = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139\8\0\0\0\0\0\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" + hsr:decode_message(gz_nested) + ok, json = pcall(heka_json.parse_message, hsr, "Payload") + assert(ok, json) + values = json:find("payload", "values") + assert(values) + + gz_too_large = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\51\31\139\8\0\0\0\0\0\0\3\237\193\49\13\0\0\8\3\176\31\25\147\129\178\201\71\7\73\219\52\155\2\0\0\0\0\0\0\0\0\255\100\14\116\172\116\102\0\36\0\0" + hsr:decode_message(gz_too_large) + ok, json = pcall(heka_json.parse_message, hsr, "Payload") + assert("lsb_ungzip failed" == json, json) + + gz_corrupt = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139foobar\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" + hsr:decode_message(gz_corrupt) + ok, json = pcall(heka_json.parse_message, hsr, "Payload") + assert("lsb_ungzip failed" == json, json) +end minimal = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0" hsr:decode_message(minimal) diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 1210fda..5a396c2 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -604,6 +604,9 @@ static char* test_heka_json() { lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/heka_json.lua", NULL, +#ifdef HAVE_ZLIB + "have_zlib = true\n" +#endif "max_message_size = 8196\n" "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); diff --git a/src/util/util.c b/src/util/util.c index 3375043..90866f7 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -142,8 +142,8 @@ char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len) do { if (ret == Z_BUF_ERROR) { if (max_len && buf_len == max_len) { - free(buf); - return NULL; + ret = Z_MEM_ERROR; + break; } buf_len *= 2; if (max_len && buf_len > max_len) { @@ -151,8 +151,8 @@ char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len) } unsigned char *tmp = realloc(buf, buf_len); if (!tmp) { - free(buf); - return NULL; + ret = Z_MEM_ERROR; + break; } else { buf = tmp; strm.avail_out = buf_len - strm.total_out; From 72a170f5698a8264d6401f0385c7f4a3336ae576 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 26 Feb 2016 12:56:24 -0800 Subject: [PATCH 109/235] Remove the erroneous message clear and buffer reset from hsr_find_message - use the shorthand buffer pointer since it is available --- src/heka/stream_reader.c | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c index d288801..ae5702e 100644 --- a/src/heka/stream_reader.c +++ b/src/heka/stream_reader.c @@ -107,13 +107,8 @@ static int hsr_find_message(lua_State *lua) { int n = lua_gettop(lua); luaL_argcheck(lua, n > 1 && n < 4, 0, "incorrect number of arguments"); - heka_stream_reader *hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); lsb_input_buffer *b = &hsr->buf; - if (hsr->msg.raw.s) { - lsb_clear_heka_message(&hsr->msg); - b->msglen = b->readpos = b->scanpos = 0; - } FILE *fh = NULL; switch (lua_type(lua, 2)) { @@ -175,15 +170,15 @@ static int hsr_find_message(lua_State *lua) if (found) { lua_pushinteger(lua, 0); } else { - if (lsb_expand_input_buffer(&hsr->buf, need)) { + if (lsb_expand_input_buffer(b, need)) { return luaL_error(lua, "buffer reallocation failed\tname:%s", hsr->name); } - size_t nread = fread(hsr->buf.buf + hsr->buf.readpos, + size_t nread = fread(b->buf + b->readpos, 1, - hsr->buf.size - hsr->buf.readpos, + b->size - b->readpos, fh); - hsr->buf.readpos += nread; + b->readpos += nread; lua_pushnumber(lua, (lua_Number)nread); } } else { // update bytes needed From 6d789de8012e5765e721129143ea9407765a25f6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sat, 27 Feb 2016 12:44:32 -0800 Subject: [PATCH 110/235] Include read_message in the Heka output plugin API - add tests to verify the proper API for each Heka sandbox type - add the example plugins from Hindsight --- CMakeLists.txt | 2 +- docs/sandbox.md | 2 +- sandboxes/heka/analysis/throughput.lua | 45 ++++++++++ sandboxes/heka/input/heka_stdin.lua | 32 +++++++ sandboxes/heka/input/heka_tcp.lua | 89 ++++++++++++++++++++ sandboxes/heka/input/syslog_udp.lua | 98 ++++++++++++++++++++++ sandboxes/heka/output/debug.lua | 65 ++++++++++++++ sandboxes/heka/output/heka_log_rolling.lua | 52 ++++++++++++ sandboxes/heka/output/heka_tcp.lua | 60 +++++++++++++ sandboxes/heka/output/inject_payload.lua | 52 ++++++++++++ src/heka/sandbox.c | 2 +- src/heka/test/lua/analysis.lua | 9 ++ src/heka/test/lua/input.lua | 10 +++ src/heka/test/lua/output.lua | 9 ++ src/heka/test/test_sandbox.c | 6 +- 15 files changed, 527 insertions(+), 6 deletions(-) create mode 100644 sandboxes/heka/analysis/throughput.lua create mode 100644 sandboxes/heka/input/heka_stdin.lua create mode 100644 sandboxes/heka/input/heka_tcp.lua create mode 100644 sandboxes/heka/input/syslog_udp.lua create mode 100644 sandboxes/heka/output/debug.lua create mode 100644 sandboxes/heka/output/heka_log_rolling.lua create mode 100644 sandboxes/heka/output/heka_tcp.lua create mode 100644 sandboxes/heka/output/inject_payload.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index e91d730..7c44ebf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 14) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/docs/sandbox.md b/docs/sandbox.md index de79406..b14c12a 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -127,5 +127,5 @@ The best place to start is with some examples: ### Heka Sandbox -[Heka Sandbox](heka/index.html) +[Heka Sandbox](heka/index.md) diff --git a/sandboxes/heka/analysis/throughput.lua b/sandboxes/heka/analysis/throughput.lua new file mode 100644 index 0000000..5bbb2c2 --- /dev/null +++ b/sandboxes/heka/analysis/throughput.lua @@ -0,0 +1,45 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Simple message throughput counter/visualization + +-- .cfg +filename = "throughput.lua" +message_matcher = "TRUE" +ticker_interval = 60 +preserve_data = true +preservation_version = 0 + +-- rows (integer) - Number of aggregate count buckets +-- rows = 10080 -- one week + +-- sec_per_row (integer) - Number of seconds each aggregate bucket represents +-- sec_per_row = 60 + +-- preservation_version (integer) - This must be incremented if the rows or +-- sec_per_row value is change when using preservation (the old data is cleared) +-- preservation_version = 0 + +--]] +_PRESERVATION_VERSION = read_config("preservation_version") or 0 + +require "circular_buffer" +require "os" +local time = os.time + +local rows = read_config("rows") or 10080 +local sec_per_row = read_config("sec_per_row") or 60 + +local cb = circular_buffer.new(rows, 1, sec_per_row) +cb:set_header(1, "messages") + +function process_message() + cb:add(time() * 1e9, 1, 1) + return 0 +end + +function timer_event(ns) + inject_payload("cbuf", "counts", cb) +end diff --git a/sandboxes/heka/input/heka_stdin.lua b/sandboxes/heka/input/heka_stdin.lua new file mode 100644 index 0000000..2231636 --- /dev/null +++ b/sandboxes/heka/input/heka_stdin.lua @@ -0,0 +1,32 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Reads a Heka protobuf stream from the stdin file handle + +-- .cfg +filename = "heka_stdin.lua" + +--]] + +local stdin = require "io".stdin +require "heka_stream_reader" +require "string" + +local hsr = heka_stream_reader.new(read_config("Logger")) + +function process_message() + local cnt = 0 + local found, consumed, read + repeat + repeat + found, consumed, read = hsr:find_message(stdin) + if found then + inject_message(hsr) + cnt = cnt + 1 + end + until not found + until read == 0 + return 0, string.format("processed %d messages", cnt) +end diff --git a/sandboxes/heka/input/heka_tcp.lua b/sandboxes/heka/input/heka_tcp.lua new file mode 100644 index 0000000..cd464b2 --- /dev/null +++ b/sandboxes/heka/input/heka_tcp.lua @@ -0,0 +1,89 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Heka compatible TCP input + +-- .cfg +filename = "heka_tcp.lua" +instruction_limit = 0 + +address = "127.0.0.1" +port = 5565 + +--]] + +require "coroutine" +local socket = require "socket" +require "heka_stream_reader" +require "string" +require "table" + +local address = read_config("address") or "127.0.0.1" +local port = read_config("port") or 5565 +local server = assert(socket.bind(address, port)) +server:settimeout(0) +local threads = {} +local sockets = {server} + +local function handle_client(client) + local found, consumed, need = false, 0, 8192 * 4 + local caddr, cport = client:getpeername() + if not caddr then + caddr = "unknown" + cport = 0 + end + + local hsr = heka_stream_reader.new( + string.format("%s:%d -> %s:%d", caddr, cport, address, port)) + client:settimeout(0) + while client do + local buf, err, partial = client:receive(need) + if partial then buf = partial end + if not buf then break end + + repeat + found, consumed, need = hsr:find_message(buf) + if found then inject_message(hsr) end + buf = nil + until not found + + if err == "closed" then break end + + coroutine.yield() + end +end + +function process_message() + while true do + local ready = socket.select(sockets, nil, 1) + if ready then + for _, s in ipairs(ready) do + if s == server then + local client = s:accept() + if client then + sockets[#sockets + 1] = client + threads[client] = coroutine.create( + function() handle_client(client) end) + end + else + if threads[s] then + local status = coroutine.resume(threads[s]) + if not status then + s:close() + for i = #sockets, 2, -1 do + if s == sockets[i] then + table.remove(sockets, i) + break + end + end + threads[s] = nil + end + end + end + end + end + end + return 0 +end diff --git a/sandboxes/heka/input/syslog_udp.lua b/sandboxes/heka/input/syslog_udp.lua new file mode 100644 index 0000000..1e5b17e --- /dev/null +++ b/sandboxes/heka/input/syslog_udp.lua @@ -0,0 +1,98 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Syslog Collector + +-- .cfg +filename = "syslog_udp.lua" +instruction_limit = 0 + +-- address (string) - IP address or * for all interfaces +-- address = "127.0.0.1" + +-- port (integer) - IP port to listen on +-- port = 514 + +-- template (string) - The 'template' configuration string from rsyslog.conf +-- see http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html +-- template = "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" -- RSYSLOG_TraditionalForwardFormat + +--]] + +local syslog = require "syslog" +local socket = require "socket" + +local address = read_config("address") or "127.0.0.1" +local port = read_config("port") or 514 +local hostname_keep = read_config("hostname_keep") +local template = read_config("template") or "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" +local grammar = syslog.build_rsyslog_grammar(template) + +local msg = { +Timestamp = nil, +Type = read_config("type"), +Hostname = nil, +Payload = nil, +Pid = nil, +Severity = nil, +Fields = nil +} + +local err_msg = { + Logger = read_config("Logger"), + Type = "error", + Payload = nil, +} + +local server = assert(socket.udp()) +assert(server:setsockname(address, port)) +server:settimeout(1) + +function process_message() + while true do + local data, ip, port = server:receivefrom() + if data then + local fields = grammar:match(data) + if fields then + if fields.pri then + msg.Severity = fields.pri.severity + fields.syslogfacility = fields.pri.facility + fields.pri = nil + else + msg.Severity = fields.syslogseverity or fields["syslogseverity-text"] + or fields.syslogpriority or fields["syslogpriority-text"] + + fields.syslogseverity = nil + fields["syslogseverity-text"] = nil + fields.syslogpriority = nil + fields["syslogpriority-text"] = nil + end + + if fields.syslogtag then + fields.programname = fields.syslogtag.programname + msg.Pid = fields.syslogtag.pid + fields.syslogtag = nil + end + + msg.Hostname = fields.hostname or fields.source + fields.hostname = nil + fields.source = nil + + msg.Payload = fields.msg + fields.msg = nil + + fields.sender_ip = ip + fields.sender_port = {value = port, value_type = 2} + + msg.Fields = fields + pcall(inject_message, msg) + end + elseif ip ~= "timeout" then + err_msg.Payload = ip + pcall(inject_message, err_msg) + end + end + return 0 +end diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua new file mode 100644 index 0000000..f160df4 --- /dev/null +++ b/sandboxes/heka/output/debug.lua @@ -0,0 +1,65 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Outputs a more user friendly version (RST format) of the full Heka message to stdout. + +-- .cfg +filename = "debug.lua" +message_matcher = "TRUE" + +--]] + +local write = require "io".write +local floor = require "math".floor +local date = require "os".date +local byte = require "string".byte +local concat = require "table".concat + +local function get_uuid(uuid) + return string.format("%X%X%X%X-%X%X-%X%X-%X%X-%X%X%X%X%X", byte(uuid, 1, 16)) +end + +local function get_timestamp(ts) + local time_t = floor(ts / 1e9) + local ns = ts - time_t * 1e9 + local ds = date("%Y-%m-%d %H:%M:%S", time_t) + return string.format("%s.%09d +0000 UTC", ds, ns) +end + +function process_message() + local raw = read_message("raw") + local msg = decode_message(raw) + write(":Uuid: ", get_uuid(msg.Uuid), "\n") + write(":Timestamp: ", get_timestamp(msg.Timestamp), "\n") + write(":Type: ", msg.Type or "", "\n") + write(":Logger: ", msg.Logger or "", "\n") + write(":Severity: ", msg.Severity or 7, "\n") + write(":Payload: ", msg.Payload or "", "\n") + write(":EnvVersion: ", msg.EnvVersion or "", "\n") + write(":Pid: ", msg.Pid or "", "\n") + write(":Hostname: ", msg.Hostname or "", "\n") + write(":Fields:\n") + for i, v in ipairs(msg.Fields) do + write(" | name:", v.name, + " type:", v.value_type or 0, + " representation:", v.representation or "", + " value:") + if v.value_type == 4 then + for j, w in ipairs(v.value) do + if j ~= 1 then write(",") end + if w then write("true") else write("false") end + end + write("\n") + else + write(concat(v.value, ","), "\n") + end + end + write("\n") + return 0 +end + +function timer_event(ns) + -- no op +end diff --git a/sandboxes/heka/output/heka_log_rolling.lua b/sandboxes/heka/output/heka_log_rolling.lua new file mode 100644 index 0000000..f46cf2c --- /dev/null +++ b/sandboxes/heka/output/heka_log_rolling.lua @@ -0,0 +1,52 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "io" +require "string" + +--[[ +Outputs a Heka protobuf stream rolling the log file every time it reaches the +roll_size. + +-- .cfg + +filename = "heka_log_rolling.lua" +message_matcher = "TRUE" +ticker_interval = 0 +preserve_data = true + +--location where the payload is written +output_dir = "/tmp" +roll_size = 1024 * 1024 * 1024 + +--]] + +file_num = 0 + +local output_dir = read_config("output_dir") or "/tmp" +local output_prefix = read_config("Logger") +local roll_size = read_config("roll_size") or 1e9 +local fh + +function process_message() + if not fh then + local fn = string.format("%s/%s.%d.log", output_dir, output_prefix, file_num) + fh, err = io.open(fn, "a") + if err then return -1, err end + end + + local msg = read_message("framed") + fh:write(msg) + + if fh:seek() >= roll_size then + fh:close() + fh = nil + file_num = file_num + 1 + end + return 0 +end + +function timer_event(ns) + -- no op +end diff --git a/sandboxes/heka/output/heka_tcp.lua b/sandboxes/heka/output/heka_tcp.lua new file mode 100644 index 0000000..d6731a3 --- /dev/null +++ b/sandboxes/heka/output/heka_tcp.lua @@ -0,0 +1,60 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Heka compatible TCP output + +-- .cfg +filename = "heka_tcp.lua" +message_matcher = "TRUE" + +address = "127.0.0.1" +port = 5565 +timeout = 10 + +--]] + +local socket = require "socket" + +local address = read_config("address") or "127.0.0.1" +local port = read_config("port") or 5565 +local timeout = read_config("timeout") or 10 + +local function create_client() + local c, err = socket.connect(address, port) + if c then + c:setoption("tcp-nodelay", true) + c:setoption("keepalive", true) + c:settimeout(timeout) + end + return c, err +end + +local client, err = create_client() + +local function send_message(msg, i) + local len, err, i = client:send(msg, i) + if not len then + if err == "timeout" or err == "closed" then + client:close() + client = nil + return -3, err + end + return send_message(msg, i) + end + return 0 +end + +function process_message() + if not client then + client, err = create_client() + end + if not client then return -3, err end -- retry indefinitely + local ret, err = send_message(read_message("framed"), 1) + return ret, err +end + +function timer_event(ns) + -- no op +end diff --git a/sandboxes/heka/output/inject_payload.lua b/sandboxes/heka/output/inject_payload.lua new file mode 100644 index 0000000..796b1e3 --- /dev/null +++ b/sandboxes/heka/output/inject_payload.lua @@ -0,0 +1,52 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "io" +require "string" + +--[[ +Outputs inject_payload messages to the configured directory. +Filename: output_dir/logger.payload_name.payload_type +Contents: message.Payload + +-- .cfg +filename = "inject_payload.lua" +message_matcher = "Type == 'inject_payload'" +ticker_interval = 0 + +-- location where the payload is written (e.g. make them accessible from a web +-- server for external consumption) +output_dir = "/var/www/hindsight/payload" + +--]] + +local output_dir = read_config("output_dir") or "/tmp" + +function process_message() + local pt = read_message("Fields[payload_type]") + if type(pt) ~= "string" then return -1, "invalid payload_type" end + + local pn = read_message("Fields[payload_name]") or "" + if type(pn) ~= "string" then return -1, "invalid payload_name" end + + local logger = read_message("Logger") or "" + + pn = string.gsub(pn, "%W", "_") + pt = string.gsub(pt, "%W", "_") + logger = string.gsub(logger, "%W", "_") + + local fn = string.format("%s/%s.%s.%s", output_dir, logger, pn, pt) + local fh, err = io.open(fn, "w") + if err then return -1, err end + + local payload = read_message("Payload") or "" + fh:write(payload) + fh:close() + + return 0 +end + +function timer_event(ns) + -- no op +end diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index e246205..65a4cd5 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -382,7 +382,6 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, // todo link print to the logger or a no-op lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); - lsb_add_function(hsb->lsb, read_message, "read_message"); lsb_add_function(hsb->lsb, inject_message_input, "inject_message"); // inject_payload is intentionally excluded from input plugins // you can construct whatever you need with inject_message @@ -661,6 +660,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, set_restrictions(lua, hsb); // todo link print to the logger or a no-op + lsb_add_function(hsb->lsb, read_message, "read_message"); lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); lsb_add_function(hsb->lsb, update_checkpoint, "update_checkpoint"); diff --git a/src/heka/test/lua/analysis.lua b/src/heka/test/lua/analysis.lua index 8726898..fac796d 100644 --- a/src/heka/test/lua/analysis.lua +++ b/src/heka/test/lua/analysis.lua @@ -2,6 +2,15 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +assert(read_config) +assert(read_message) +assert(decode_message) +assert(inject_message) +assert(add_to_payload) +assert(inject_payload) +assert(not encode_message) +assert(not update_checkpoint) + function process_message() return 0 end diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index 56bcfac..7109678 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -1,6 +1,16 @@ -- This Source Code Form is subject to the terms of the Mozilla Public -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +assert(read_config) +assert(decode_message) +assert(inject_message) +assert(not read_message) +assert(not encode_message) +assert(not update_checkpoint) +assert(not add_to_payload) +assert(not inject_payload) + require "string" msg = {Timestamp = 8} diff --git a/src/heka/test/lua/output.lua b/src/heka/test/lua/output.lua index 18b1110..6f3288f 100644 --- a/src/heka/test/lua/output.lua +++ b/src/heka/test/lua/output.lua @@ -4,6 +4,15 @@ local sid +assert(read_config) +assert(read_message) +assert(decode_message) +assert(encode_message) +assert(update_checkpoint) +assert(not inject_message) +assert(not add_to_payload) +assert(not inject_payload) + function process_message(sequence_id) if not sequence_id then update_checkpoint() diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 5a396c2..091d382 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -346,7 +346,7 @@ static char* test_timer_event() e = lsb_heka_destroy_sandbox(hsb); - hsb = lsb_heka_create_analysis(NULL, "lua/input.lua", NULL, NULL, NULL, aim); + hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, NULL, aim); mu_assert(hsb, "lsb_heka_create_analysis succeeded"); mu_assert_rv(1, lsb_heka_timer_event(hsb, 0, false)); const char *err = lsb_heka_get_error(hsb); @@ -420,10 +420,10 @@ static char* test_pm_error() }; struct pm_result results[] = { - { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:15: boom" }, + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:25: boom" }, { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, - { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:21: aaaaaaaaaaaaaaaaaaaaaaaaa" + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:31: aaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, // >max error message From 61dbded3c711ffe8130e32406b576a9f1e16bed2 Mon Sep 17 00:00:00 2001 From: Alexandros Tsourakis Date: Tue, 1 Mar 2016 16:34:05 +0100 Subject: [PATCH 111/235] Modified the mysql decoder to accept more than alpha characters in usernames and hostnames --- modules/mysql.lua | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/mysql.lua b/modules/mysql.lua index c9c3081..350b7f6 100644 --- a/modules/mysql.lua +++ b/modules/mysql.lua @@ -18,8 +18,10 @@ local float = l.digit^1 * "." * l.digit^1 local time = l.P"# Time: " * line -local user_name = l.alpha^1 * "[" * l.alpha^1 * "]" -local host_name = l.alpha^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" +local user_legal = (1 - l.S"[]") +local host_legal = (l.alnum + l.S".i-") +local user_name = user_legal^1 * "[" * l.Cg(user_legal^1, "Username") * "]" +local host_name = host_legal^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Query_time") From 3248c0cbc248fbe16e4f1e09808a17a43310b476 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 1 Mar 2016 20:26:27 -0800 Subject: [PATCH 112/235] port to gcc 5 --- src/heka/rapidjson.cpp | 2 +- src/heka/sandbox.c | 24 ++++++++++++------------ src/luasandbox.c | 18 +++++++++--------- src/util/heka_message.c | 12 ++++++------ 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp index 80f10f5..b202a83 100644 --- a/src/heka/rapidjson.cpp +++ b/src/heka/rapidjson.cpp @@ -485,7 +485,7 @@ static int hj_parse_message(lua_State *lua) lsb_lua_sandbox *lsb = static_cast (lua_touserdata(lua, lua_upvalueindex(1))); if (!lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __FUNCTION__); + return luaL_error(lua, "%s() invalid lightuserdata", __func__); } int n = lua_gettop(lua); int idx = 1; diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 65a4cd5..8b8eea6 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -165,7 +165,7 @@ static int inject_payload(lua_State *lua) lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); if (!lsb) { - return luaL_error(lua, "%s invalid lightuserdata", __FUNCTION__); + return luaL_error(lua, "%s invalid lightuserdata", __func__); } int n = lua_gettop(lua); @@ -180,14 +180,14 @@ static int inject_payload(lua_State *lua) if (n > 0) { if (lua_type(lua, 1) != LUA_TSTRING) { return luaL_error(lua, "%s() payload_type argument must be a string", - __FUNCTION__); + __func__); } } if (n > 1) { if (lua_type(lua, 2) != LUA_TSTRING) { return luaL_error(lua, "%s() payload_name argument must be a string", - __FUNCTION__); + __func__); } } @@ -348,19 +348,19 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lsb_heka_im_input im) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!im) { - if (logger) logger(__FUNCTION__, 3, "inject_message callback must be " + if (logger) logger(__func__, 3, "inject_message callback must be " "specified"); return NULL; } lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } @@ -543,12 +543,12 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lsb_heka_im_analysis im) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!im) { - if (logger) logger(__FUNCTION__, 3, "inject_message callback must be " + if (logger) logger(__func__, 3, "inject_message callback must be " "specified"); return NULL; } @@ -556,7 +556,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } @@ -626,12 +626,12 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_heka_update_checkpoint ucp) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!ucp) { - if (logger) logger(__FUNCTION__, 3, "update_checkpoint callback must be " + if (logger) logger(__func__, 3, "update_checkpoint callback must be " "specified"); return NULL; } @@ -639,7 +639,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } diff --git a/src/luasandbox.c b/src/luasandbox.c index 1abe7b5..85c8ee7 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -223,7 +223,7 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) { lua_State *L = luaL_newstate(); if (!L) { - if (logger) logger(__FUNCTION__, 3, "lua_State creation failed"); + if (logger) logger(__func__, 3, "lua_State creation failed"); return NULL; } @@ -250,7 +250,7 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) cleanup: if (ret) { if (logger) { - logger(__FUNCTION__, 3, "config error: %s", lua_tostring(L, -1)); + logger(__func__, 3, "config error: %s", lua_tostring(L, -1)); } lua_close(L); return NULL; @@ -305,7 +305,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) break; default: if (logger) { - logger(__FUNCTION__, 4, "skipping config value type: %s", + logger(__func__, 4, "skipping config value type: %s", lua_typename(cfg, vt)); } break; @@ -313,7 +313,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) break; default: if (logger) { - logger(__FUNCTION__, 4, "skipping config key type: %s", + logger(__func__, 4, "skipping config key type: %s", lua_typename(cfg, kt)); } break; @@ -381,12 +381,12 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb_logger logger) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!set_tz()) { - if (logger) logger(__FUNCTION__, 3, "fail to set the TZ to UTC"); + if (logger) logger(__func__, 3, "fail to set the TZ to UTC"); return NULL; } @@ -394,7 +394,7 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb_lua_sandbox *lsb = malloc(sizeof*lsb); if (!lsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } memset(lsb->usage, 0, sizeof(lsb->usage)); @@ -406,7 +406,7 @@ lsb_lua_sandbox* lsb_create(void *parent, #endif if (!lsb->lua) { - if (logger) logger(__FUNCTION__, 3, "lua state creation failed"); + if (logger) logger(__func__, 3, "lua state creation failed"); free(lsb); return NULL; } @@ -444,7 +444,7 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb->state_file = NULL; if (!lsb->lua_file || lsb_init_output_buffer(&lsb->output, ol)) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed failed"); + if (logger) logger(__func__, 3, "memory allocation failed failed"); lsb_free_output_buffer(&lsb->output); free(lsb->lua_file); lua_close(lsb->lua); diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 5c2b999..4513783 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -205,7 +205,7 @@ bool lsb_decode_heka_message(lsb_heka_message *m, { if (!m || !buf || len == 0) { if (logger) { - logger(__FUNCTION__, 4, LSB_ERR_UTIL_NULL); + logger(__func__, 4, LSB_ERR_UTIL_NULL); } return false; } @@ -275,7 +275,7 @@ bool lsb_decode_heka_message(lsb_heka_message *m, lsb_heka_field *tmp = realloc(m->fields, m->fields_size * sizeof(lsb_heka_field)); if (!tmp) { - if (logger) logger(__FUNCTION__, 0, "fields reallocation failed"); + if (logger) logger(__func__, 0, "fields reallocation failed"); return false; } m->fields = tmp; @@ -294,19 +294,19 @@ bool lsb_decode_heka_message(lsb_heka_message *m, if (!cp) { if (logger) { - logger(__FUNCTION__, 4, "tag:%d wiretype:%d position:%d", tag, + logger(__func__, 4, "tag:%d wiretype:%d position:%d", tag, wiretype, lp - buf); } return false; } if (!m->uuid.s) { - if (logger) logger(__FUNCTION__, 4, "missing " LSB_UUID); + if (logger) logger(__func__, 4, "missing " LSB_UUID); return false; } if (!timestamp) { - if (logger) logger(__FUNCTION__, 4, "missing " LSB_TIMESTAMP); + if (logger) logger(__func__, 4, "missing " LSB_TIMESTAMP); return false; } @@ -324,7 +324,7 @@ bool lsb_find_heka_message(lsb_heka_message *m, { if (!m || !ib || !discarded_bytes) { if (logger) { - logger(__FUNCTION__, 4, LSB_ERR_UTIL_NULL); + logger(__func__, 4, LSB_ERR_UTIL_NULL); } return false; } From f6c4a891ef521ace2889794de13e71cab35aa804 Mon Sep 17 00:00:00 2001 From: Alexandros Tsourakis Date: Wed, 2 Mar 2016 08:45:08 +0100 Subject: [PATCH 113/235] Removed stray character and added test case --- modules/mysql.lua | 2 +- src/test/lua/lpeg_mysql.lua | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/modules/mysql.lua b/modules/mysql.lua index 350b7f6..6b33f3c 100644 --- a/modules/mysql.lua +++ b/modules/mysql.lua @@ -19,7 +19,7 @@ local float = l.digit^1 * "." * l.digit^1 local time = l.P"# Time: " * line local user_legal = (1 - l.S"[]") -local host_legal = (l.alnum + l.S".i-") +local host_legal = (l.alnum + l.S".-") local user_name = user_legal^1 * "[" * l.Cg(user_legal^1, "Username") * "]" local host_name = host_legal^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep diff --git a/src/test/lua/lpeg_mysql.lua b/src/test/lua/lpeg_mysql.lua index a3f092f..2e12022 100644 --- a/src/test/lua/lpeg_mysql.lua +++ b/src/test/lua/lpeg_mysql.lua @@ -31,6 +31,24 @@ SET last_insert_id=999,timestamp=1399500744; /* [queryName=FIND_ITEMS] */ SELECT * FROM widget WHERE id = 10; +]], +[[ +# Time: 140507 18:14:18 +# User@Host: sync.rw[sync.rw] @ db01.example.com [127.0.0.1] +# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 +SET timestamp=1399500744; +/* [queryName=FIND_ITEMS] */ SELECT * +FROM widget +WHERE id = 10; +]], +[[ +# Time: 140507 18:14:18 +# User@Host: sync_rw[sync_rw] @ db-01.example.com [127.0.0.1] +# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 +SET timestamp=1399500744; +/* [queryName=FIND_ITEMS] */ SELECT * +FROM widget +WHERE id = 10; ]] } From b762c90aa3eb275b73fae838120fd77eb6c40e67 Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Tue, 1 Mar 2016 18:22:42 -0800 Subject: [PATCH 114/235] add max_depth and sep char params for util.table_to_fields --- modules/util.lua | 19 ++++++++++++++++--- src/heka/rapidjson.cpp | 2 +- src/heka/sandbox.c | 24 ++++++++++++------------ src/luasandbox.c | 18 +++++++++--------- src/test/lua/util_test.lua | 22 +++++++++++++++------- src/util/heka_message.c | 12 ++++++------ 6 files changed, 59 insertions(+), 38 deletions(-) diff --git a/modules/util.lua b/modules/util.lua index 52ac840..7b154c2 100644 --- a/modules/util.lua +++ b/modules/util.lua @@ -6,20 +6,33 @@ local pairs = pairs local type = type local string = require "string" +local cjson = require "cjson" local M = {} setfenv(1, M) -- Remove external access to contain everything in the module -- Flattens a Lua table so it can be encoded as a protobuf fields object. -function table_to_fields(t, fields, parent) +function table_to_fields(t, fields, parent, char, max_depth) + if type(char) ~= "string" then + char = "." + end + for k,v in pairs(t) do if parent then - full_key = string.format("%s.%s", parent, k) + full_key = string.format("%s%s%s", parent, char, k) else full_key = k end + if type(v) == "table" then - table_to_fields(v, fields, full_key) + local _, sep_count = string.gsub(full_key, char, "") + local depth = sep_count + 1 + + if type(max_depth) == "number" and depth >= max_depth then + fields[full_key] = cjson.encode(v) + else + table_to_fields(v, fields, full_key, char, max_depth) + end else if type(v) ~= "userdata" then fields[full_key] = v diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp index 80f10f5..b202a83 100644 --- a/src/heka/rapidjson.cpp +++ b/src/heka/rapidjson.cpp @@ -485,7 +485,7 @@ static int hj_parse_message(lua_State *lua) lsb_lua_sandbox *lsb = static_cast (lua_touserdata(lua, lua_upvalueindex(1))); if (!lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __FUNCTION__); + return luaL_error(lua, "%s() invalid lightuserdata", __func__); } int n = lua_gettop(lua); int idx = 1; diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 65a4cd5..8b8eea6 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -165,7 +165,7 @@ static int inject_payload(lua_State *lua) lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); if (!lsb) { - return luaL_error(lua, "%s invalid lightuserdata", __FUNCTION__); + return luaL_error(lua, "%s invalid lightuserdata", __func__); } int n = lua_gettop(lua); @@ -180,14 +180,14 @@ static int inject_payload(lua_State *lua) if (n > 0) { if (lua_type(lua, 1) != LUA_TSTRING) { return luaL_error(lua, "%s() payload_type argument must be a string", - __FUNCTION__); + __func__); } } if (n > 1) { if (lua_type(lua, 2) != LUA_TSTRING) { return luaL_error(lua, "%s() payload_name argument must be a string", - __FUNCTION__); + __func__); } } @@ -348,19 +348,19 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lsb_heka_im_input im) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!im) { - if (logger) logger(__FUNCTION__, 3, "inject_message callback must be " + if (logger) logger(__func__, 3, "inject_message callback must be " "specified"); return NULL; } lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } @@ -543,12 +543,12 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lsb_heka_im_analysis im) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!im) { - if (logger) logger(__FUNCTION__, 3, "inject_message callback must be " + if (logger) logger(__func__, 3, "inject_message callback must be " "specified"); return NULL; } @@ -556,7 +556,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } @@ -626,12 +626,12 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_heka_update_checkpoint ucp) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!ucp) { - if (logger) logger(__FUNCTION__, 3, "update_checkpoint callback must be " + if (logger) logger(__func__, 3, "update_checkpoint callback must be " "specified"); return NULL; } @@ -639,7 +639,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } diff --git a/src/luasandbox.c b/src/luasandbox.c index 1abe7b5..85c8ee7 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -223,7 +223,7 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) { lua_State *L = luaL_newstate(); if (!L) { - if (logger) logger(__FUNCTION__, 3, "lua_State creation failed"); + if (logger) logger(__func__, 3, "lua_State creation failed"); return NULL; } @@ -250,7 +250,7 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) cleanup: if (ret) { if (logger) { - logger(__FUNCTION__, 3, "config error: %s", lua_tostring(L, -1)); + logger(__func__, 3, "config error: %s", lua_tostring(L, -1)); } lua_close(L); return NULL; @@ -305,7 +305,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) break; default: if (logger) { - logger(__FUNCTION__, 4, "skipping config value type: %s", + logger(__func__, 4, "skipping config value type: %s", lua_typename(cfg, vt)); } break; @@ -313,7 +313,7 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) break; default: if (logger) { - logger(__FUNCTION__, 4, "skipping config key type: %s", + logger(__func__, 4, "skipping config key type: %s", lua_typename(cfg, kt)); } break; @@ -381,12 +381,12 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb_logger logger) { if (!lua_file) { - if (logger) logger(__FUNCTION__, 3, "lua_file must be specified"); + if (logger) logger(__func__, 3, "lua_file must be specified"); return NULL; } if (!set_tz()) { - if (logger) logger(__FUNCTION__, 3, "fail to set the TZ to UTC"); + if (logger) logger(__func__, 3, "fail to set the TZ to UTC"); return NULL; } @@ -394,7 +394,7 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb_lua_sandbox *lsb = malloc(sizeof*lsb); if (!lsb) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed"); + if (logger) logger(__func__, 3, "memory allocation failed"); return NULL; } memset(lsb->usage, 0, sizeof(lsb->usage)); @@ -406,7 +406,7 @@ lsb_lua_sandbox* lsb_create(void *parent, #endif if (!lsb->lua) { - if (logger) logger(__FUNCTION__, 3, "lua state creation failed"); + if (logger) logger(__func__, 3, "lua state creation failed"); free(lsb); return NULL; } @@ -444,7 +444,7 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb->state_file = NULL; if (!lsb->lua_file || lsb_init_output_buffer(&lsb->output, ol)) { - if (logger) logger(__FUNCTION__, 3, "memory allocation failed failed"); + if (logger) logger(__func__, 3, "memory allocation failed failed"); lsb_free_output_buffer(&lsb->output); free(lsb->lua_file); lua_close(lsb->lua); diff --git a/src/test/lua/util_test.lua b/src/test/lua/util_test.lua index 440cf8c..533b497 100644 --- a/src/test/lua/util_test.lua +++ b/src/test/lua/util_test.lua @@ -5,15 +5,23 @@ local string = require("string") local util = require("util") -local t = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} -local f = {} +local ta = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} +local fa = {} + +local tb = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} +local fb = {} local function table_to_fields() - util.table_to_fields(t, f, nil) - assert(f.toplevel == 0, f.toplevel) - assert(f["struct.item0"] == 0, f["struct.item0"]) - assert(f["struct.item1"] == 1, f["struct.item1"]) - assert(f["struct.item2.nested"] == "n1", f["struct.item2.nested"]) + util.table_to_fields(ta, fa, nil) + assert(fa.toplevel == 0, fa.toplevel) + assert(fa["struct.item0"] == 0, fa["struct.item0"]) + assert(fa["struct.item1"] == 1, fa["struct.item1"]) + assert(fa["struct.item2.nested"] == "n1", fa["struct.item2.nested"]) + util.table_to_fields(tb, fb, nil, "_", 2) + assert(fb.toplevel == 0, fb.toplevel) + assert(fb["struct_item0"] == 0, fb["struct_item0"]) + assert(fb["struct_item1"] == 1, fb["struct_item1"]) + assert(fb["struct_item2"] == '{"nested":"n1"}', fb["struct_item2"]) end local function alpha() diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 5c2b999..4513783 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -205,7 +205,7 @@ bool lsb_decode_heka_message(lsb_heka_message *m, { if (!m || !buf || len == 0) { if (logger) { - logger(__FUNCTION__, 4, LSB_ERR_UTIL_NULL); + logger(__func__, 4, LSB_ERR_UTIL_NULL); } return false; } @@ -275,7 +275,7 @@ bool lsb_decode_heka_message(lsb_heka_message *m, lsb_heka_field *tmp = realloc(m->fields, m->fields_size * sizeof(lsb_heka_field)); if (!tmp) { - if (logger) logger(__FUNCTION__, 0, "fields reallocation failed"); + if (logger) logger(__func__, 0, "fields reallocation failed"); return false; } m->fields = tmp; @@ -294,19 +294,19 @@ bool lsb_decode_heka_message(lsb_heka_message *m, if (!cp) { if (logger) { - logger(__FUNCTION__, 4, "tag:%d wiretype:%d position:%d", tag, + logger(__func__, 4, "tag:%d wiretype:%d position:%d", tag, wiretype, lp - buf); } return false; } if (!m->uuid.s) { - if (logger) logger(__FUNCTION__, 4, "missing " LSB_UUID); + if (logger) logger(__func__, 4, "missing " LSB_UUID); return false; } if (!timestamp) { - if (logger) logger(__FUNCTION__, 4, "missing " LSB_TIMESTAMP); + if (logger) logger(__func__, 4, "missing " LSB_TIMESTAMP); return false; } @@ -324,7 +324,7 @@ bool lsb_find_heka_message(lsb_heka_message *m, { if (!m || !ib || !discarded_bytes) { if (logger) { - logger(__FUNCTION__, 4, LSB_ERR_UTIL_NULL); + logger(__func__, 4, LSB_ERR_UTIL_NULL); } return false; } From 10efd7fc307cb35dfb5540e4dbebb12986ab721b Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Wed, 2 Mar 2016 21:19:00 -0800 Subject: [PATCH 115/235] fix cached serialize.lua51.data due to rearranged memory layout --- src/test/output/serialize.lua51.data | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index b7d5c76..561570a 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -8,6 +8,7 @@ _G["dataRef"]:fromstring("2 2 nan nan nan nan nan nan nan nan nan") if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end _G["delta"]:set_header(1, "Column_1", "count", "sum") _G["delta"]:fromstring("1 1 2 nan 0 2") +_G["data"] = _G["dataRef"] _G["rate"] = 0.12345678 _G["kvp"] = {} _G["kvp"]["a"] = "foo" @@ -19,7 +20,11 @@ _G["kvp"]["r"][4] = 92.002 _G["kvp"]["r"][5] = 91.10001 _G["kvp"]["r"]["key"] = "val" _G["kvp"]["b"] = "bar" -_G["data"] = _G["dataRef"] +_G["cycleb"] = {} +_G["cycleb"]["a"] = {} +_G["cycleb"]["a"]["b"] = _G["cycleb"] +_G["cycleb"]["a"]["type"] = "cycle a" +_G["cycleb"]["type"] = "cycle b" _G["uuids"] = {} _G["uuids"][1] = {} _G["uuids"][1]["type"] = "test" @@ -28,11 +33,6 @@ _G["uuids"][2] = {} _G["uuids"][2]["type"] = "test1" _G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" _G["inf"] = 1/0 -_G["cycleb"] = {} -_G["cycleb"]["a"] = {} -_G["cycleb"]["a"]["b"] = _G["cycleb"] -_G["cycleb"]["a"]["type"] = "cycle a" -_G["cycleb"]["type"] = "cycle b" _G["cyclea"] = _G["cycleb"]["a"] _G["large_key"] = {} _G["large_key"]["aaaaaaaaaaaaaaaaaaa"] = {} From 1197ba9df53216c5cd4dcc203d845748b5cba55c Mon Sep 17 00:00:00 2001 From: Nathan Williams Date: Thu, 3 Mar 2016 01:34:03 -0800 Subject: [PATCH 116/235] remove extraneous table from test --- src/test/lua/util_test.lua | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/test/lua/util_test.lua b/src/test/lua/util_test.lua index 533b497..be0757a 100644 --- a/src/test/lua/util_test.lua +++ b/src/test/lua/util_test.lua @@ -5,19 +5,17 @@ local string = require("string") local util = require("util") -local ta = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} +local t = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} local fa = {} - -local tb = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} local fb = {} local function table_to_fields() - util.table_to_fields(ta, fa, nil) + util.table_to_fields(t, fa, nil) assert(fa.toplevel == 0, fa.toplevel) assert(fa["struct.item0"] == 0, fa["struct.item0"]) assert(fa["struct.item1"] == 1, fa["struct.item1"]) assert(fa["struct.item2.nested"] == "n1", fa["struct.item2.nested"]) - util.table_to_fields(tb, fb, nil, "_", 2) + util.table_to_fields(t, fb, nil, "_", 2) assert(fb.toplevel == 0, fb.toplevel) assert(fb["struct_item0"] == 0, fb["struct_item0"]) assert(fb["struct_item1"] == 1, fb["struct_item1"]) From 9a13cf55c88e685ed17e694f31fb4f743e56aa0b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 4 Mar 2016 21:26:37 -0800 Subject: [PATCH 117/235] Add Kafka client/producer support --- CMakeLists.txt | 11 +- covfn.txt | 125 +++-- docs/heka/index.md | 4 +- docs/heka/kafka_consumer.md | 49 ++ docs/heka/kafka_producer.md | 118 +++++ include/luasandbox/heka/sandbox.h | 1 + sandboxes/heka/input/heka_kafka.lua | 59 +++ sandboxes/heka/output/heka_kafka.lua | 62 +++ src/heka/CMakeLists.txt | 9 + src/heka/kafka.c | 753 +++++++++++++++++++++++++++ src/heka/kafka_impl.h | 20 + src/heka/sandbox.c | 25 +- src/heka/test/CMakeLists.txt | 6 + src/heka/test/lua/kafka_consumer.lua | 79 +++ src/heka/test/lua/kafka_producer.lua | 74 +++ src/heka/test/test_kafka.c | 114 ++++ 16 files changed, 1448 insertions(+), 61 deletions(-) create mode 100644 docs/heka/kafka_consumer.md create mode 100644 docs/heka/kafka_producer.md create mode 100644 sandboxes/heka/input/heka_kafka.lua create mode 100644 sandboxes/heka/output/heka_kafka.lua create mode 100644 src/heka/kafka.c create mode 100644 src/heka/kafka_impl.h create mode 100644 src/heka/test/lua/kafka_consumer.lua create mode 100644 src/heka/test/lua/kafka_producer.lua create mode 100644 src/heka/test/test_kafka.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 7c44ebf..773acb0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 14) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_MINOR 15) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) @@ -30,6 +30,13 @@ if (ZLIB_FOUND) add_definitions(-DHAVE_ZLIB) endif() +find_library(LIBRTKAFKA_LIBRARY rdkafka) +if (LIBRTKAFKA_LIBRARY) + add_definitions(-DHAVE_KAFKA) +else() + message(STATUS "The Kafka module will not be built") +endif() + find_package(Git REQUIRED) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") diff --git a/covfn.txt b/covfn.txt index d5d2e5f..24713b8 100644 --- a/covfn.txt +++ b/covfn.txt @@ -1,40 +1,44 @@ Function Source Line FnCov C/D Coverage -------------------------------------------------------------------------------------------------- ------------------------------ ----- --------------------- add_table_ref(table_ref_array*,const void*,size_t) ...uasandbox_serialize.c 170 1 / 1 1 / 4 = 25% +consumer_receive(lua_State*) ../src/heka/kafka.c 671 1 / 1 5 / 16 = 31% restore_global_data(lsb_lua_sandbox*) ...uasandbox_serialize.c 495 1 / 1 7 / 18 = 38% lsb_create(void*,const char*,const char*,lsb_logger) ../src/luasandbox.c 378 1 / 1 11 / 26 = 42% serialize_data(lsb_lua_sandbox*,int,lsb_output_buffer*) ...uasandbox_serialize.c 229 1 / 1 8 / 18 = 44% -lsb_ungzip(const char*,size_t,size_t*) ../src/util/util.c 113 1 / 1 10 / 22 = 45% hsr_new(lua_State*) .../heka/stream_reader.c 21 1 / 1 4 / 8 = 50% lsb_create_message_match_builder(const char*,const char*) ...eka/message_matcher.c 443 1 / 1 3 / 6 = 50% +msg_delivered(rd_kafka_t*,void*,size_t,int,void*,void*) ../src/heka/kafka.c 56 1 / 1 1 / 2 = 50% +producer_destroy_topic(lua_State*) ../src/heka/kafka.c 370 1 / 1 1 / 2 = 50% +luaopen_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 685 1 / 1 1 / 2 = 50% instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 114 1 / 1 1 / 2 = 50% read_config(lua_State*) ../src/luasandbox.c 138 1 / 1 1 / 2 = 50% set_tz() ../src/luasandbox.c 335 1 / 1 1 / 2 = 50% heka_encode_message_table(lsb_lua_sandbox*,int) ../src/heka/message.c 881 1 / 1 14 / 24 = 58% set_random_seed() ../src/luasandbox.c 350 1 / 1 7 / 12 = 58% lsb_read_file(const char*) ../src/util/util.c 51 1 / 1 6 / 10 = 60% -lsb_heka_create_input(void*,const char*,const char*,const char*,lsb_logger,lsb_heka_im_input) ../src/heka/sandbox.c 343 1 / 1 11 / 18 = 61% -lsb_heka_create_analysis(void*,const char*,const...r*,const char*,lsb_logger,lsb_heka_im_analysis) ../src/heka/sandbox.c 539 1 / 1 11 / 18 = 61% -lsb_heka_create_output(void*,const char*,const c...st char*,lsb_logger,lsb_heka_update_checkpoint) ../src/heka/sandbox.c 622 1 / 1 11 / 18 = 61% -hj_parse(lua_State*) ...rc/heka/rapidjson.cpp 126 1 / 1 5 / 8 = 62% -hj_remove(lua_State*) ...rc/heka/rapidjson.cpp 578 1 / 1 5 / 8 = 62% +lsb_heka_create_analysis(void*,const char*,const...r*,const char*,lsb_logger,lsb_heka_im_analysis) ../src/heka/sandbox.c 547 1 / 1 11 / 18 = 61% +lsb_heka_create_output(void*,const char*,const c...st char*,lsb_logger,lsb_heka_update_checkpoint) ../src/heka/sandbox.c 630 1 / 1 11 / 18 = 61% +hj_parse(lua_State*) ...rc/heka/rapidjson.cpp 134 1 / 1 5 / 8 = 62% +hj_remove(lua_State*) ...rc/heka/rapidjson.cpp 616 1 / 1 5 / 8 = 62% min_expand(MatchState*,const char*,const char*,const char*) ...util/string_matcher.c 171 1 / 1 5 / 8 = 62% preserve_global_data(lsb_lua_sandbox*) ...uasandbox_serialize.c 366 1 / 1 27 / 42 = 64% -output_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 559 1 / 1 8 / 12 = 66% -hj_iter(lua_State*) ...rc/heka/rapidjson.cpp 388 1 / 1 6 / 9 = 66% +lsb_heka_create_input(void*,const char*,const char*,const char*,lsb_logger,lsb_heka_im_input) ../src/heka/sandbox.c 346 1 / 1 12 / 18 = 66% +producer_poll(lua_State*) ../src/heka/kafka.c 389 1 / 1 4 / 6 = 66% lsb_init_heka_message(lsb_heka_message*,int) ...c/util/heka_message.c 417 1 / 1 4 / 6 = 66% process_fields(lua_State*,const char*,const char*) ../src/heka/message.c 93 1 / 1 55 / 82 = 67% encode_field_array(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 330 1 / 1 19 / 28 = 67% -lsb_heka_pm_input(lsb_heka_sandbox*,double,const char*,_Bool) ../src/heka/sandbox.c 505 1 / 1 11 / 16 = 68% +lsb_ungzip(const char*,size_t,size_t,size_t*) ../src/util/util.c 113 1 / 1 30 / 44 = 68% +load_topic_conf(lua_State*,rd_kafka_topic_conf_t*,int) ../src/heka/kafka.c 154 1 / 1 15 / 22 = 68% +lsb_heka_pm_input(lsb_heka_sandbox*,double,const char*,_Bool) ../src/heka/sandbox.c 513 1 / 1 11 / 16 = 68% read_string(lua_State*,int,const char*,const char*) ../src/heka/message.c 52 1 / 1 7 / 10 = 70% -set_restrictions(lua_State*,lsb_heka_sandbox*) ../src/heka/sandbox.c 253 1 / 1 19 / 27 = 70% check_int(lua_State*,int,const char*,int) ../src/luasandbox.c 191 1 / 1 5 / 7 = 71% -hj_parse_message(lua_State*) ...rc/heka/rapidjson.cpp 454 1 / 1 25 / 34 = 73% -process_message(lsb_heka_sandbox*,lsb_heka_message*,lua_State*,int,_Bool) ../src/heka/sandbox.c 420 1 / 1 29 / 39 = 74% +hj_parse_message(lua_State*) ...rc/heka/rapidjson.cpp 483 1 / 1 26 / 36 = 72% +producer_create_topic(lua_State*) ../src/heka/kafka.c 303 1 / 1 8 / 11 = 72% +hj_iter(lua_State*) ...rc/heka/rapidjson.cpp 413 1 / 1 8 / 11 = 72% encode_fields(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 618 1 / 1 27 / 36 = 75% serialize_kvp(lsb_lua_sandbox*,serialization_data*,size_t) ...uasandbox_serialize.c 286 1 / 1 18 / 24 = 75% decode_header(char*,size_t,size_t) ...c/util/heka_message.c 19 1 / 1 12 / 16 = 75% -read_message(lua_State*) ../src/heka/sandbox.c 37 1 / 1 6 / 8 = 75% +read_message(lua_State*) ../src/heka/sandbox.c 40 1 / 1 6 / 8 = 75% hsr_decode_message(lua_State*) .../heka/stream_reader.c 77 1 / 1 6 / 8 = 75% process_varint(lua_State*,const char*,int,int,const char*,const char*) ../src/heka/message.c 72 1 / 1 3 / 4 = 75% encode_int(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 287 1 / 1 3 / 4 = 75% @@ -46,27 +50,33 @@ lsb_get_output(lsb_lua_sandbox*,size_t*) lsb_clear_heka_message(lsb_heka_message*) ...c/util/heka_message.c 431 1 / 1 3 / 4 = 75% lsb_outputc(lsb_output_buffer*,char) .../util/output_buffer.c 78 1 / 1 3 / 4 = 75% lsb_outputs(lsb_output_buffer*,const char*,size_t) .../util/output_buffer.c 146 1 / 1 3 / 4 = 75% +consumer_new(lua_State*) ../src/heka/kafka.c 591 1 / 1 16 / 21 = 76% encode_field_value(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 403 1 / 1 84 / 110 = 76% +process_message(lsb_heka_sandbox*,lsb_heka_message*,lua_State*,int,_Bool) ../src/heka/sandbox.c 428 1 / 1 30 / 39 = 76% lsb_outputfd(lsb_output_buffer*,double) .../util/output_buffer.c 177 1 / 1 31 / 40 = 77% lsb_init(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 460 1 / 1 28 / 36 = 77% lsb_pcall_setup(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 675 1 / 1 14 / 18 = 77% -hsr_find_message(lua_State*) .../heka/stream_reader.c 106 1 / 1 25 / 32 = 78% +producer_new(lua_State*) ../src/heka/kafka.c 238 1 / 1 7 / 9 = 77% +output_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 593 1 / 1 11 / 14 = 78% +producer_send(lua_State*) ../src/heka/kafka.c 410 1 / 1 8 / 10 = 80% read_string(int,const char*,const char*,lsb_const_string*) ...c/util/heka_message.c 39 1 / 1 8 / 10 = 80% read_string_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 57 1 / 1 8 / 10 = 80% read_integer_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 78 1 / 1 8 / 10 = 80% check_string(lua_State*,int,const char*,const char*) ../src/luasandbox.c 169 1 / 1 4 / 5 = 80% -lsb_heka_timer_event(lsb_heka_sandbox*,time_t,_Bool) ../src/heka/sandbox.c 744 1 / 1 13 / 16 = 81% matchbalance(MatchState*,const char*,const char*) ...util/string_matcher.c 137 1 / 1 13 / 16 = 81% +set_restrictions(lua_State*,lsb_heka_sandbox*) ../src/heka/sandbox.c 256 1 / 1 22 / 27 = 81% +load_conf(lua_State*,rd_kafka_conf_t*,int) ../src/heka/kafka.c 72 1 / 1 18 / 22 = 81% lsb_serialize_binary(lsb_output_buffer*,const void*,size_t) ...uasandbox_serialize.c 547 1 / 1 9 / 11 = 81% lsb_outputf(lsb_output_buffer*,const char*,...) .../util/output_buffer.c 92 1 / 1 23 / 28 = 82% lsb_create_message_matcher(const lsb_message_match_builder*,const char*) ...eka/message_matcher.c 493 1 / 1 24 / 29 = 82% heka_decode_message(lua_State*) ../src/heka/message.c 693 1 / 1 54 / 65 = 83% heka_read_message(lua_State*,lsb_heka_message*) ../src/heka/message.c 922 1 / 1 45 / 54 = 83% -set_missing_headers(lua_State*,int,lsb_heka_sandbox*) ../src/heka/message.c 32 1 / 1 10 / 12 = 83% -lsb_heka_pm_analysis(lsb_heka_sandbox*,lsb_heka_message*,_Bool) ../src/heka/sandbox.c 603 1 / 1 10 / 12 = 83% +hsr_find_message(lua_State*) .../heka/stream_reader.c 106 1 / 1 25 / 30 = 83% +lsb_heka_pm_analysis(lsb_heka_sandbox*,lsb_heka_message*,_Bool) ../src/heka/sandbox.c 611 1 / 1 10 / 12 = 83% ignore_value_type(lsb_lua_sandbox*,serialization_data*,int,lua_CFunction*) ...uasandbox_serialize.c 114 1 / 1 10 / 12 = 83% -hj_parse_schema(lua_State*) ...rc/heka/rapidjson.cpp 100 1 / 1 5 / 6 = 83% -inject_message_analysis(lua_State*) ../src/heka/sandbox.c 132 1 / 1 5 / 6 = 83% +producer_gc(lua_State*) ../src/heka/kafka.c 455 1 / 1 5 / 6 = 83% +hj_parse_schema(lua_State*) ...rc/heka/rapidjson.cpp 108 1 / 1 5 / 6 = 83% +inject_message_analysis(lua_State*) ../src/heka/sandbox.c 135 1 / 1 5 / 6 = 83% lsb_destroy(lsb_lua_sandbox*) ../src/luasandbox.c 573 1 / 1 5 / 6 = 83% lsb_pb_output_varint(char*,unsigned long long) ../src/util/protobuf.c 56 1 / 1 5 / 6 = 83% lsb_output(lsb_lua_sandbox*,int,int,int) ...c/luasandbox_output.c 42 1 / 1 27 / 32 = 84% @@ -74,63 +84,72 @@ heka_encode_message(lua_State*) match_class(int,int) ...util/string_matcher.c 68 1 / 1 11 / 13 = 84% match(MatchState*,const char*,const char*) ...util/string_matcher.c 183 1 / 1 39 / 46 = 84% lsb_pb_read_varint(const char*,const char*,long long*) ../src/util/protobuf.c 36 1 / 1 17 / 20 = 85% -inject_message_input(lua_State*) ../src/heka/sandbox.c 53 1 / 1 18 / 21 = 85% -lsb_heka_pm_output(lsb_heka_sandbox*,lsb_heka_message*,void*,_Bool) ../src/heka/sandbox.c 718 1 / 1 12 / 14 = 85% +inject_message_input(lua_State*) ../src/heka/sandbox.c 56 1 / 1 18 / 21 = 85% +lsb_heka_pm_output(lsb_heka_sandbox*,lsb_heka_message*,void*,_Bool) ../src/heka/sandbox.c 733 1 / 1 12 / 14 = 85% lsb_expand_input_buffer(lsb_input_buffer*,size_t) ...c/util/input_buffer.c 49 1 / 1 12 / 14 = 85% numeric_test(match_node*,double) ...eka/message_matcher.c 213 1 / 1 6 / 7 = 85% +add_consumer_topics(lua_State*,heka_kafka_consumer*,int) ../src/heka/kafka.c 528 1 / 1 19 / 22 = 86% load_sandbox_config(const char*,lsb_logger) ../src/luasandbox.c 222 1 / 1 19 / 22 = 86% matchbracketclass(int,const char*,const char*) ...util/string_matcher.c 99 1 / 1 19 / 22 = 86% eval_tree(match_node*,lsb_heka_message*) ...eka/message_matcher.c 297 1 / 1 14 / 16 = 87% -hj_type(lua_State*) ...rc/heka/rapidjson.cpp 237 1 / 1 7 / 8 = 87% -hj_size(lua_State*) ...rc/heka/rapidjson.cpp 266 1 / 1 7 / 8 = 87% -update_checkpoint(lua_State*) ../src/heka/sandbox.c 220 1 / 1 7 / 8 = 87% +lsb_heka_timer_event(lsb_heka_sandbox*,time_t,_Bool) ../src/heka/sandbox.c 759 1 / 1 14 / 16 = 87% +update_checkpoint(lua_State*) ../src/heka/sandbox.c 223 1 / 1 7 / 8 = 87% lsb_init_input_buffer(lsb_input_buffer*,size_t) ...c/util/input_buffer.c 20 1 / 1 7 / 8 = 87% lsb_outputd(lsb_output_buffer*,double) .../util/output_buffer.c 160 1 / 1 7 / 8 = 87% process_fields(lsb_heka_field*,const char*,const char*) ...c/util/heka_message.c 121 1 / 1 50 / 57 = 87% copy_table(lua_State*,lua_State*,lsb_logger) ../src/luasandbox.c 262 1 / 1 22 / 25 = 88% +hj_value(lua_State*) ...rc/heka/rapidjson.cpp 380 1 / 1 8 / 9 = 88% load_expression_node(lua_State*,match_node*) ...eka/message_matcher.c 335 1 / 1 42 / 47 = 89% lsb_decode_heka_message(lsb_heka_message*,const char*,size_t,lsb_logger) ...c/util/heka_message.c 201 1 / 1 51 / 57 = 89% -inject_payload(lua_State*) ../src/heka/sandbox.c 162 1 / 1 18 / 20 = 90% +inject_payload(lua_State*) ../src/heka/sandbox.c 165 1 / 1 18 / 20 = 90% +check_value(lua_State*) ...rc/heka/rapidjson.cpp 55 1 / 1 9 / 10 = 90% +hj_type(lua_State*) ...rc/heka/rapidjson.cpp 246 1 / 1 9 / 10 = 90% +hj_size(lua_State*) ...rc/heka/rapidjson.cpp 279 1 / 1 9 / 10 = 90% lsb_add_function(lsb_lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 663 1 / 1 9 / 10 = 90% lsb_init_output_buffer(lsb_output_buffer*,size_t) .../util/output_buffer.c 27 1 / 1 9 / 10 = 90% -lsb_find_heka_message(lsb_heka_message*,lsb_input_buffer*,_Bool,size_t*,lsb_logger) ...c/util/heka_message.c 319 1 / 1 29 / 32 = 90% eval_node(match_node*,lsb_heka_message*) ...eka/message_matcher.c 235 1 / 1 31 / 34 = 91% +set_missing_headers(lua_State*,int,lsb_heka_sandbox*) ../src/heka/message.c 32 1 / 1 11 / 12 = 91% memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 77 1 / 1 11 / 12 = 91% lsb_read_heka_field(lsb_heka_message*,lsb_const_string*,int,int,lsb_read_value*) ...c/util/heka_message.c 462 1 / 1 24 / 26 = 92% +lsb_find_heka_message(lsb_heka_message*,lsb_input_buffer*,_Bool,size_t*,lsb_logger) ...c/util/heka_message.c 319 1 / 1 30 / 32 = 93% classend(const char*) ...util/string_matcher.c 43 1 / 1 16 / 17 = 94% lsb_expand_output_buffer(lsb_output_buffer*,size_t) .../util/output_buffer.c 52 1 / 1 17 / 18 = 94% string_test(match_node*,lsb_const_string*) ...eka/message_matcher.c 167 1 / 1 18 / 19 = 94% +check_producer(lua_State*,int) ../src/heka/kafka.c 47 1 / 1 0 / 0 +get_topic(lua_State*,heka_kafka_producer*,const char*) ../src/heka/kafka.c 288 1 / 1 2 / 2 = 100% +producer_has_topic(lua_State*) ../src/heka/kafka.c 355 1 / 1 2 / 2 = 100% +luaopen_heka_kafka_producer(lua_State*) ../src/heka/kafka.c 500 1 / 1 0 / 0 +check_consumer(lua_State*,int) ../src/heka/kafka.c 519 1 / 1 0 / 0 +consumer_gc(lua_State*) ../src/heka/kafka.c 720 1 / 1 6 / 6 = 100% +luaopen_heka_kafka_consumer(lua_State*) ../src/heka/kafka.c 745 1 / 1 0 / 0 encode_string(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 258 1 / 1 2 / 2 = 100% encode_field_object(lsb_lua_sandbox*,lsb_output_buffer*) ../src/heka/message.c 381 1 / 1 4 / 4 = 100% lsb_destroy_message_matcher(lsb_message_matcher*) ...eka/message_matcher.c 559 1 / 1 6 / 6 = 100% lsb_eval_message_matcher(lsb_message_matcher*,lsb_heka_message*) ...eka/message_matcher.c 578 1 / 1 0 / 0 -check_value(lua_State*) ...rc/heka/rapidjson.cpp 54 1 / 1 6 / 6 = 100% -schema_gc(lua_State*) ...rc/heka/rapidjson.cpp 71 1 / 1 0 / 0 -iter_gc(lua_State*) ...rc/heka/rapidjson.cpp 80 1 / 1 0 / 0 -hj_gc(lua_State*) ...rc/heka/rapidjson.cpp 89 1 / 1 0 / 0 -hj_validate(lua_State*) ...rc/heka/rapidjson.cpp 152 1 / 1 4 / 4 = 100% -hj_find(lua_State*) ...rc/heka/rapidjson.cpp 181 1 / 1 19 / 19 = 100% -hj_make_field(lua_State*) ...rc/heka/rapidjson.cpp 292 1 / 1 0 / 0 -hj_object_iter(lua_State*) ...rc/heka/rapidjson.cpp 305 1 / 1 4 / 4 = 100% -hj_array_iter(lua_State*) ...rc/heka/rapidjson.cpp 331 1 / 1 4 / 4 = 100% -hj_value(lua_State*) ...rc/heka/rapidjson.cpp 359 1 / 1 7 / 7 = 100% -read_message(lua_State*,int,lsb_heka_message*) ...rc/heka/rapidjson.cpp 428 1 / 1 14 / 14 = 100% -OutputBufferWrapper::OutputBufferWrapper(lsb_output_buffer*) ...rc/heka/rapidjson.cpp 532 1 / 1 0 / 0 -OutputBufferWrapper::Put(Ch) ...rc/heka/rapidjson.cpp 544 1 / 1 2 / 2 = 100% -OutputBufferWrapper::Flush() ...rc/heka/rapidjson.cpp 549 1 / 1 0 / 0 -OutputBufferWrapper::GetError() ...rc/heka/rapidjson.cpp 550 1 / 1 0 / 0 -luaopen_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 646 1 / 1 0 / 0 -lsb_heka_stop_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 694 1 / 1 0 / 0 -lsb_heka_terminate_sandbox(lsb_heka_sandbox*,const char*) ../src/heka/sandbox.c 700 1 / 1 0 / 0 -lsb_heka_destroy_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 706 1 / 1 2 / 2 = 100% -lsb_heka_get_error(lsb_heka_sandbox*) ../src/heka/sandbox.c 785 1 / 1 2 / 2 = 100% -lsb_heka_get_lua_file(lsb_heka_sandbox*) ../src/heka/sandbox.c 791 1 / 1 2 / 2 = 100% -lsb_heka_get_stats(lsb_heka_sandbox*) ../src/heka/sandbox.c 797 1 / 1 2 / 2 = 100% -lsb_heka_is_running(lsb_heka_sandbox*) ../src/heka/sandbox.c 818 1 / 1 4 / 4 = 100% +schema_gc(lua_State*) ...rc/heka/rapidjson.cpp 78 1 / 1 0 / 0 +iter_gc(lua_State*) ...rc/heka/rapidjson.cpp 87 1 / 1 0 / 0 +hj_gc(lua_State*) ...rc/heka/rapidjson.cpp 96 1 / 1 0 / 0 +hj_validate(lua_State*) ...rc/heka/rapidjson.cpp 161 1 / 1 4 / 4 = 100% +hj_find(lua_State*) ...rc/heka/rapidjson.cpp 190 1 / 1 19 / 19 = 100% +hj_make_field(lua_State*) ...rc/heka/rapidjson.cpp 309 1 / 1 2 / 2 = 100% +hj_object_iter(lua_State*) ...rc/heka/rapidjson.cpp 326 1 / 1 4 / 4 = 100% +hj_array_iter(lua_State*) ...rc/heka/rapidjson.cpp 352 1 / 1 4 / 4 = 100% +read_message(lua_State*,int,lsb_heka_message*) ...rc/heka/rapidjson.cpp 457 1 / 1 14 / 14 = 100% +OutputBufferWrapper::OutputBufferWrapper(lsb_output_buffer*) ...rc/heka/rapidjson.cpp 566 1 / 1 0 / 0 +OutputBufferWrapper::Put(Ch) ...rc/heka/rapidjson.cpp 578 1 / 1 2 / 2 = 100% +OutputBufferWrapper::Flush() ...rc/heka/rapidjson.cpp 583 1 / 1 0 / 0 +OutputBufferWrapper::GetError() ...rc/heka/rapidjson.cpp 584 1 / 1 0 / 0 +lsb_heka_stop_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 709 1 / 1 0 / 0 +lsb_heka_terminate_sandbox(lsb_heka_sandbox*,const char*) ../src/heka/sandbox.c 715 1 / 1 0 / 0 +lsb_heka_destroy_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 721 1 / 1 2 / 2 = 100% +lsb_heka_get_error(lsb_heka_sandbox*) ../src/heka/sandbox.c 800 1 / 1 2 / 2 = 100% +lsb_heka_get_lua_file(lsb_heka_sandbox*) ../src/heka/sandbox.c 806 1 / 1 2 / 2 = 100% +lsb_heka_get_stats(lsb_heka_sandbox*) ../src/heka/sandbox.c 812 1 / 1 2 / 2 = 100% +lsb_heka_is_running(lsb_heka_sandbox*) ../src/heka/sandbox.c 833 1 / 1 4 / 4 = 100% check_hsr(lua_State*,int) .../heka/stream_reader.c 68 1 / 1 0 / 0 -hsr_read_message(lua_State*) .../heka/stream_reader.c 196 1 / 1 6 / 6 = 100% -hsr_gc(lua_State*) .../heka/stream_reader.c 208 1 / 1 0 / 0 -luaopen_heka_stream_reader(lua_State*) .../heka/stream_reader.c 235 1 / 1 0 / 0 +hsr_read_message(lua_State*) .../heka/stream_reader.c 192 1 / 1 6 / 6 = 100% +hsr_gc(lua_State*) .../heka/stream_reader.c 204 1 / 1 0 / 0 +luaopen_heka_stream_reader(lua_State*) .../heka/stream_reader.c 231 1 / 1 0 / 0 libsize(const luaL_Reg*) ../src/luasandbox.c 43 1 / 1 2 / 2 = 100% preload_modules(lua_State*) ../src/luasandbox.c 50 1 / 1 2 / 2 = 100% instruction_usage(lsb_lua_sandbox*) ../src/luasandbox.c 108 1 / 1 0 / 0 @@ -178,4 +197,4 @@ lsb_string_match(const char*,size_t,const char*) lsb_lp2(unsigned long long) ../src/util/util.c 37 1 / 1 2 / 2 = 100% lsb_get_time() ../src/util/util.c 77 1 / 1 0 / 0 -------------------------------------------------------------------------------------------------- ------------------------------ ----- --------------------- -Total 100% 1712 / 2138 = 80% +Total 100% 1874 / 2337 = 80% diff --git a/docs/heka/index.md b/docs/heka/index.md index dae722b..4e59df4 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -13,11 +13,13 @@ of that infrastructure has been created to replace it called * [Message Matcher](message_matcher.md) * [Input Sandbox](input.md) - * [Heka Stream Reader](heka_stream_reader.md) + * [Heka Stream Reader](stream_reader.md) * [Heka JSON](heka_json.md) + * [Heka Kafka Consumer](kafka_consumer.md) * [Analysis Sandbox](analysis.md) * [Output Sandbox](output.md) * [Heka JSON](heka_json.md) + * [Heka Kafka Producer](kafka_producer.md) ## Sandbox API Changes from the Go Heka Sandbox diff --git a/docs/heka/kafka_consumer.md b/docs/heka/kafka_consumer.md new file mode 100644 index 0000000..a11584e --- /dev/null +++ b/docs/heka/kafka_consumer.md @@ -0,0 +1,49 @@ +## Heka Kafka Consumer + +Kafka Lua module for input sandbox plugins. + +### API + +#### new + +Creates a Heka Kafka consumer. + +```lua +local brokerlist = "localhost:9092" +local topics = {"test"} +local consumer_conf = {["group.id"] = "test_g1"}) +local topic_conf = nil +local consumer = heka_kafka_consumer.new(brokerlist, topics, consumer_conf, topic_conf) + +``` + +*Arguments* +* brokerlist (string) - [librdkafka broker string](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205) +* topics (array of 'topic[:partition]' strings) - Balanced consumer group mode a + consumer can only subscribe on topics, not topics:partitions. The partition + syntax is only used for manual assignments (without balanced consumer groups). +* consumer_conf (table) - must contain 'group.id' see: [librdkafka consumer configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties) +* topic_conf (table, optional) - [librdkafka topic configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties) + +*Return* +* consumer (userdata) - Kafka consumer or an error is thrown + +### API Methods + +#### receive + +Receives a message from the specified Kafka topic(s). + +```lua +local msg, topic, partition, key = consumer:receive() + +``` + +*Arguments* +* none + +*Return* +* msg (string) - Kafka message payload +* topic (string) - Topic name the message was received from +* partition (number) - Topic partition the message was received from +* key (string) - Message key (if available) diff --git a/docs/heka/kafka_producer.md b/docs/heka/kafka_producer.md new file mode 100644 index 0000000..623b5ae --- /dev/null +++ b/docs/heka/kafka_producer.md @@ -0,0 +1,118 @@ +## Heka Kafka Producer + +Kafka Lua module for output sandboxes. + +### API + +#### new + +Creates a Heka Kafka producer. + +```lua +local brokerlist = "localhost:9092" +local producer_conf = producer_conf = { + ["queue.buffering.max.messages"] = 20000, + ["batch.num.messages"] = 200, + ["message.max.bytes"] = 1024 * 1024, + ["queue.buffering.max.ms"] = 10, + ["topic.metadata.refresh.interval.ms"] = -1, +} +local producer = heka_kafka_producer.new(brokerlist, producer_conf) + +``` + +*Arguments* +* brokerlist (string) - [librdkafka broker string](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205) +* producer_conf (table) - [librdkafka producer configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties) + +*Return* +* producer (userdata) - Kafka producer or an error is thrown + +### API Methods + +#### create_topic + +Creates a topic to be used by a producer, no-op if the topic already exists. + +```lua +producer:create_topic(topic) -- creates the topic if it does not exist + +``` + +*Arguments* +* topic (string) - Name of the topic + +*Return* +* none + + +#### has_topic + +Tests if a producer is managing a topic. + +```lua +local b = producer:has_topic(topic) + +``` + +*Arguments* +* topic (string) - Name of the topic + +*Return* +* bool - True if the producer is managing a topic with the specificed name + + +#### destroy_topic + +Removes a topic from the producer. + +```lua +producer:destroy_topic(topic) + +``` + +*Arguments* +* topic (string) - Name of the topic + +*Return* +* none - no-op on non-existent topic + + +#### send + +Sends a message using the specified topic. + +```lua +local ret = producer:send(topic, -1, sequence_id, payload) + +``` + +*Arguments* +* topic (string) - Name of the topic +* partition (number) - Topic partition number (-1 for automatic assignment) +* sequence_id (lightuserdata) - Opaque pointer for checkpointing (passed into process_message) +* payload (string or (nil/none)) - Use nil/none to send the current active message while in + the process_message call + +*Return* +* ret (number) - 0 on success or errno + - ENOBUFS (105) maximum number of outstanding messages has been reached + - EMSGSIZE (90) message is larger than configured max size + - ESRCH (2) requested partition is unknown in the Kafka cluster + - ENOENT (3) topic is unknown in the Kafka cluster + +#### poll + +Polls the provided Kafka producer for events and invokes callback. This should +be called after every send. + +```lua +producer:poll() + +``` + +*Arguments* +* none + +*Return* +* none diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index da8292b..815d27c 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -30,6 +30,7 @@ #endif #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" +#define LSB_HEKA_UPDATE_CHECKPOINT "update_checkpoint" enum lsb_heka_pm_rv { LSB_HEKA_PM_SENT = 0, diff --git a/sandboxes/heka/input/heka_kafka.lua b/sandboxes/heka/input/heka_kafka.lua new file mode 100644 index 0000000..bdca290 --- /dev/null +++ b/sandboxes/heka/input/heka_kafka.lua @@ -0,0 +1,59 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "heka_kafka_consumer" + +--[[ + +*Example Configuration* + + filename = "heka_kafka.lua" + output_limit = 8 * 1024 * 1024 + brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 + + -- in balanced consumer group mode a consumer can only subscribe on topics, not topics:partitions. + -- The partition syntax is only used for manual assignments (without balanced consumer groups). + topics = {"test"} + ticker_interval = 60 + + -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties + consumer_conf = { + ["group.id"] = "test_group", -- must always be provided (a single consumer is considered a group of one + -- in that case make this a unique identifier) + ["message.max.bytes"] = output_limit, + } + + -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties + topic_conf = { + -- ["auto.commit.enable"] = true, -- cannot be overridden + -- ["offset.store.method"] = "broker, -- cannot be overridden + } + +--]] +local brokerlist = read_config("brokerlist") or error("brokerlist must be set") +local topics = read_config("topics") or error("topics must be set") +local consumer_conf = read_config("consumer_conf") +local topic_conf = read_config("topic_conf") + +local consumer = heka_kafka_consumer.new(brokerlist, topics, consumer_conf, topic_conf) + +local err_msg = { + Logger = read_config("Logger"), + Type = "error", + Payload = nil, +} + +function process_message() + while true do + local msg, topic, partition, key = consumer:receive() + if msg then + local ok, err = pcall(inject_message, msg) + if not ok then + err_msg.Payload = err + pcall(inject_message, err_msg) + end + end + end + return 0 -- unreachable but here for consistency +end diff --git a/sandboxes/heka/output/heka_kafka.lua b/sandboxes/heka/output/heka_kafka.lua new file mode 100644 index 0000000..6cb697d --- /dev/null +++ b/sandboxes/heka/output/heka_kafka.lua @@ -0,0 +1,62 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "heka_kafka_producer" + +--[[ + +*Example Configuration* + + filename = "heka_kafka.lua" + message_matcher = "TRUE" + output_limit = 8 * 1024 * 1024 + brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 + ticker_interval = 60 + async_buffer_size = 20000 + + topic_constant = "test" + producer_conf = { + ["queue.buffering.max.messages"] = async_buffer_size, + ["batch.num.messages"] = 200, + ["message.max.bytes"] = output_limit, + ["queue.buffering.max.ms"] = 10, + ["topic.metadata.refresh.interval.ms"] = -1, + } + +--]] +local brokerlist = read_config("brokerlist") or error("brokerlist must be set") +local topic_constant = read_config("topic_constant") +local topic_variable = read_config("topic_variable") or "Logger" +local producer_conf = read_config("producer_conf") + +local producer = heka_kafka_producer.new(brokerlist, producer_conf) + +function process_message(sequence_id) + local topic = topic_constant + if not topic then + topic = read_message(topic_variable) or "unknown" + end + producer:create_topic(topic) -- creates the topic if it does not exist + + producer:poll() + local ret = producer:send(topic, -1, sequence_id) -- sends the current message + + if ret ~= 0 then + if ret == 105 then + return -3, "queue full" -- retry + elseif ret == 90 then + return -1, "message too large" -- fail + elseif ret == 2 then + error("unknown topic: " .. topic) + elseif ret == 3 then + error("unknown partition") + end + end + + return -5 -- asynchronous checkpoint management +end + +function timer_event(ns) + producer:poll() +end diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index f82f3bc..1e99f1b 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -10,6 +10,10 @@ sandbox.c stream_reader.c ) +if(LIBRTKAFKA_LIBRARY) + set(HEKA_SRC ${HEKA_SRC} kafka.c) +endif() + add_library(luasandboxheka SHARED ${HEKA_SRC}) set_target_properties(luasandboxheka PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) target_compile_definitions(luasandboxheka PRIVATE -Dluasandboxheka_EXPORTS) @@ -17,9 +21,14 @@ target_link_libraries(luasandboxheka luasandbox) if(WIN32) target_link_libraries(luasandboxheka ws2_32) endif() + if(LIBM_LIBRARY) target_link_libraries(luasandboxheka ${LIBM_LIBRARY}) endif() +if(LIBRTKAFKA_LIBRARY) + target_link_libraries(luasandboxheka ${LIBRTKAFKA_LIBRARY}) +endif() + install(TARGETS luasandboxheka DESTINATION ${LIB_DIR} COMPONENT core) add_subdirectory(test) diff --git a/src/heka/kafka.c b/src/heka/kafka.c new file mode 100644 index 0000000..b242bf6 --- /dev/null +++ b/src/heka/kafka.c @@ -0,0 +1,753 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua Kafka functions @file */ + +#include +#include +#include +#include +#include +#include +#include + +#include "luasandbox/heka/sandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "kafka_impl.h" +#include "sandbox_impl.h" + +const char *mozsvc_heka_kafka_producer = "mozsvc.heka_kafka_producer"; +const char *mozsvc_heka_kafka_producer_table = "heka_kafka_producer"; + +const char *mozsvc_heka_kafka_consumer = "mozsvc.heka_kafka_consumer"; +const char *mozsvc_heka_kafka_consumer_table = "heka_kafka_consumer"; + +typedef struct heka_kafka_producer { + rd_kafka_t *rk; + void *msg_opaque; + int failures; +} heka_kafka_producer; + + +typedef struct heka_kafka_consumer { + rd_kafka_t *rk; + rd_kafka_topic_partition_list_t *topics; +} heka_kafka_consumer; + + +typedef struct heka_kafka_topic { + rd_kafka_topic_t *rkt; +} heka_kafka_topic; + + +static heka_kafka_producer* check_producer(lua_State *lua, int min_args) +{ + heka_kafka_producer *kp = luaL_checkudata(lua, 1, mozsvc_heka_kafka_producer); + int n = lua_gettop(lua); + luaL_argcheck(lua, min_args <= n, n, "incorrect number of arguments"); + return kp; +} + + +static void msg_delivered(rd_kafka_t *rk, + void *payload, + size_t len, + int error_code, + void *opaque, + void *msg_opaque) +{ + (void)rk; + (void)payload; + (void)len; + heka_kafka_producer *kp = (heka_kafka_producer *)opaque; + kp->msg_opaque = msg_opaque; + if (error_code) ++kp->failures; +} + + +static bool load_conf(lua_State *lua, rd_kafka_conf_t *conf, int idx) +{ + if (!conf) { + lua_pushstring(lua, "rd_kafka_conf_new() failed"); + return false; + } + if (lua_isnil(lua, idx)) { + return true; + } + + char errstr[512]; + lua_pushnil(lua); + while (lua_next(lua, idx) != 0) { + int kt = lua_type(lua, -2); + int vt = lua_type(lua, -1); + switch (kt) { + case LUA_TSTRING: + switch (vt) { + case LUA_TSTRING: + { + const char *key = lua_tostring(lua, -2); + const char *value = lua_tostring(lua, -1); + if (value) { + rd_kafka_conf_res_t r; + r = rd_kafka_conf_set(conf, key, value, errstr, sizeof errstr); + if (r) { + lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, + errstr); + return false; + } + } + } + break; + case LUA_TNUMBER: + { + const char *key = lua_tostring(lua, -2); + int i = (int)lua_tointeger(lua, -1); + char value[12]; + snprintf(value, sizeof value, "%d", i); + rd_kafka_conf_res_t r; + r = rd_kafka_conf_set(conf, key, value, errstr, sizeof errstr); + if (r) { + lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, + errstr); + return false; + } + } + break; + case LUA_TBOOLEAN: + { + const char *key = lua_tostring(lua, -2); + const char *value = "false"; + if (lua_toboolean(lua, -1)) { + value = "true"; + } + rd_kafka_conf_res_t r; + r = rd_kafka_conf_set(conf, key, value, errstr, sizeof errstr); + if (r) { + lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, + errstr); + return false; + } + } + break; + default: + lua_pushfstring(lua, "invalid config value type: %s", + lua_typename(lua, vt)); + return false; + } + break; + default: + lua_pushfstring(lua, "invalid config key type: %s", + lua_typename(lua, kt)); + return false; + } + lua_pop(lua, 1); + } + return true; +} + + +static +bool load_topic_conf(lua_State *lua, rd_kafka_topic_conf_t *conf, int idx) +{ + if (!conf) { + lua_pushstring(lua, "rd_kafka_topic_conf_new() failed"); + return false; + } + if (lua_isnil(lua, idx)) { + return true; + } + + char errstr[512]; + lua_pushnil(lua); + while (lua_next(lua, idx) != 0) { + int kt = lua_type(lua, -2); + int vt = lua_type(lua, -1); + switch (kt) { + case LUA_TSTRING: + switch (vt) { + case LUA_TSTRING: + { + const char *key = lua_tostring(lua, -2); + const char *value = lua_tostring(lua, -1); + if (value) { + rd_kafka_conf_res_t r; + r = rd_kafka_topic_conf_set(conf, key, value, errstr, + sizeof errstr); + if (r) { + lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, + errstr); + return false; + } + } + } + break; + case LUA_TNUMBER: + { + const char *key = lua_tostring(lua, -2); + int i = (int)lua_tointeger(lua, -1); + char value[12]; + snprintf(value, sizeof value, "%d", i); + rd_kafka_conf_res_t r; + r = rd_kafka_topic_conf_set(conf, key, value, errstr, + sizeof errstr); + if (r) { + lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, + errstr); + return false; + } + } + break; + case LUA_TBOOLEAN: + { + const char *key = lua_tostring(lua, -2); + const char *value = "false"; + if (lua_toboolean(lua, -1)) { + value = "true"; + } + rd_kafka_conf_res_t r; + r = rd_kafka_topic_conf_set(conf, key, value, errstr, + sizeof errstr); + if (r) { + lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, + errstr); + return false; + } + } + break; + default: + lua_pushfstring(lua, "invalid config value type: %s", + lua_typename(lua, vt)); + return false; + } + break; + default: + lua_pushfstring(lua, "invalid config key type: %s", + lua_typename(lua, kt)); + return false; + } + lua_pop(lua, 1); + } + return true; +} + + +static int producer_new(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n >= 1 && n <= 2, n, "incorrect number of arguments"); + + const char *brokerlist = luaL_checkstring(lua, 1); + int t = lua_type(lua, 2); + switch (t) { + case LUA_TNONE: + lua_pushnil(lua); + break; + case LUA_TNIL: + break; + default: + luaL_checktype(lua, 2, LUA_TTABLE); // producer config + } + + heka_kafka_producer *kp = lua_newuserdata(lua, sizeof(heka_kafka_producer)); + kp->rk = NULL; + kp->msg_opaque = NULL; + kp->failures = 0; + lua_pushlightuserdata(lua, kp); // setup a topic table for this producer + lua_newtable(lua); + lua_rawset(lua, LUA_ENVIRONINDEX); + luaL_getmetatable(lua, mozsvc_heka_kafka_producer); + lua_setmetatable(lua, -2); + + rd_kafka_conf_t *conf = rd_kafka_conf_new(); + if (!load_conf(lua, conf, 2)) { + rd_kafka_conf_destroy(conf); + return lua_error(lua); + } + rd_kafka_conf_set_opaque(conf, kp); + rd_kafka_conf_set_dr_cb(conf, msg_delivered); + rd_kafka_conf_set_log_cb(conf, NULL); // disable logging + + char errstr[512]; + kp->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof errstr); + if (!kp->rk) { + rd_kafka_conf_destroy(conf); // the producer has not taken ownership + return luaL_error(lua, "rd_kafka_new failed: %s", errstr); + } + + if (rd_kafka_brokers_add(kp->rk, brokerlist) == 0) { + return luaL_error(lua, "invalid broker list"); + } + return 1; +} + + +static heka_kafka_topic* get_topic(lua_State *lua, heka_kafka_producer *kp, + const char *topic) +{ + heka_kafka_topic *kt = NULL; + lua_pushlightuserdata(lua, kp); + lua_rawget(lua, LUA_ENVIRONINDEX); + lua_getfield(lua, -1, topic); + if (lua_type(lua, -1) == LUA_TLIGHTUSERDATA) { + kt = lua_touserdata(lua, -1);; + } + lua_pop(lua, 2); + return kt; +} + + +static int producer_create_topic(lua_State *lua) +{ + heka_kafka_producer *kp = check_producer(lua, 2); + const char *topic = NULL; + int n = lua_gettop(lua); + luaL_argcheck(lua, n >= 2 && n <= 3, n, "incorrect number of arguments"); + + topic = luaL_checkstring(lua, 2); + + int t = lua_type(lua, 3); + switch (t) { + case LUA_TNONE: + lua_pushnil(lua); + break; + case LUA_TNIL: + break; + default: + luaL_checktype(lua, 3, LUA_TTABLE); // topic config + } + + if (get_topic(lua, kp, topic)) { + return 0; + } + + heka_kafka_topic *kt = malloc(sizeof(heka_kafka_topic)); + if (!kt) { + return luaL_error(lua, "memory allocation failed"); + } + + rd_kafka_topic_conf_t *tconf = rd_kafka_topic_conf_new(); + if (!load_topic_conf(lua, tconf, 3)) { + rd_kafka_topic_conf_destroy(tconf); + free(kt); + return lua_error(lua); + } + + char errstr[512]; + kt->rkt = rd_kafka_topic_new(kp->rk, topic, tconf); + if (!kt->rkt) { + rd_kafka_topic_conf_destroy(tconf); + free(kt); + return luaL_error(lua, "rd_kafka_topic_new failed: %s", errstr); + } + lua_pushlightuserdata(lua, kp); + lua_rawget(lua, LUA_ENVIRONINDEX); + lua_pushstring(lua, topic); + lua_pushlightuserdata(lua, kt); + lua_rawset(lua, -3); // update the producer topic table + return 0; +} + + +static int producer_has_topic(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, 2 == n, n, "incorrect number of arguments"); + heka_kafka_producer *kp = luaL_checkudata(lua, 1, mozsvc_heka_kafka_producer); + const char *topic = luaL_checkstring(lua, 2); + if (get_topic(lua, kp, topic)) { + lua_pushboolean(lua, true); + } else { + lua_pushboolean(lua, false); + } + return 1; +} + + +static int producer_destroy_topic(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, 2 == n, n, "incorrect number of arguments"); + heka_kafka_producer *kp = luaL_checkudata(lua, 1, mozsvc_heka_kafka_producer); + const char *topic = luaL_checkstring(lua, 2); + heka_kafka_topic *kt = get_topic(lua, kp, topic); + if (kt) { + lua_pushlightuserdata(lua, kp); + lua_rawget(lua, LUA_ENVIRONINDEX); + lua_pushnil(lua); + lua_setfield(lua, -2, topic); + rd_kafka_topic_destroy(kt->rkt); + free(kt); + } + return 0; +} + + +static int producer_poll(lua_State *lua) +{ + heka_kafka_producer *kp = check_producer(lua, 1); + kp->failures = 0; + kp->msg_opaque = NULL; + rd_kafka_poll(kp->rk, 0); + if (kp->msg_opaque) { + lua_getfield(lua, LUA_GLOBALSINDEX, LSB_HEKA_UPDATE_CHECKPOINT); + if (lua_type(lua, -1) == LUA_TFUNCTION) { + lua_pushlightuserdata(lua, kp->msg_opaque); + lua_pushinteger(lua, kp->failures); + if (lua_pcall(lua, 2, 0, 0)) { + lua_error(lua); + } + } + lua_pop(lua, 1); + } + return 0; +} + + +static int producer_send(lua_State *lua) +{ + heka_kafka_producer *kp = check_producer(lua, 4); + const char *topic = luaL_checkstring(lua, 2); + heka_kafka_topic *kt = get_topic(lua, kp, topic); + if (!kt) return luaL_error(lua, "invalid topic"); + + int32_t partition = (int32_t)luaL_checkinteger(lua, 3); + luaL_checktype(lua, 4, LUA_TLIGHTUSERDATA); + void *sequence_id = lua_touserdata(lua, 4); + + size_t len = 0; + const char *msg = NULL; + if (lua_type(lua, 5) == LUA_TSTRING) { + msg = lua_tolstring(lua, 5, &len); + } else { + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "%s() invalid lightuserdata", __func__); + } + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + if (hsb->msg) { + len = hsb->msg->raw.len; + msg = hsb->msg->raw.s; + } else { + return luaL_error(lua, "no active message", __func__); + } + } + + errno = 0; + int ret = rd_kafka_produce(kt->rkt, partition, + RD_KAFKA_MSG_F_COPY, + (void *)msg, len, + NULL, 0, // optional key/len + sequence_id // opaque pointer + ); + if (ret == -1) { + lua_pushinteger(lua, errno); + } else { + lua_pushinteger(lua, 0); + } + return 1; +} + + +static int producer_gc(lua_State *lua) +{ + heka_kafka_producer *kp = check_producer(lua, 1); + lua_pushlightuserdata(lua, kp); + lua_rawget(lua, LUA_ENVIRONINDEX); + assert(lua_type(lua, -1) == LUA_TTABLE); + lua_pushnil(lua); /* first key */ + while (lua_next(lua, -2) != 0) { + heka_kafka_topic *kt = lua_touserdata(lua, -1); + if (kt) { + rd_kafka_topic_destroy(kt->rkt); + free(kt); + } + lua_pop(lua, 1); + } + if (kp->rk) rd_kafka_destroy(kp->rk); + + lua_pushlightuserdata(lua, kp); + lua_pushnil(lua); + lua_rawset(lua, LUA_ENVIRONINDEX); // remove the producer topic table + + // This may timeout because it might not be the last plugin running. + rd_kafka_wait_destroyed(1000); + return 0; +} + + +static const struct luaL_reg producerlib_f[] = +{ + { "new", producer_new }, + { NULL, NULL } +}; + + +static const struct luaL_reg producerlib_m[] = +{ + { "create_topic", producer_create_topic }, + { "has_topic", producer_has_topic }, + { "destroy_topic", producer_destroy_topic }, + { "poll", producer_poll }, + { "__gc", producer_gc }, + { NULL, NULL } +}; + + +int luaopen_heka_kafka_producer(lua_State *lua) +{ + lua_newtable(lua); + lua_replace(lua, LUA_ENVIRONINDEX); + + luaL_newmetatable(lua, mozsvc_heka_kafka_producer); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, producerlib_m); + // special case send since it needs access to the sandbox + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + lua_pushlightuserdata(lua, lsb); + lua_pushcclosure(lua, producer_send, 1); + lua_setfield(lua, -2, "send"); + luaL_register(lua, mozsvc_heka_kafka_producer_table, producerlib_f); + return 1; +} + + +static heka_kafka_consumer* check_consumer(lua_State *lua, int args) +{ + heka_kafka_consumer *kc = luaL_checkudata(lua, 1, mozsvc_heka_kafka_consumer); + int n = lua_gettop(lua); + luaL_argcheck(lua, args == n, n, "incorrect number of arguments"); + return kc; +} + + +static bool add_consumer_topics(lua_State *lua, + heka_kafka_consumer *kc, + int cnt) +{ + bool is_subscription = true; + + kc->topics = rd_kafka_topic_partition_list_new(cnt); + if (!kc->topics) { + lua_pushstring(lua, "rd_kafka_topic_partition_list_new failed"); + return false; + } + + lua_pushnil(lua); + while (lua_next(lua, 2) != 0) { + int kt = lua_type(lua, -2); + int vt = lua_type(lua, -1); + if (kt != LUA_TNUMBER || vt != LUA_TSTRING) { + lua_pushstring(lua, "topics must be an array of strings"); + return false; + } + const char *topic = lua_tostring(lua, -1); + char *t; + long partition = -1; + + if ((t = strstr(topic, ":"))) { // Parse "topic[:partition] + char s[strlen(topic)]; + memcpy(s, topic, t - topic); + s[t - topic] = 0; + + partition = strtol(t + 1, NULL, 10); + if (partition > INT32_MAX) { + lua_pushstring(lua, "invalid topic partition > INT32_MAX"); + return false; + } else if (partition < 0) { + lua_pushstring(lua, "invalid topic partition < 0"); + return false; + } + is_subscription = false; + rd_kafka_topic_partition_list_add(kc->topics, s, (int32_t)partition); + } else { + rd_kafka_topic_partition_list_add(kc->topics, topic, (int32_t)partition); + } + lua_pop(lua, 1); + } + + rd_kafka_resp_err_t err; + if (is_subscription) { + if ((err = rd_kafka_subscribe(kc->rk, kc->topics))) { + lua_pushfstring(lua, "rd_kafka_subscribe failed: %s", + rd_kafka_err2str(err)); + return false; + } + + } else { + if ((err = rd_kafka_assign(kc->rk, kc->topics))) { + lua_pushfstring(lua, "rd_kafka_assign failed: %s", rd_kafka_err2str(err)); + return false; + } + } + return true; +} + + +static int consumer_new(lua_State *lua) +{ + static const char *group_id = "group.id"; + int n = lua_gettop(lua); + luaL_argcheck(lua, n >= 3 && n <= 4, n, "incorrect number of arguments"); + + const char *brokerlist = luaL_checkstring(lua, 1); + + luaL_checktype(lua, 2, LUA_TTABLE); // topics + int topic_cnt = (int)lua_objlen(lua, 2); + luaL_argcheck(lua, topic_cnt > 0, 2, "the topics array is empty"); + + luaL_checktype(lua, 3, LUA_TTABLE); // consumer config + + int t = lua_type(lua, 4); // topic config + switch (t) { + case LUA_TNONE: + lua_pushnil(lua); + break; + case LUA_TNIL: + break; + default: + luaL_checktype(lua, 3, LUA_TTABLE); + } + + heka_kafka_consumer *kc = lua_newuserdata(lua, sizeof(heka_kafka_consumer)); + kc->rk = NULL; + kc->topics = NULL; + luaL_getmetatable(lua, mozsvc_heka_kafka_consumer); + lua_setmetatable(lua, -2); + + rd_kafka_conf_t *conf = rd_kafka_conf_new(); + if (!load_conf(lua, conf, 3)) { + rd_kafka_conf_destroy(conf); + return lua_error(lua); + } + + rd_kafka_conf_set_log_cb(conf, NULL); // disable logging + + char errstr[512]; + size_t len; + if (rd_kafka_conf_get(conf, group_id, NULL, &len) != RD_KAFKA_CONF_OK) { + rd_kafka_conf_destroy(conf); + return luaL_error(lua, "%s must be set", group_id); + } + + rd_kafka_topic_conf_t *tconf = rd_kafka_topic_conf_new(); + if (!load_topic_conf(lua, tconf, 4)) { + rd_kafka_topic_conf_destroy(tconf); + rd_kafka_conf_destroy(conf); + return lua_error(lua); + } + if (rd_kafka_topic_conf_set(tconf, "offset.store.method", "broker", + errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK + || rd_kafka_topic_conf_set(tconf, "auto.commit.enable", "true", + errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { + rd_kafka_topic_conf_destroy(tconf); + rd_kafka_conf_destroy(conf); + return luaL_error(lua, "rd_kafka_topic_conf_set failed: %s", errstr); + } + rd_kafka_conf_set_default_topic_conf(conf, tconf); + + kc->rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof errstr); + if (!kc->rk) { + rd_kafka_conf_destroy(conf); + return luaL_error(lua, "rd_kafka_new failed: %s", errstr); + } + + if (rd_kafka_brokers_add(kc->rk, brokerlist) == 0) { + return luaL_error(lua, "invalid broker list"); + } + + rd_kafka_poll_set_consumer(kc->rk); + if (!add_consumer_topics(lua, kc, topic_cnt)) { + return lua_error(lua); + } + return 1; +} + + +static int consumer_receive(lua_State *lua) +{ + bool err = false; + heka_kafka_consumer *kc = check_consumer(lua, 1); + rd_kafka_message_t *rkmessage = rd_kafka_consumer_poll(kc->rk, 1000); + if (rkmessage) { + if (rkmessage->err) { + if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || + rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) { + if (rkmessage->rkt) { + err = true; + lua_pushfstring(lua, "topic: %s partition: %d offset: %g err: %s", + rd_kafka_topic_name(rkmessage->rkt), + (int)rkmessage->partition, + (double)rkmessage->offset, + rd_kafka_message_errstr(rkmessage)); + } else { + err = true; + lua_pushfstring(lua, "%s err: %s", rd_kafka_err2str(rkmessage->err), + rd_kafka_message_errstr(rkmessage)); + } + } else { + lua_pushnil(lua); + lua_pushnil(lua); + lua_pushnil(lua); + lua_pushnil(lua); + } + } else { + lua_pushlstring(lua, rkmessage->payload, rkmessage->len); + lua_pushstring(lua, rd_kafka_topic_name(rkmessage->rkt)); + lua_pushinteger(lua, (lua_Integer)rkmessage->partition); + if (rkmessage->key_len) { + lua_pushlstring(lua, rkmessage->key, rkmessage->key_len); + } else { + lua_pushnil(lua); + } + } + rd_kafka_message_destroy(rkmessage); + } else { + lua_pushnil(lua); + lua_pushnil(lua); + lua_pushnil(lua); + lua_pushnil(lua); + } + if (err) return lua_error(lua); + return 4; +} + + +static int consumer_gc(lua_State *lua) +{ + heka_kafka_consumer *kc = check_consumer(lua, 1); + if (kc->rk) rd_kafka_consumer_close(kc->rk); + if (kc->topics) rd_kafka_topic_partition_list_destroy(kc->topics); + if (kc->rk) rd_kafka_destroy(kc->rk); + rd_kafka_wait_destroyed(1000); + return 0; +} + +static const struct luaL_reg consumerlib_f[] = +{ + { "new", consumer_new }, + { NULL, NULL } +}; + + +static const struct luaL_reg consumerlib_m[] = +{ + { "receive", consumer_receive }, + { "__gc", consumer_gc }, + { NULL, NULL } +}; + + +int luaopen_heka_kafka_consumer(lua_State *lua) +{ + luaL_newmetatable(lua, mozsvc_heka_kafka_consumer); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, consumerlib_m); + luaL_register(lua, mozsvc_heka_kafka_consumer_table, consumerlib_f); + return 1; +} diff --git a/src/heka/kafka_impl.h b/src/heka/kafka_impl.h new file mode 100644 index 0000000..be65866 --- /dev/null +++ b/src/heka/kafka_impl.h @@ -0,0 +1,20 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua Kafka constants/types @file */ + +#ifndef luasandbox_heka_kafka_impl_h_ +#define luasandbox_heka_kafka_impl_h_ + +#include "luasandbox/lua.h" + +extern const char* mozsvc_heka_kafka_producer_table; +extern const char* mozsvc_heka_kafka_consumer_table; + +int luaopen_heka_kafka_producer(lua_State *lua); +int luaopen_heka_kafka_consumer(lua_State *lua); + +#endif diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 8b8eea6..1e79b50 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -21,6 +21,9 @@ #include "sandbox_impl.h" #include "stream_reader_impl.h" #include "rapidjson_impl.h" +#ifdef HAVE_KAFKA +#include "kafka_impl.h" +#endif #ifdef _WIN32 #include @@ -219,7 +222,7 @@ static int inject_payload(lua_State *lua) static int update_checkpoint(lua_State *lua) { - static const char *func_name = "update_checkpoint"; + static const char *func_name = LSB_HEKA_UPDATE_CHECKPOINT; lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); if (!lsb) { @@ -396,6 +399,12 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lua_pushlightuserdata(lua, (void *)hsb->lsb); lua_pushcclosure(lua, luaopen_heka_json, 1); lua_rawset(lua, -3); +#ifdef HAVE_KAFKA +// preload the Heka Kafka consumer + lua_pushstring(lua, mozsvc_heka_kafka_consumer_table); + lua_pushcfunction(lua, luaopen_heka_kafka_consumer); + lua_rawset(lua, -3); +#endif lua_pop(lua, 1); // remove the preloaded table @@ -631,8 +640,8 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, } if (!ucp) { - if (logger) logger(__func__, 3, "update_checkpoint callback must be " - "specified"); + if (logger) logger(__func__, 3, LSB_HEKA_UPDATE_CHECKPOINT + " callback must be specified"); return NULL; } @@ -663,14 +672,20 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_add_function(hsb->lsb, read_message, "read_message"); lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); - lsb_add_function(hsb->lsb, update_checkpoint, "update_checkpoint"); - + lsb_add_function(hsb->lsb, update_checkpoint, LSB_HEKA_UPDATE_CHECKPOINT); // preload Heka JSON with a pointer back to the sandbox luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); lua_pushstring(lua, mozsvc_heka_json_table); lua_pushlightuserdata(lua, (void *)hsb->lsb); lua_pushcclosure(lua, luaopen_heka_json, 1); lua_rawset(lua, -3); +#ifdef HAVE_KAFKA +// preload the Heka Kafka producer + lua_pushstring(lua, mozsvc_heka_kafka_producer_table); + lua_pushlightuserdata(lua, (void *)hsb->lsb); + lua_pushcclosure(lua, luaopen_heka_kafka_producer, 1); + lua_rawset(lua, -3); +#endif lua_pop(lua, 1); // remove the preloaded table diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt index 9b759bf..0f23ca1 100644 --- a/src/heka/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -16,6 +16,12 @@ add_executable(test_message_matcher test_message_matcher.c) target_link_libraries(test_message_matcher luasandboxheka luasandboxutil) add_test(NAME test_message_matcher COMMAND test_message_matcher) +if(LIBRTKAFKA_LIBRARY) + add_executable(test_kafka test_kafka.c) + target_link_libraries(test_kafka luasandboxheka luasandbox) + add_test(NAME test_kafka COMMAND test_kafka CONFIGURATIONS integration) +endif() + if(WIN32) STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) diff --git a/src/heka/test/lua/kafka_consumer.lua b/src/heka/test/lua/kafka_consumer.lua new file mode 100644 index 0000000..f8c7763 --- /dev/null +++ b/src/heka/test/lua/kafka_consumer.lua @@ -0,0 +1,79 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "heka_kafka_consumer" +require "string" + +local ok, err +ok, err = pcall(heka_kafka_consumer.new) +assert(err == "bad argument #0 to '?' (incorrect number of arguments)", err) + +ok, err = pcall(heka_kafka_consumer.new, true, nil, nil) +assert(err == "bad argument #1 to '?' (string expected, got boolean)", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", true, nil) +assert(err == "bad argument #2 to '?' (table expected, got boolean)", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {}, nil) +assert(err == "bad argument #2 to '?' (the topics array is empty)", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {}) +assert(err == "group.id must be set", err) + +ok, err = pcall(heka_kafka_consumer.new, "", {"test"}, {["group.id"] = "foo"}) +assert(err == "invalid broker list", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = "foobar"}) +assert(err == 'Failed to set auto.offset.reset = foobar : Invalid value for configuration property "auto.offset.reset"', err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = 0}) +assert(err == 'Failed to set auto.offset.reset = 0 : Invalid value for configuration property "auto.offset.reset"', err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = true}) +assert(err == 'Failed to set auto.offset.reset = true : Invalid value for configuration property "auto.offset.reset"', err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = assert}) +assert(err == "invalid config value type: function", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {[assert] = true}) +assert(err == "invalid config key type: function", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test:9000000000"}, {["group.id"] = "foo"}) +assert(err == "invalid topic partition > INT32_MAX", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test:-1"}, {["group.id"] = "foo"}) +assert(err == "invalid topic partition < 0", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {true}, {["group.id"] = "foo"}) +assert(err == "topics must be an array of strings", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"one", item = "test"}, {["group.id"] = "foo"}) +assert(err == "topics must be an array of strings", err) + +ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo", ["message.max.bytes"] = true}) +assert(err == 'Failed to set message.max.bytes = true : Configuration property "message.max.bytes" value 0 is outside allowed range 1000..1000000000\n', err) + + +local consumer = heka_kafka_consumer.new("localhost:9092", {"test"}, {["group.id"] = "integration_testing"}) +local consumer1 = heka_kafka_consumer.new("localhost:9092", {"test:1"}, {["group.id"] = "other"}) +local pb, topic, partition, key = consumer1:receive() +assert(not pb) + +local payloads = {"one", "two", "three"} + +function process_message() + local cnt = 0 + for i=1, 10 do + pb, topic, partition, key = consumer:receive() + if pb then + cnt = cnt + 1 + local msg = decode_message(pb) + if msg.Payload ~= payloads[cnt] then + return -1, string.format("expected: %s received: %s", payloads[cnt], msg.Payload) + end + if cnt == 3 then return 0 end + end + end + return -1, string.format("received %d/3 messages", cnt) +end diff --git a/src/heka/test/lua/kafka_producer.lua b/src/heka/test/lua/kafka_producer.lua new file mode 100644 index 0000000..5943802 --- /dev/null +++ b/src/heka/test/lua/kafka_producer.lua @@ -0,0 +1,74 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "heka_kafka_producer" + +local ok, err +ok, err = pcall(heka_kafka_producer.new) +assert(err == "bad argument #0 to '?' (incorrect number of arguments)", err) + +ok, err = pcall(heka_kafka_producer.new, "") +assert(err == "invalid broker list", err) + +ok, err = pcall(heka_kafka_producer.new, "brokerlist", true) +assert(err == "bad argument #2 to '?' (table expected, got boolean)", err) + +ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = "foo"}) +assert(err == 'Failed to set message.max.bytes = foo : Configuration property "message.max.bytes" value 0 is outside allowed range 1000..1000000000\n', err) + +ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = 1}) +assert(err == 'Failed to set message.max.bytes = 1 : Configuration property "message.max.bytes" value 1 is outside allowed range 1000..1000000000\n', err) + +ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = true}) +assert(err == 'Failed to set message.max.bytes = true : Configuration property "message.max.bytes" value 0 is outside allowed range 1000..1000000000\n', err) + +ok, err = pcall(heka_kafka_producer.new, "brokerlist", {[assert] = true}) +assert(err == "invalid config key type: function", err) + +ok, err = pcall(heka_kafka_producer.new, "brokerlist", {foo = assert}) +assert(err == "invalid config value type: function", err) + +local producer = heka_kafka_producer.new("localhost:9092", + { + ["topic.metadata.refresh.interval.ms"] = -1, + ["batch.num.messages"] = 1, + ["queue.buffering.max.ms"] = 1, + }) +local cnt = 0 +local topic = "test" +local sid +function process_message(sequence_id) + sid = sequence_id + if cnt == 0 then + local topic_tmp = "tmp" + producer:create_topic(topic) + assert(producer:has_topic(topic)) + assert(0 == producer:send(topic, -1, sequence_id)) -- send the original + + producer:create_topic(topic_tmp) + producer:create_topic(topic_tmp, nil) + assert(producer:has_topic(topic_tmp)) + producer:destroy_topic(topic_tmp) + assert(not producer:has_topic(topic_tmp)) + ok, err = pcall(producer.create_topic, producer, "topic", true) + assert(err == "bad argument #3 to '?' (table expected, got boolean)", err) + ok, err = pcall(producer.send, producer, "foobar", -1, sequence_id) + assert(err == "invalid topic", err) + elseif cnt == 1 then + assert(0 == producer:send(topic, -1, sequence_id, encode_message({Payload = "two"}))) + elseif cnt == 2 then + assert(0 == producer:send(topic, -1, sequence_id, encode_message({Payload = "three"}))) + end + cnt = cnt + 1 + return 0 +end + +function timer_event(ns) + if sid then + ok, err = pcall(producer.send, producer, topic, -1, sid) + assert(err == "no active message", err) + sid = nil + end + producer:poll() +end diff --git a/src/heka/test/test_kafka.c b/src/heka/test/test_kafka.c new file mode 100644 index 0000000..cb66b12 --- /dev/null +++ b/src/heka/test/test_kafka.c @@ -0,0 +1,114 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Heka Kafka sandbox integration tests @file */ + +#include "test.h" + +#include +#include + +#include "luasandbox/heka/sandbox.h" + +char *e = NULL; + +void dlog(const char *component, int level, const char *fmt, ...) +{ + + va_list args; + va_start(args, fmt); + fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, + component ? component : "unnamed"); + vfprintf(stderr, fmt, args); + fwrite("\n", 1, 1, stderr); + va_end(args); +} + +static volatile ptrdiff_t g_sequence = 0; +static int ucp(void *parent, void *sequence_id) +{ + (void)parent; + g_sequence = (ptrdiff_t)sequence_id; + return 0; +} + + +static char* test_producer() +{ + static const char pb[] = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x32\x03one"; + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, NULL), "failed"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/kafka_producer.lua", NULL, NULL, dlog, + ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)1, false), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)2, false), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)3, false), "err: %s", + lsb_heka_get_error(hsb)); + while (g_sequence != 3) { + mu_assert(0 == lsb_heka_timer_event(hsb, 0, false), "err: %s", + lsb_heka_get_error(hsb)); + } + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(stats.pm_failures == 0, "pm_failures: %llu", stats.pm_failures); + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "%s", e); + lsb_free_heka_message(&m); + return NULL; +} + + +static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, + const char *cp_string) +{ + (void)parent; + (void)pb; + (void)pb_len; + (void)cp_numeric; + (void)cp_string; + return 0; +} + + +static char* test_consumer() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/kafka_consumer.lua", NULL, NULL, + dlog, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + mu_assert(0 == lsb_heka_pm_input(hsb, 0, NULL, false), "err: %s", + lsb_heka_get_error(hsb)); + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "%s", e); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_producer); + mu_run_test(test_consumer); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + free(e); + + return result != 0; +} From a0c791cf790f446deb8b1f1b727f3d4a864154be Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 7 Mar 2016 06:32:46 -0800 Subject: [PATCH 118/235] Pull in the latest hyperloglog module with merge support --- CMakeLists.txt | 2 +- cmake/externals.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 773acb0..c6a5b5a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 15) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 7206318..a75cc02 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -131,7 +131,7 @@ externalproject_add( externalproject_add( lua_hyperloglog GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 745756887dd4c277bee0337ab307eae9ea95890e + GIT_TAG 792b11dd811d89d6f0ad973b04a742c02c4ef7b5 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) From 6d4dff9287ea6eafaa5fb0101031635f17032ded Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 14 Mar 2016 07:22:03 -0700 Subject: [PATCH 119/235] Add a size option to read_message - quick way to get message stats without copying the entire string to Lua --- CMakeLists.txt | 2 +- docs/heka/analysis.md | 1 + src/heka/message.c | 2 ++ src/heka/test/lua/read_message.lua | 3 +++ src/util/heka_message.c | 2 +- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6a5b5a..8abf7a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 15) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index a609146..a37d683 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -53,6 +53,7 @@ Provides access to the Heka message data. Note that both fieldIndex and arrayInd * variableName (string) * framed (returns the Heka message protobuf string including the framing header) * raw (returns the Heka message protobuf string) + * size (returns the size of the raw Heka message protobuf string) * Uuid * Type * Logger diff --git a/src/heka/message.c b/src/heka/message.c index bf1bd41..80671fa 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -988,6 +988,8 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) luaL_addlstring(&b, m->raw.s, m->raw.len); luaL_pushresult(&b); } + } else if (strcmp(field, "size") == 0) { + lua_pushnumber(lua, (lua_Number)m->raw.len); } else { if (field_len >= 8 && memcmp(field, LSB_FIELDS "[", 7) == 0 diff --git a/src/heka/test/lua/read_message.lua b/src/heka/test/lua/read_message.lua index 1133bf4..4831ae8 100644 --- a/src/heka/test/lua/read_message.lua +++ b/src/heka/test/lua/read_message.lua @@ -15,6 +15,7 @@ local tests = { {"Severity", 9}, {"Pid", 0}, {"raw", 208}, + {"size", 208}, {"framed", 214}, {"Fields[string]", "string"}, {"Fields[notfound]", nil}, @@ -41,6 +42,8 @@ function process_message() local r = read_message(v[1]) if v[1] == "raw" or v[1] == "framed" then assert(v[2] == string.len(r), string.format("test: %d expected: %d received: %d", i, string.len(v[2]), string.len(r))) + elseif v[1] == "size" then + assert(v[2] == r, string.format("test: %d expected: %d received: %d", i, v[2], r)) else assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) end diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 4513783..e93a564 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -38,7 +38,7 @@ static size_t decode_header(char *buf, size_t len, size_t max_message_size) static const char* read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) { - if (wiretype != 2) { + if (wiretype != LSB_PB_WT_LENGTH) { return NULL; } From c1f88e220d4f1be2af374623ac76dd8ad363521c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 15 Mar 2016 11:35:59 -0700 Subject: [PATCH 120/235] Improve handling of nil error messages on plugin failures --- CMakeLists.txt | 2 +- include/luasandbox.h | 1 + src/heka/sandbox.c | 8 +++++--- src/heka/test/lua/input.lua | 2 ++ src/heka/test/test_sandbox.c | 13 +++++++------ src/luasandbox.c | 3 ++- src/luasandbox_serialize.c | 3 ++- src/test/test_luasandbox.c | 6 ++++-- 8 files changed, 24 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8abf7a0..14a3496 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 15) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_PATCH 3) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/include/luasandbox.h b/include/luasandbox.h index 335f912..092781e 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -34,6 +34,7 @@ #define LSB_OUTPUT_LIMIT "output_limit" #define LSB_LUA_PATH "path" #define LSB_LUA_CPATH "cpath" +#define LSB_NIL_ERROR "" typedef enum { LSB_UNKNOWN = 0, diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 1e79b50..1cd2620 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -437,10 +437,11 @@ static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, if (lua_pcall(lua, nargs, 2, 0) != 0) { char err[LSB_ERROR_SIZE]; const char *em = lua_tostring(lua, -1); - if (hsb->type == 'i' && strcmp(em, LSB_SHUTTING_DOWN) == 0) { + if (hsb->type == 'i' && em && strcmp(em, LSB_SHUTTING_DOWN) == 0) { return 0; } - size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", pm_func_name, em); + size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", pm_func_name, + em ? em : LSB_NIL_ERROR); if (len >= LSB_ERROR_SIZE) { err[LSB_ERROR_SIZE - 1] = 0; } @@ -781,8 +782,9 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) start = lsb_get_time(); if (lua_pcall(lua, 2, 0, 0) != 0) { char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - lua_tostring(lua, -1)); + em ? em : LSB_NIL_ERROR); if (len >= LSB_ERROR_SIZE) { err[LSB_ERROR_SIZE - 1] = 0; } diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index 7109678..0a974ee 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -31,6 +31,8 @@ function process_message(cp) error(string.rep("a", 255)) elseif cp == "string" then return 0, "string" + elseif cp == 7 then + error(nil) end return 0, "no cp" end diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 091d382..e3a7888 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -199,7 +199,7 @@ static char* test_api_assertion() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); - lsb_heka_sandbox *isb, *asb, *osb; + lsb_heka_sandbox * isb,*asb,*osb; isb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); asb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, aim); osb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); @@ -239,7 +239,7 @@ static char* test_create_input_sandbox() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); - const char* lfn = lsb_heka_get_lua_file(hsb); + const char *lfn = lsb_heka_get_lua_file(hsb); mu_assert(strcmp(lua_file, lfn) == 0, "expected %s received %s", lua_file, lfn); e = lsb_heka_destroy_sandbox(hsb); @@ -359,7 +359,7 @@ static char* test_timer_event() static char* test_stop_input() { - static const char* state_file = "stop.data"; + static const char *state_file = "stop.data"; remove(state_file); lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, dlog, iim); @@ -391,7 +391,7 @@ static char* test_pm_input() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); - for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i) { + for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); const char *err = lsb_heka_get_error(hsb); mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", @@ -427,10 +427,11 @@ static char* test_pm_error() "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, // >max error message + { .ncp = 7, .scp = NULL, .rv = 1, .err = "process_message() " }, }; lsb_heka_sandbox *hsb; - for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i) { + for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); mu_assert(hsb, "lsb_heka_create_input failed"); int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); @@ -509,7 +510,7 @@ static char* test_pm_output() mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); mu_assert_rv(-3, lsb_heka_pm_output(hsb, &m, (void *)100, - false)); + false)); mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); stats = lsb_heka_get_stats(hsb); mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); diff --git a/src/luasandbox.c b/src/luasandbox.c index 85c8ee7..7e3def8 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -501,8 +501,9 @@ lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) } lua_pushstring(lsb->lua, LUA_BASELIBNAME); if (lua_pcall(lsb->lua, 1, 0, 0)) { + const char *em = lua_tostring(lsb->lua, -1); snprintf(lsb->error_message, LSB_ERROR_SIZE, - "lsb_init %s", lua_tostring(lsb->lua, -1)); + "lsb_init %s", em ? em : LSB_NIL_ERROR); lsb_terminate(lsb, NULL); return LSB_ERR_LUA; } diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 95ee7c0..7bc65fb 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -250,7 +250,8 @@ serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *ob) lua_pushstring(lsb->lua, "%q"); lua_pushvalue(lsb->lua, index - 3); if (lua_pcall(lsb->lua, 2, 1, 0) == 0) { - ret = lsb_outputf(ob, "%s", lua_tostring(lsb->lua, -1)); + const char* em = lua_tostring(lsb->lua, -1); + ret = lsb_outputf(ob, "%s", em ? em : LSB_NIL_ERROR); if (ret) { lua_pop(lsb->lua, 1); // Remove the string table. return ret; diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 3993d84..123328d 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -82,8 +82,9 @@ int process(lsb_lua_sandbox *lsb, double ts) lua_pushnumber(lua, ts); if (lua_pcall(lua, 1, 2, 0) != 0) { char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - lua_tostring(lua, -1)); + em ? em : LSB_NIL_ERROR); if (len >= LSB_ERROR_SIZE || len < 0) { err[LSB_ERROR_SIZE - 1] = 0; } @@ -143,8 +144,9 @@ int report(lsb_lua_sandbox *lsb, double tc) lua_pushnumber(lua, tc); if (lua_pcall(lua, 1, 0, 0) != 0) { char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - lua_tostring(lua, -1)); + em ? em : LSB_NIL_ERROR); if (len >= LSB_ERROR_SIZE || len < 0) { err[LSB_ERROR_SIZE - 1] = 0; } From c2abbfabb3764acb4f29045499d8111c51522e6e Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Thu, 17 Mar 2016 10:31:40 +0100 Subject: [PATCH 121/235] Do not fail if msg doesn't have any field --- sandboxes/heka/output/debug.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua index f160df4..7367335 100644 --- a/sandboxes/heka/output/debug.lua +++ b/sandboxes/heka/output/debug.lua @@ -41,7 +41,7 @@ function process_message() write(":Pid: ", msg.Pid or "", "\n") write(":Hostname: ", msg.Hostname or "", "\n") write(":Fields:\n") - for i, v in ipairs(msg.Fields) do + for i, v in ipairs(msg.Fields or {}) do write(" | name:", v.name, " type:", v.value_type or 0, " representation:", v.representation or "", From 8110093357036977aeadab56cab44eaa14b3e708 Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Thu, 17 Mar 2016 10:33:46 +0100 Subject: [PATCH 122/235] Force flush stdout so that messages are printed live --- sandboxes/heka/output/debug.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua index 7367335..161abaa 100644 --- a/sandboxes/heka/output/debug.lua +++ b/sandboxes/heka/output/debug.lua @@ -12,6 +12,7 @@ message_matcher = "TRUE" --]] local write = require "io".write +local flush = require "io".flush local floor = require "math".floor local date = require "os".date local byte = require "string".byte @@ -57,6 +58,7 @@ function process_message() end end write("\n") + flush() return 0 end From 3f5717f3fe30462cd2ae202e0d7b765730a7abbc Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Mon, 21 Mar 2016 18:25:50 +0100 Subject: [PATCH 123/235] Fix example in doc: s/batch_checkpoint_update/update_checkpoint/ --- docs/heka/output.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/heka/output.md b/docs/heka/output.md index 8644847..52636e5 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -335,7 +335,7 @@ function timer_event(ns, shutdown) if buffer_len > 1 and (shutdown or last_load + ticker_interval <= ns / 1e9)then local err = bulk_load() if not err then - batch_checkpoint_update() + update_checkpoint() end end end From 88a1b75b8ea52481596d5f88da5588d91db844f1 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 22 Mar 2016 07:14:47 -0700 Subject: [PATCH 124/235] Pull in the package.seeall SHA --- CMakeLists.txt | 2 +- cmake/externals.cmake | 2 +- src/test/lua/sandbox_config.lua | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 14a3496..7e01c7d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 15) -set(CPACK_PACKAGE_VERSION_PATCH 3) +set(CPACK_PACKAGE_VERSION_PATCH 4) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index a75cc02..d463b95 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -65,7 +65,7 @@ else() externalproject_add( ${LUA_PROJECT} GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG 354ec76addd9f161a973d80ddf81c809552fb127 + GIT_TAG 308080049143009486221653a490c9e16f711f78 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/src/test/lua/sandbox_config.lua b/src/test/lua/sandbox_config.lua index 796fa94..da78b56 100644 --- a/src/test/lua/sandbox_config.lua +++ b/src/test/lua/sandbox_config.lua @@ -98,7 +98,10 @@ local tests = { } }, - {name = "package", t = package, values = {}}, + {name = "package", t = package, values = { + seeall=1, + } + }, {name = "string", t = string, values = { byte=1, From dd70bd57ba0286ad19e169754f5a0d31468ace1b Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Wed, 23 Mar 2016 16:05:24 +0100 Subject: [PATCH 125/235] Add luasec module to the sandbox (from LuaDist fork) --- cmake/externals.cmake | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index d463b95..dddf362 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -160,3 +160,12 @@ externalproject_add( INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "no install" ) include_directories("${EP_BASE}/Source/rapidjson/include") + +externalproject_add( + lua_sec + GIT_REPOSITORY https://github.com/LuaDist/luasec.git + GIT_TAG 329a8789f142bbbfef4fd7ed506285a25c3c0e0e + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + INSTALL_ARGS ${INST_ARGS} +) From 2132444179d653f8787c831b979ab41dd4a6f950 Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Wed, 23 Mar 2016 16:11:44 +0100 Subject: [PATCH 126/235] Add SSL support for heka_tcp.lua output sandbox --- sandboxes/heka/output/heka_tcp.lua | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sandboxes/heka/output/heka_tcp.lua b/sandboxes/heka/output/heka_tcp.lua index d6731a3..698451e 100644 --- a/sandboxes/heka/output/heka_tcp.lua +++ b/sandboxes/heka/output/heka_tcp.lua @@ -13,6 +13,16 @@ address = "127.0.0.1" port = 5565 timeout = 10 +ssl_params = { + mode = "client", + protocol = "tlsv1", + key = "/etc/hindsight/certs/clientkey.pem", + certificate = "/etc/hindsight/certs/client.pem", + cafile = "/etc/hindsight/certs/CA.pem", + verify = "peer", + options = {"all", "no_sslv3"} +} + --]] local socket = require "socket" @@ -20,6 +30,13 @@ local socket = require "socket" local address = read_config("address") or "127.0.0.1" local port = read_config("port") or 5565 local timeout = read_config("timeout") or 10 +local ssl_params = read_config("ssl_params") + +local ssl_ctx = nil +if ssl_params then + require "ssl" + ssl_ctx = assert(ssl.newcontext(ssl_params)) +end local function create_client() local c, err = socket.connect(address, port) @@ -27,6 +44,12 @@ local function create_client() c:setoption("tcp-nodelay", true) c:setoption("keepalive", true) c:settimeout(timeout) + if ssl_ctx then + c, err = ssl.wrap(c, ssl_ctx) + if c then + c:dohandshake() + end + end end return c, err end From e928d25d9595e529fb5e7cf1c34ac02e1ad6d3e6 Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Wed, 23 Mar 2016 16:21:16 +0100 Subject: [PATCH 127/235] Add SSL support for heka_tcp.lua input sandbox --- sandboxes/heka/input/heka_tcp.lua | 35 ++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/sandboxes/heka/input/heka_tcp.lua b/sandboxes/heka/input/heka_tcp.lua index cd464b2..9675c4a 100644 --- a/sandboxes/heka/input/heka_tcp.lua +++ b/sandboxes/heka/input/heka_tcp.lua @@ -12,6 +12,16 @@ instruction_limit = 0 address = "127.0.0.1" port = 5565 +ssl_params = { + mode = "server", + protocol = "tlsv1", + key = "/etc/hindsight/certs/serverkey.pem", + certificate = "/etc/hindsight/certs/server.pem", + cafile = "/etc/hindsight/certs/CA.pem", + verify = {"peer", "fail_if_no_peer_cert"}, + options = {"all", "no_sslv3"} +} + --]] require "coroutine" @@ -22,19 +32,19 @@ require "table" local address = read_config("address") or "127.0.0.1" local port = read_config("port") or 5565 +local ssl_params = read_config("ssl_params") +local ssl_ctx = nil +if ssl_params then + require "ssl" + ssl_ctx = assert(ssl.newcontext(ssl_params)) +end local server = assert(socket.bind(address, port)) server:settimeout(0) local threads = {} local sockets = {server} -local function handle_client(client) +local function handle_client(client, caddr, cport) local found, consumed, need = false, 0, 8192 * 4 - local caddr, cport = client:getpeername() - if not caddr then - caddr = "unknown" - cport = 0 - end - local hsr = heka_stream_reader.new( string.format("%s:%d -> %s:%d", caddr, cport, address, port)) client:settimeout(0) @@ -63,9 +73,18 @@ function process_message() if s == server then local client = s:accept() if client then + local caddr, cport = client:getpeername() + if not caddr then + caddr = "unknown" + cport = 0 + end + if ssl_ctx then + client = ssl.wrap(client, ssl_ctx) + client:dohandshake() + end sockets[#sockets + 1] = client threads[client] = coroutine.create( - function() handle_client(client) end) + function() handle_client(client, caddr, cport) end) end else if threads[s] then From 8f95ff8acbaffbb3259ad7bcd4e98cb9d079a9a1 Mon Sep 17 00:00:00 2001 From: Bruno Binet Date: Wed, 23 Mar 2016 17:05:55 +0100 Subject: [PATCH 128/235] Bump lua_sandbox version to 0.16.0 --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e01c7d..c066534 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 15) -set(CPACK_PACKAGE_VERSION_PATCH 4) +set(CPACK_PACKAGE_VERSION_MINOR 16) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) From 173bc6833fc046c94983ee62cdec419a12616f00 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 28 Mar 2016 07:02:49 -0700 Subject: [PATCH 129/235] Add conditional build support for the LuaSec module --- CMakeLists.txt | 5 +++++ cmake/externals.cmake | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c066534..bd07938 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,11 @@ else() message(STATUS "The Kafka module will not be built") endif() +find_package(OpenSSL) +if (NOT OPENSSL_FOUND) + message(STATUS "OpenSSL was not found, the LuaSec module will not be built") +endif() + find_package(Git REQUIRED) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") diff --git a/cmake/externals.cmake b/cmake/externals.cmake index dddf362..bf3bff7 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -161,11 +161,13 @@ externalproject_add( ) include_directories("${EP_BASE}/Source/rapidjson/include") -externalproject_add( - lua_sec - GIT_REPOSITORY https://github.com/LuaDist/luasec.git - GIT_TAG 329a8789f142bbbfef4fd7ed506285a25c3c0e0e - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - INSTALL_ARGS ${INST_ARGS} -) +if (OPENSSL_FOUND) + externalproject_add( + lua_sec + GIT_REPOSITORY https://github.com/LuaDist/luasec.git + GIT_TAG 329a8789f142bbbfef4fd7ed506285a25c3c0e0e + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + INSTALL_ARGS ${INST_ARGS} + ) +endif() From dd3963c7c19f31c882a9f70937488bb0b131168f Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 30 Mar 2016 21:09:21 -0700 Subject: [PATCH 130/235] Add the print function to the generic sandbox for debugging --- CMakeLists.txt | 3 +- docs/sandbox.md | 21 ++++++- src/heka/sandbox.c | 5 +- src/luasandbox.c | 71 ++++++++++++++++++++++-- src/luasandbox_impl.h | 1 + src/test/lua/print.lua | 19 +++++++ src/test/lua/sandbox_config.lua | 1 + src/test/test_luasandbox.c | 98 +++++++++++++++++++++++++++++++++ 8 files changed, 206 insertions(+), 13 deletions(-) create mode 100644 src/test/lua/print.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index bd07938..6a62e67 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 16) +set(CPACK_PACKAGE_VERSION_MINOR 17) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") @@ -62,6 +62,7 @@ set(INCLUDE_INSTALL_DIR include/) include_directories("${EP_BASE}/include") install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${LIB_INSTALL_DIR}${PROJECT_NAME}/modules COMPONENT core) install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION share/doc/${PROJECT_NAME} COMPONENT core) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandboxes/" DESTINATION share/${PROJECT_NAME}/sandboxes COMPONENT core) if(WIN32 AND NOT CYGWIN) set(INSTALL_CMAKE_DIR cmake) diff --git a/docs/sandbox.md b/docs/sandbox.md index b14c12a..fdb19bc 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -21,6 +21,7 @@ Sandbox API string = {"dump"} } ``` +* **log_level** - Integer specifying the syslog severity level, when set to debug (7) the print function will be wired to the specified logger * *user defined* any other variable (string, bool, number, table) is passed through as-is and available via [read_config](#read_config) ### Lua functions exposed to C by the core sandbox @@ -91,9 +92,9 @@ Provides access to the sandbox configuration variables. * value (string, number, bool, table) #### output -Appends data to the output buffer, which cannot exceed the output_limit -configuration parameter. See lsb_get_output() to connect the output to the -host application. +Receives any number of arguments and appends data to the output buffer, which +cannot exceed the output_limit configuration parameter. See lsb_get_output() to +connect the output to the host application. *Arguments* - arg (number, string, bool, nil, userdata implementing output support) - Lua @@ -102,6 +103,20 @@ host application. *Return* - none +#### print +Receives any number of arguments and sends a debug message to the host's logger +function as a tab delimited message. The function clears and then uses the +output buffer so if pending output has been queued and not flushed it will be +lost (the same output_limit restrictions apply). Non-printable characters +are replaced with spaces to preserve the host's log integrity and any embedded +NULs terminate each argument. + +*Arguments* +- arg (anything that can be converted to a string with the tostring function) + +*Return* +- none + **Note:** To extend the function set exposed to Lua see lsb_add_function() ## How to interact with the sandbox (creating an API) diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 1cd2620..542824e 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -256,7 +256,7 @@ static int update_checkpoint(lua_State *lua) static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) { static const char *io[] = { - NULL, "", "dofile", "load", "loadfile", "loadstring", "newproxy", "print", + NULL, "", "dofile", "load", "loadfile", "loadstring", "newproxy", NULL, "os", "getenv", "exit", "setlocale", NULL, "string", "dump" }; @@ -383,7 +383,6 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lua_State *lua = lsb_get_lua(hsb->lsb); set_restrictions(lua, hsb); -// todo link print to the logger or a no-op lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); lsb_add_function(hsb->lsb, inject_message_input, "inject_message"); // inject_payload is intentionally excluded from input plugins @@ -586,7 +585,6 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lua_State *lua = lsb_get_lua(hsb->lsb); set_restrictions(lua, hsb); -// todo link print to the logger or a no-op lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); lsb_add_function(hsb->lsb, read_message, "read_message"); lsb_add_function(hsb->lsb, inject_message_analysis, "inject_message"); @@ -669,7 +667,6 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lua_State *lua = lsb_get_lua(hsb->lsb); set_restrictions(lua, hsb); -// todo link print to the logger or a no-op lsb_add_function(hsb->lsb, read_message, "read_message"); lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); diff --git a/src/luasandbox.c b/src/luasandbox.c index 7e3def8..6ab593a 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -122,7 +122,7 @@ static void instruction_manager(lua_State *lua, lua_Debug *ar) static int output(lua_State *lua) { lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == lsb) { + if (!lsb) { return luaL_error(lua, "output() invalid lightuserdata"); } @@ -135,6 +135,62 @@ static int output(lua_State *lua) } +static int output_print(lua_State *lua) +{ + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "print() invalid lightuserdata"); + } + lsb->output.pos = 0; // clear the buffer + + int n = lua_gettop(lua); + if (!lsb->logger || n == 0) { + return 0; + } + + lua_getglobal(lua, "tostring"); + for (int i = 1; i <= n; ++i) { + lua_pushvalue(lua, -1); // tostring + lua_pushvalue(lua, i); // value + lua_call(lua, 1, 1); + const char *s = lua_tostring(lua, -1); + if (s == NULL) { + return luaL_error(lua, LUA_QL("tostring") " must return a string to " + LUA_QL("print")); + } + if (i > 1) { + lsb_outputc(&lsb->output, '\t'); + } + + while (*s) { + if (isprint(*s)) { + lsb_outputc(&lsb->output, *s); + } else { + lsb_outputc(&lsb->output, ' '); + } + ++s; + } + lua_pop(lua, 1); + } + + const char *component = NULL; + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + // this makes an assumptions by looking for a Heka sandbox specific cfg + // variable but will fall back to the lua filename in the generic case + lua_getfield(lua, -1, "Logger"); + component = lua_tostring(lua, -1); + if (!component) { + component = lsb->lua_file; + } + } + + lsb->logger(component, 7, "%s", lsb->output.buf); + lsb->output.pos = 0; + return 0; +} + + static int read_config(lua_State *lua) { luaL_checkstring(lua, 1); @@ -157,7 +213,7 @@ static int unprotected_panic(lua_State *lua) } -static int get_usage_config(lua_State *lua, int idx, const char *item) +static int get_int(lua_State *lua, int idx, const char *item) { lua_getfield(lua, idx, item); int i = (int)lua_tointeger(lua, -1); @@ -423,9 +479,10 @@ lsb_lua_sandbox* lsb_create(void *parent, copy_table(lsb->lua, lua_cfg, logger); lua_pop(lua_cfg, 2); lua_close(lua_cfg); - size_t ml = get_usage_config(lsb->lua, -1, "memory_limit"); - size_t il = get_usage_config(lsb->lua, -1, "instruction_limit"); - size_t ol = get_usage_config(lsb->lua, -1, "output_limit"); + size_t ml = get_int(lsb->lua, -1, "memory_limit"); + size_t il = get_int(lsb->lua, -1, "instruction_limit"); + size_t ol = get_int(lsb->lua, -1, "output_limit"); + int log_level = get_int(lsb->lua, -1, "log_level"); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); lua_pushcclosure(lsb->lua, &read_config, 0); lua_setglobal(lsb->lua, "read_config"); @@ -442,6 +499,9 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb->error_message[0] = 0; lsb->lua_file = malloc(strlen(lua_file) + 1); lsb->state_file = NULL; + lsb->logger = log_level == 7 ? logger : NULL; // give the sandbox access to + // the logger (print) when + // debugging if (!lsb->lua_file || lsb_init_output_buffer(&lsb->output, ol)) { if (logger) logger(__func__, 3, "memory allocation failed failed"); @@ -507,6 +567,7 @@ lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) lsb_terminate(lsb, NULL); return LSB_ERR_LUA; } + lsb_add_function(lsb, output_print, "print"); if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index af7e60f..91f04b9 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -22,6 +22,7 @@ struct lsb_lua_sandbox { void *parent; char *lua_file; char *state_file; + lsb_logger logger; lsb_state state; lsb_output_buffer output; size_t usage[LSB_UT_MAX][LSB_US_MAX]; diff --git a/src/test/lua/print.lua b/src/test/lua/print.lua new file mode 100644 index 0000000..dc74cfc --- /dev/null +++ b/src/test/lua/print.lua @@ -0,0 +1,19 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +function process(tc) + if tc == 0 then + print() + elseif tc == 1 then + print("foo\n", 10, true) + elseif tc == 2 then + print("f\r\0", 10, true) + elseif tc == 3 then + tostring = nil + local ok, err = pcall(print, "foo") + assert(not ok) + assert(err == "attempt to call a nil value", err) + end + return 0 +end diff --git a/src/test/lua/sandbox_config.lua b/src/test/lua/sandbox_config.lua index da78b56..7c71b79 100644 --- a/src/test/lua/sandbox_config.lua +++ b/src/test/lua/sandbox_config.lua @@ -38,6 +38,7 @@ local tests = { package=1, pairs=1, pcall=1, + print=1, rawequal=1, rawget=1, rawset=1, diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 123328d..287f72b 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -24,6 +24,7 @@ char *e = NULL; const char *written_data = NULL; size_t written_data_len = 0; +static char print_out[2048] = {0}; #ifdef _WIN32 @@ -71,6 +72,17 @@ void dlog(const char *component, int level, const char *fmt, ...) } +void print(const char *component, int level, const char *fmt, ...) +{ + va_list args; + va_start(args, fmt); + int n = snprintf(print_out, sizeof print_out, "%d %s ", level, + component ? component : "unnamed"); + n = vsnprintf(print_out + n, sizeof print_out - n, fmt, args); + va_end(args); +} + + int process(lsb_lua_sandbox *lsb, double ts) { static const char *func_name = "process"; @@ -1296,6 +1308,89 @@ static char* test_sax() } +static char* test_print() +{ + const char *tests[] = + { + "" + , "7 lua/print.lua foo \t10\ttrue" + , "7 lua/print.lua f \t10\ttrue" + , "" + , NULL + }; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;", print); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + for (int i = 0; tests[i]; ++i) { + print_out[0] = 0; + int result = process(sb, i); + mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); + mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_print_disabled() +{ + const char *tests[] = + { + "" + , "" + , NULL + }; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 6;", print); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + for (int i = 0; tests[i]; ++i) { + print_out[0] = 0; + int result = process(sb, i); + mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); + mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_print_logger() +{ + const char *tests[] = + { + "" + , "7 test.print foo \t10\ttrue" + , NULL + }; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;Logger = 'test.print';", print); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + for (int i = 0; tests[i]; ++i) { + print_out[0] = 0; + int result = process(sb, i); + mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); + mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -1592,6 +1687,9 @@ static char* all_tests() mu_run_test(test_sandbox_config); mu_run_test(test_cuckoo_filter); mu_run_test(test_sax); + mu_run_test(test_print); + mu_run_test(test_print_disabled); + mu_run_test(test_print_logger); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); From 3acef9c3d67877130069a017310ae27f91ca89e5 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 31 Mar 2016 13:02:15 -0700 Subject: [PATCH 131/235] Add an S3 output - no SDK integration and no compression option (yet) --- CMakeLists.txt | 2 +- sandboxes/heka/output/heka_s3_partition.lua | 281 ++++++++++++++++++++ 2 files changed, 282 insertions(+), 1 deletion(-) create mode 100644 sandboxes/heka/output/heka_s3_partition.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a62e67..1603a1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 17) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/sandboxes/heka/output/heka_s3_partition.lua b/sandboxes/heka/output/heka_s3_partition.lua new file mode 100644 index 0000000..276e888 --- /dev/null +++ b/sandboxes/heka/output/heka_s3_partition.lua @@ -0,0 +1,281 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "cjson" +require "io" +require "os" +require "string" +require "table" + +files = {} +local fh_cnt = 0 +local time_t = 0 +local buffer_cnt = 0 + +local hostname = read_config("Hostname") +local batch_path = read_config("batch_path") or error("batch_path must be specified") +local s3_path = read_config("s3_path") or error("s3_path must be specified") +local max_file_handles = read_config("max_file_handles") or 1000 +local max_file_size = read_config("max_file_size") or 1024 * 1024 * 500 +local max_file_age = read_config("max_file_age") or 60 * 60 +local flush_on_shutdown = read_config("flush_on_shutdown") + +--[[ +## Heka Protobuf Message S3 Output Partitioner + +Batches message data into Heka protobuf stream files based on the specified path +dimensions and copies them to S3 when they reach the maximum size or maximum +age. + +### Configuration + +#### Dimension Specification File +This file contains a JSON specification for how the data should be partitioned +into files and the location each file will reside at in S3. The JSON is shared +with other tools so it is not directly included in the configuration. + +TODO: This specifification should be expanded (when necessary) to include +- header values +- field/array index support +- patterns instead of exact matches + +##### Sample Specification File +```json + { + "version": 1, + "dimensions": [ + {"field_name": "submissionDate", + "allowed_values": {"min": "20140120", "max": "20140125"}}, + {"field_name": "sourceName", + "allowed_values": "*"}, + {"field_name": "sourceVersion", + "allowed_values": "*"}, + {"field_name": "reason", + "allowed_values": ["idle-daily","saved-session"]}, + {"field_name": "appName", + "allowed_values": ["Firefox", "Fennec", "Thunderbird", "FirefoxOS", "B2G"]}, + {"field_name": "appUpdateChannel", + "allowed_values": ["default", "nightly", "aurora", "beta", "release", "esr"]}, + {"field_name": "appVersion", + "allowed_values": "*"} + ] + } +``` + +#### Sample Configuration + +```lua +filename = "heka_s3_partition.lua" + +-- see the specification above +dimension_file = "foobar.json" + +-- directory location to store the intermediate output files +batch_path = "/var/tmp/foobar" + +-- Specifies how many data files to keep open at once. If there are more +-- "current" files than this, the least-recently used file will be closed +-- and then re-opened if more messages arrive before it is copied to S3. The +-- default is 1000. A value of 0 means no maximum. +max_file_handles = 1000 + +-- Specifies how much data (in bytes) can be written to a single file before +-- it is copied to s3 (default 500MB) +max_file_size = 1024 * 1024 * 500 + +-- Specifies how long (in seconds) to wait before it is copied to s3 +-- (default 1 hour). Idle files are only checked every tickec_interval seconds. +max_file_age = 60 * 60 + +-- Specifies that all local files will be copied S3 before exiting (default false). +flush_on_shutdown = true +preserve_data = not flush_on_shutdown -- should always be the inverse of flush_on_shutdown + +s3_path = "s3://foo" + +ticker_interval = 60 +``` +--]] + +local function sanitize_dimension(d) + if type(d) == "string" then + return string.gsub(d, "[^a-zA-Z0-9_.]", "_") + end +end + + +local function is_in_list(v, av) + for i, j in ipairs(av) do + if v == j then return true end + end + return false +end + + +local function is_in_range(v, min, max) + if min and v < min then return false end + if max and v > max then return false end + return true +end + + +local function validate_dimensions(fn) + local fh = assert(io.open(fn)) + local json = fh:read("*a") + fh:close() + local df = cjson.decode(json) + for i,d in ipairs(df.dimensions) do + local name = string.format("Fields[%s]", d.field_name) + d.field_name = name + local av = d.allowed_values + if type(av) == "string" then + if av == "*" then + d.matcher = function (v) return true end + else + av = sanitize_dimension(av) + d.matcher = function (v) return v == av end + end + elseif type(av) == "table" then + if av[1] ~= nil then + for m,n in ipairs(av) do + if type(n) ~= "string" then + error(string.format("field '$s' allowed_values array must contain only strings", name)) + end + av[m] = sanitize_dimension(n) + end + d.matcher = function (v) return is_in_list(v, av) end + else + if not av.min and not av.max then + error(string.format("field '%s' allowed_values range must have a 'min' or 'max'", name)) + end + if av.min and type(av.min) ~= "string" + or av.max and type(av.max) ~= "string" then + error(string.format("field '%s' allowed_values range min/max must be a string", name)) + end + d.matcher = function (v) return is_in_range(v, av.min, av.max) end + end + else + error(string.format("field '%s' allowed_values invalid type: %s", type(av))) + end + end + return df.dimensions +end + + +local function get_fqfn(path) + return string.format("%s/%s", batch_path, path) +end + + +local function close_fh(entry) + if not entry[2] then return end + entry[2]:close() + entry[2] = nil + fh_cnt = fh_cnt - 1 +end + + +local function copy_file(path, entry) + close_fh(entry) + local t = os.time() + local cmd + if t == time_t then + buffer_cnt = buffer_cnt + 1 + else + time_t = t + buffer_cnt = 0 + end + + local src = get_fqfn(path) + cmd = string.format("aws s3 cp %s %s/%s/%d_%d_%s", src, s3_path, + string.gsub(path, "+", "/"), time_t, buffer_cnt, hostname) + + local ret = os.execute(cmd) + if ret ~= 0 then + return string.format("ret: %d, cmd: %s", ret, cmd) + end + files[path] = nil + + local ok, err = os.remove(src); + if not ok then + return string.format("os.remove('%s') failed: %s", path, err) + end +end + + +local function get_entry(path) + local ct = os.time() + local t = files[path] + if not t then + t = {ct, nil} -- last active, file handle + files[path] = t + else + t[1] = ct + end + + if not t[2] then + if max_file_handles ~= 0 then + if fh_cnt >= max_file_handles then + local oldest = ct + 60 + local entry + for k,v in pairs(files) do -- if we max out file handles a lot we will want to make this more efficient + local et = v[1] + if v[2] and et < oldest then + entry = v + oldest = et + end + end + if entry then close_fh(entry) end + end + end + t[2] = assert(io.open(get_fqfn(path), "a")) + fh_cnt = fh_cnt + 1 + end + return t +end + +local dimensions = validate_dimensions(read_config("dimension_file")) + +function process_message() + local dims = {} + for i,d in ipairs(dimensions) do + local v = sanitize_dimension(read_message(d.field_name)) + if v then + if d.matcher(v) then + dims[i] = v + else + dims[i] = "OTHER" + end + else + dims[i] = "UNKNOWN" + end + end + local path = table.concat(dims, "+") + local entry = get_entry(path) + local fh = entry[2] + fh:write(read_message("framed")) + local size = fh:seek() + if size >= max_file_size then + local err = copy_file(path, entry) + if err then return -1, err end + end + return 0 +end + + +function timer_event(ns, shutdown) + local err + local ct = os.time() + for k,v in pairs(files) do + if (shutdown and flush_on_shutdown) or (ct - v[1] >= max_file_age) then + local e = copy_file(k, v) + if e then err = e end + elseif shutdown then + close_fh(v) + end + end + if shutdown and flush_on_shutdown and err then + error(string.format("flush on shutdown failed, last error: %s", err)) + end +end From fc4f4f68322970337fd87bb3258f5e9d8108c07b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 4 Apr 2016 09:53:27 -0700 Subject: [PATCH 132/235] Add an ElasticSearch output --- CMakeLists.txt | 2 +- modules/heka/elasticsearch.lua | 120 ++++++++++++ modules/heka/elasticsearch/moz_telemetry.lua | 133 +++++++++++++ modules/heka/elasticsearch/payload.lua | 41 ++++ modules/heka/msg_interpolate.lua | 99 ++++++++++ sandboxes/heka/output/debug.lua | 19 +- .../heka/output/elasticsearch_bulk_api.lua | 178 ++++++++++++++++++ 7 files changed, 575 insertions(+), 17 deletions(-) create mode 100644 modules/heka/elasticsearch.lua create mode 100644 modules/heka/elasticsearch/moz_telemetry.lua create mode 100644 modules/heka/elasticsearch/payload.lua create mode 100644 modules/heka/msg_interpolate.lua create mode 100644 sandboxes/heka/output/elasticsearch_bulk_api.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index 6a62e67..1603a1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 17) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/modules/heka/elasticsearch.lua b/modules/heka/elasticsearch.lua new file mode 100644 index 0000000..5fe13b3 --- /dev/null +++ b/modules/heka/elasticsearch.lua @@ -0,0 +1,120 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +## Elasticsearch Utility Functions + +### Module Configuration Table (common options) +```lua +-- Boolean flag, if true then any time interpolation (often used to generate the +-- ElasticSeach index) will use the timestamp from the processed message rather +-- than the system time. +es_index_from_timestamp == false -- optional, default shown + +-- String to use as the `_index` key's value in the generated JSON. +-- Supports field interpolation as described below. +index = "heka-%{%Y.%m.%d}" -- optional, default shown + +-- String to use as the `_type` key's value in the generated JSON. +-- Supports field interpolation as described below. +type_name = "message" -- optional, default shown + +-- String to use as the `_id` key's value in the generated JSON. +-- Supports field interpolation as described below. +id = nil -- optional, default shown + +``` + +### API + +#### bulkapi_index_json(index, type_name, id, ns) + +Returns a simple JSON 'index' structure satisfying the [ElasticSearch BulkAPI](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html) + +*Arguments* +* index (string or nil) - Used as the `_index` key's value in the generated JSON + or nil to omit the key. Supports field interpolation as described below. +* type_name (string or nil) - Used as the `_type` key's value in the generated + JSON or nil to omit the key. Supports field interpolation as described below. +* id (string or nil) - Used as the `_id` key's value in the generated JSON or + nil to omit the key. Supports field interpolation as described below. +* ns (number or nil) - Nanosecond timestamp to use for any strftime field + interpolation into the above fields. Current system time will be used if nil. + +*Return* +* JSON - String suitable for use as ElasticSearch BulkAPI index directive. + +*See* +[Field Interpolation](msg_interpolate.lua#L11) +--]] + +local cjson = require "cjson" +local mi = require "heka.msg_interpolate" +local string = require "string" +local assert = assert +local type = type +local read_message = read_message +local read_config = read_config + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module. + +local result_inner = { + _index = nil, + _type = nil, + _id = nil +} + +--[[ Public Interface --]] + +function bulkapi_index_json(index, type_name, id, ns) + local secs + if ns then + secs = ns / 1e9 + end + if index then + result_inner._index = string.lower(mi.interpolate(index, secs)) + else + result_inner._index = nil + end + if type_name then + result_inner._type = mi.interpolate(type_name, secs) + else + result_inner._type = nil + end + if id then + result_inner._id = mi.interpolate(id, secs) + else + result_inner._id = nil + end + return cjson.encode({index = result_inner}) +end + + +function load_encoder_cfg() + local cfg = read_config("encoder_cfg") + + if cfg.es_index_from_timestamp == nil then + cfg.es_index_from_timestamp = false + else + assert(type(cfg.es_index_from_timestamp) == "boolean", + "es_index_from_timestamp must be nil or boolean") + end + + if cfg.index == nil then + cfg.index = "heka-%{%Y.%m.%d}" + else + assert(type(cfg.index) == "string", "index must be nil or a string") + end + + if cfg.type_name == nil then + cfg.type_name = "message" + else + assert(type(cfg.type_name) == "string", "type_name must be nil or a string") + end + + return cfg +end + +return M diff --git a/modules/heka/elasticsearch/moz_telemetry.lua b/modules/heka/elasticsearch/moz_telemetry.lua new file mode 100644 index 0000000..bee2f4c --- /dev/null +++ b/modules/heka/elasticsearch/moz_telemetry.lua @@ -0,0 +1,133 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +## Elasticsearch Encoder for Unified Telemetry Messages + +### Module Configuration Table + +[Common Options](../elasticsearch.lua#8) +```lua +-- Array of Heka message field names that should be passed to Elasticsearch. +fields = {"Payload", "Fields[docType]"} -- required +``` +### Sample Output +```json +{"index":{"_index":"mylogger-2014.06.05","_type":"mytype-host.domain.com"}} +{"Payload":"data","docType":"main"} +``` +--]] + +-- Imports +local cjson = require "cjson" +local string = require "string" +local os = require "os" +local math = require "math" +local mi = require "heka.msg_interpolate" +local es = require "heka.elasticsearch" +local hj = require "heka_json" +local ipairs = ipairs +local pcall = pcall +local read_message = read_message +local cfg = es.load_encoder_cfg() + +if not cfg.fields or type(cfg.fields) ~= "table" or #cfg.fields == 0 then + error("fields must be specified") +end + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +local info_fields = { + "sessionId" + , "subsessionId" + , "previousSessionId" + , "previousSubsessionId" + , "subsessionCounter" + , "profileSubsessionCounter" + , "sessionStartDate" + , "subsessionStartDate" + , "subsessionLength" + , "sessionLength" +} + +local environment_fields = { + "telemetryEnabled" +} + +local static_fields = {} +local dynamic_fields = {} + +local function key(str) + return str:match("^Fields%[(.+)%]$") or error("invalid field name: " .. str) +end + +for i, field in ipairs(cfg.fields) do + local fp = mi.header_fields[field] + if fp then + static_fields[#static_fields+1] = {field, fp} + else + dynamic_fields[#dynamic_fields+1] = {field, key(field)} + end +end + +function encode() + local ns + if cfg.es_index_from_timestamp then ns = read_message("Timestamp") end + + local idx_json = es.bulkapi_index_json(cfg.index, cfg.type_name, cfg.id, ns) + + local tbl = {} + for i, field in ipairs(static_fields) do + tbl[field[1]] = field[2]() + end + + for i, field in ipairs(dynamic_fields) do + local full_name = field[1] + local short_name = field[2] + local z = 0 + local v = read_message(full_name, nil, z) + while v do + if z == 0 then + tbl[short_name] = v + elseif z == 1 then + tbl[short_name] = {tbl[short_name], v} + elseif z > 1 then + tbl[short_name][z+1] = v + end + z = z + 1 + v = read_message(full_name, nil, z) + end + end + + local ok, doc = pcall(hj.parse_message, "Fields[payload.info]") + if ok then + for i, k in ipairs(info_fields) do + tbl[k] = doc:value(doc:find(k)) + end + end + + ok, doc = pcall(hj.parse_message, "Fields[environment.settings]") + if ok then + for i, k in ipairs(environment_fields) do + tbl[k] = doc:value(doc:find(k)) + end + end + + ok, doc = pcall(hj.parse_message, "Payload") + if ok then + tbl.architecture = doc:value(doc:find("application", "architecture")) + end + + if tbl.creationTimestamp then + if ns then + tbl.Latency = math.floor((ns - tbl.creationTimestamp) / 1e9) + end + tbl.creationTimestamp = os.date("!%Y-%m-%dT%H:%M:%SZ", tbl.creationTimestamp / 1e9) + end + + return string.format("%s\n%s\n", idx_json, cjson.encode(tbl)) +end + +return M diff --git a/modules/heka/elasticsearch/payload.lua b/modules/heka/elasticsearch/payload.lua new file mode 100644 index 0000000..e5bda54 --- /dev/null +++ b/modules/heka/elasticsearch/payload.lua @@ -0,0 +1,41 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +## Elasticsearch Encoder for Heka Payload-only Messages +The message payload must be pre-formatted JSON in an ElasticSearch compatible +fromat. + +### Module Configuration Table + +[Common Options](../elasticsearch.lua#8) + +### Sample Output +```json +{"index":{"_index":"mylogger-2014.06.05","_type":"mytype-host.domain.com"}} +{"json":"data","extracted":"from","message":"payload"} +``` +--]] + +-- Imports +local string = require "string" +local es = require "heka.elasticsearch" +local read_message = read_message +local cfg = es.load_encoder_cfg() + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +function encode() + local ns + if cfg.es_index_from_timestamp then ns = read_message("Timestamp") end + local idx_json = es.bulkapi_index_json(cfg.index, cfg.type_name, cfg.id, ns) + local payload = read_message("Payload") or "{}" + if string.match(payload, "\n$", -1) then + return string.format("%s\n%s", idx_json, payload) + end + return string.format("%s\n%s\n", idx_json, payload) +end + +return M diff --git a/modules/heka/msg_interpolate.lua b/modules/heka/msg_interpolate.lua new file mode 100644 index 0000000..46e6d67 --- /dev/null +++ b/modules/heka/msg_interpolate.lua @@ -0,0 +1,99 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +## Simple Templating to Transform Message Fields into Strings + +### API + +#### interpolate(value, secs) + +Interpolates values from the currently processed message into the provided +string value. A `%{}` enclosed field name will be replaced by the field value +from the current message. All message header fields are supported ("Uuid", +"Timestamp", "Type", "Logger", "Severity", "Payload", "EnvVersion", "Pid", +"Hostname"). Any other values will be checked against the defined dynamic +message fields. If no field matches, then a +[C strftime](http://man7.org/linux/man-pages/man3/strftime.3.html) (*nix) +or +[C89 strftime](http://msdn.microsoft.com/en-us/library/fe06s4ak.aspx) (Windows) +time substitution will be attempted. The time used for time substitution will be +the seconds-from-epoch timestamp passed in as the `secs` argument, if provided. +If `secs` is nil, local system time is used. Note that the message timestamp is +*not* automatically used; if you want to use the message timestamp for time +substitutions, then you need to manually extract it and convert it from +nanoseconds to seconds (i.e. divide by 1e9). + +*Arguments* +* value (string) - String into which message values should be interpolated. +* secs (number or nil) - Timestamp (in seconds since epoch) to use for time + substitutions. If nil, system time will be used. + +*Return* +* string - Original string value with any interpolated message values. +--]] + +local date = require "os".date +local floor = require "math".floor +local string = require "string" +local read_message = read_message +local tostring = tostring +local type = type + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module. + +local function interpolate_match(header_fields, match, secs) + -- First see if it's a message header. + local fp = header_fields[match] + if fp then return fp() end + + -- Second check for a dynamic field. + local fname = string.format("Fields[%s]", match) + local fval = read_message(fname) + if type(fval) == "boolean" then + return tostring(fval) + elseif fval then + return fval + end + -- Finally try to use it as a strftime format string. + fval = date(match, secs) + if fval ~= match then -- Only return it if a substitution happened. + return fval + end +end + +--[[ Public Interface --]] + +header_fields = { + Uuid = function() return get_uuid(read_message("Uuid")) end, + Timestamp = function() return get_timestamp(read_message("Timestamp")) end, + Type = function() return read_message("Type") end, + Logger = function() return read_message("Logger") end, + Severity = function() return read_message("Severity") end, + Payload = function() return read_message("Payload") end, + EnvVersion = function() return read_message("EnvVersion") end, + Pid = function() return read_message("Pid") end, + Hostname = function() return read_message("Hostname") end +} + + +function get_uuid(uuid) + return string.format("%X%X%X%X-%X%X-%X%X-%X%X-%X%X%X%X%X", string.byte(uuid, 1, 16)) +end + + +function get_timestamp(ns) + local time_t = floor(ns / 1e9) + local frac = ns - time_t * 1e9 + local ds = date("!%Y-%m-%dT%H:%M:%S", time_t) + return string.format("%s.%09dZ", ds, frac) +end + + +function interpolate(value, secs) + return string.gsub(value, "%%{(.-)}", function(match) return interpolate_match(header_fields, match, secs) end) +end + +return M diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua index 161abaa..5cc8dd2 100644 --- a/sandboxes/heka/output/debug.lua +++ b/sandboxes/heka/output/debug.lua @@ -13,27 +13,14 @@ message_matcher = "TRUE" local write = require "io".write local flush = require "io".flush -local floor = require "math".floor -local date = require "os".date -local byte = require "string".byte local concat = require "table".concat - -local function get_uuid(uuid) - return string.format("%X%X%X%X-%X%X-%X%X-%X%X-%X%X%X%X%X", byte(uuid, 1, 16)) -end - -local function get_timestamp(ts) - local time_t = floor(ts / 1e9) - local ns = ts - time_t * 1e9 - local ds = date("%Y-%m-%d %H:%M:%S", time_t) - return string.format("%s.%09d +0000 UTC", ds, ns) -end +local mi = require "heka.msg_interpolate" function process_message() local raw = read_message("raw") local msg = decode_message(raw) - write(":Uuid: ", get_uuid(msg.Uuid), "\n") - write(":Timestamp: ", get_timestamp(msg.Timestamp), "\n") + write(":Uuid: ", mi.get_uuid(msg.Uuid), "\n") + write(":Timestamp: ", mi.get_timestamp(msg.Timestamp), "\n") write(":Type: ", msg.Type or "", "\n") write(":Logger: ", msg.Logger or "", "\n") write(":Severity: ", msg.Severity or 7, "\n") diff --git a/sandboxes/heka/output/elasticsearch_bulk_api.lua b/sandboxes/heka/output/elasticsearch_bulk_api.lua new file mode 100644 index 0000000..bfd4279 --- /dev/null +++ b/sandboxes/heka/output/elasticsearch_bulk_api.lua @@ -0,0 +1,178 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +## Elasticsearch Bulk API Output + +### Sample Configuration +```lua +filename = "elasticsearch_bulk_api.lua" +message_matcher = "Type == 'nginx'" +ticker_interval = 10 -- flush every 10 seconds or flush_count (50000) messages +memory_limit = 200e6 + +address = "127.0.0.1" +port = 9200 +timeout = 10 +flush_count = 50000 +flush_on_shutdown = false +preserve_data = not flush_on_shutdown --in most cases this should be the inverse of flush_on_shutdown + +-- See the elasticsearch module directory for the various encoders and configuration documentation. +encoder_module = "heka.elasticsearch.moz_telemetry" +encoder_cfg = { + es_index_from_timestamp = true, + index = "%{Logger}-%{%Y.%m.%d}", + type_name = "%{Type}-%{Hostname}", + fields = {"Fields[request]", "Fields[http_user_agent]"}, +} +``` +--]] + +require "table" +require "heka_json" +require "string" +local ltn12 = require "ltn12" +local time = require "os".time +local em = require(read_config("encoder_module")) +local socket = require "socket" +local http = require("socket.http") +local address = read_config("address") or "127.0.0.1" +local port = read_config("port") or 9200 +local timeout = read_config("timeout") or 10 + +local batch_file = string.format("%s/%s.batch", read_config("output_path"), read_config("Logger")) +local flush_on_shutdown = read_config("flush_on_shutdown") +local ticker_interval = read_config("ticker_interval") +local flush_count = read_config("flush_count") or 50000 +local last_flush = time() + +local client +local function create_client() + local client = http.open(address, port) + client.c:setoption("tcp-nodelay", true) + client.c:setoption("keepalive", true) + client.c:settimeout(timeout) + return client +end +local pcreate_client = socket.protect(create_client); + + +local req_headers = { + ["user-agent"] = http.USERAGENT, + ["content-length"] = 0, + ["host"] = address .. ":" .. port, + ["accept"] = "application/json", + ["connection"] = "keep-alive", +} + +local function send_request() -- hand coded since socket.http doesn't support keep-alive connections + local fh = assert(io.open(batch_file, "r")) + req_headers["content-length"] = fh:seek("end") + client:sendrequestline("POST", "/_bulk") + client:sendheaders(req_headers) + fh:seek("set") + client:sendbody(req_headers, ltn12.source.file(fh, "invalid file handle")) + local code = client:receivestatusline() + local headers + while code == 100 do -- ignore any 100-continue messages + headers = client:receiveheaders() + code = client:receivestatusline() + end + headers = client:receiveheaders() + local ok, err, ret = true, nil, 0 + if code ~= 204 and code ~= 304 and not (code >= 100 and code < 200) then + if code == 200 and string.match(headers["content-type"], "^application/json") then + local body = {} + local sink = ltn12.sink.table(body) + client:receivebody(headers, sink) + local response = table.concat(body) + local ok, doc = pcall(heka_json.parse, response) + if ok then + if doc:value(doc:find("errors")) then + ret = -1 + err = string.format("ElasticSearch server reported errors processing the submission") + end + else + ret = -3 + err = string.format("HTTP response didn't contain valid JSON. err: %s", doc) + end + else + client:receivebody(headers, ltn12.sink.null()) + end + + if not err and code > 304 then + ret = -1 + err = string.format("HTTP response error. Status: %d", code) + end + end + + if headers.connection == "close" then + client:close() + client = nil + end + + return true, err, ret +end +local psend_request = socket.protect(function(client) return send_request(client) end) + + +local batch = assert(io.open(batch_file, "a+")) +local function send_batch() + local err + if not client then + client, err = pcreate_client() + end + if err then return -3, err end -- retry indefinitely + + batch:flush() + local ok, err, ret = psend_request(client) + if not ok then -- network error + client = nil + return -3, err + end + last_flush = time() + return ret, err +end + +batch_count = 0 +retry = false + +function process_message() + if not retry then + batch_count = batch_count + 1 + batch:write(em.encode()) + end + + if batch_count == flush_count then + local ret, err = send_batch() + if ret == 0 then + retry = false + batch_count = 0 + batch:close() + batch = assert(io.open(batch_file, "w")) + elseif ret == -3 then + retry = true + client = nil + end + return ret, err + end + return 0 +end + + +function timer_event(ns, shutdown) + local timedout = (ns / 1e9 - last_flush) >= ticker_interval + if (timedout or (shutdown and flush_on_shutdown)) and batch_count > 0 then + local ret, err = send_batch() + if ret == 0 then + retry = false + batch_count = 0 + batch:close() + if not shutdown then + batch = assert(io.open(batch_file, "w")) + end + end + end +end From a9671412263416d43c80561c2c2e1a6230510bcd Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 4 Apr 2016 15:52:21 -0700 Subject: [PATCH 133/235] Incorporate review comments - update comments - allow more than strings in sanitize_dimensions --- sandboxes/heka/output/heka_s3_partition.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sandboxes/heka/output/heka_s3_partition.lua b/sandboxes/heka/output/heka_s3_partition.lua index 276e888..1db8371 100644 --- a/sandboxes/heka/output/heka_s3_partition.lua +++ b/sandboxes/heka/output/heka_s3_partition.lua @@ -85,7 +85,7 @@ max_file_handles = 1000 max_file_size = 1024 * 1024 * 500 -- Specifies how long (in seconds) to wait before it is copied to s3 --- (default 1 hour). Idle files are only checked every tickec_interval seconds. +-- (default 1 hour). Idle files are only checked every ticker_interval seconds. max_file_age = 60 * 60 -- Specifies that all local files will be copied S3 before exiting (default false). @@ -99,8 +99,8 @@ ticker_interval = 60 --]] local function sanitize_dimension(d) - if type(d) == "string" then - return string.gsub(d, "[^a-zA-Z0-9_.]", "_") + if d then + return string.gsub(tostring(d), "[^a-zA-Z0-9_.]", "_") end end @@ -251,7 +251,7 @@ function process_message() dims[i] = "UNKNOWN" end end - local path = table.concat(dims, "+") + local path = table.concat(dims, "+") -- the plus will be converted to a path separator '/' on copy local entry = get_entry(path) local fh = entry[2] fh:write(read_message("framed")) From 034698b9b735cb50816e7b2af5586bc6f9451fdb Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 4 Apr 2016 16:02:58 -0700 Subject: [PATCH 134/235] Allow false in sanitize --- sandboxes/heka/output/heka_s3_partition.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sandboxes/heka/output/heka_s3_partition.lua b/sandboxes/heka/output/heka_s3_partition.lua index 1db8371..d4b15cc 100644 --- a/sandboxes/heka/output/heka_s3_partition.lua +++ b/sandboxes/heka/output/heka_s3_partition.lua @@ -99,7 +99,7 @@ ticker_interval = 60 --]] local function sanitize_dimension(d) - if d then + if d ~= nil then return string.gsub(tostring(d), "[^a-zA-Z0-9_.]", "_") end end From 172c581386fe255cde89677e4d13032bbf46e615 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 5 Apr 2016 12:10:10 -0700 Subject: [PATCH 135/235] Bump revision (since the s3 merge) - fix doc typo --- CMakeLists.txt | 2 +- modules/heka/elasticsearch/payload.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1603a1e..d54ac38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 17) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/modules/heka/elasticsearch/payload.lua b/modules/heka/elasticsearch/payload.lua index e5bda54..70fc58c 100644 --- a/modules/heka/elasticsearch/payload.lua +++ b/modules/heka/elasticsearch/payload.lua @@ -5,7 +5,7 @@ --[[ ## Elasticsearch Encoder for Heka Payload-only Messages The message payload must be pre-formatted JSON in an ElasticSearch compatible -fromat. +format. ### Module Configuration Table From 7ee88ff8b5487ca339b5dd73e29b922f384aa19e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 11 Apr 2016 13:39:11 -0700 Subject: [PATCH 136/235] Remove the luasandbox.h include from util/heka_message.h https://github.com/mozilla-services/lua_sandbox/pull/125#issuecomment-208508939 --- include/luasandbox/util/heka_message.h | 1 - src/heka/message_matcher.c | 1 + src/util/heka_message.c | 3 +-- src/util/test/test_heka_message.c | 1 + src/util/test/test_output_buffer.c | 1 + 5 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index 6c77435..cb8657d 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -12,7 +12,6 @@ #include #include -#include "../../luasandbox.h" #include "input_buffer.h" #include "output_buffer.h" #include "string.h" diff --git a/src/heka/message_matcher.c b/src/heka/message_matcher.c index 1c0512a..6d492ca 100644 --- a/src/heka/message_matcher.c +++ b/src/heka/message_matcher.c @@ -12,6 +12,7 @@ #include #include +#include "luasandbox.h" #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" #include "luasandbox/lualib.h" diff --git a/src/util/heka_message.c b/src/util/heka_message.c index e93a564..ad8d5e4 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -8,11 +8,10 @@ #include "luasandbox/util/heka_message.h" +#include #include #include -#include "../luasandbox_impl.h" -#include "luasandbox_output.h" #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index efe7551..9b24f70 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -7,6 +7,7 @@ /** @brief heka_message unit tests @file */ #include +#include #include #include diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index 0489657..27f7225 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -6,6 +6,7 @@ /** @brief lsb_output_buffer unit tests @file */ +#include #include #include #include From b130e8ca1d952cab80fbe113af975294e8d4b1b8 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 11 Apr 2016 14:23:55 -0700 Subject: [PATCH 137/235] Fix build for cmake 3.5.1 https://github.com/mozilla-services/lua_sandbox/pull/125#issuecomment-208556968 --- src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f81ef80..0e980a5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,6 +13,7 @@ set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) +add_dependencies(luasandbox ${LUA_PROJECT}) add_dependencies(lua_bloom_filter luasandbox) add_dependencies(lua_circular_buffer luasandbox) add_dependencies(lua_hyperloglog luasandbox) From 19fc12bd1431ca445b03defc8187884f44cd2bbd Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Thu, 24 Mar 2016 10:40:35 +0100 Subject: [PATCH 138/235] Handle Apache mod_ssl Custom Log Formats As defined in http://httpd.apache.org/docs/current/mod/mod_ssl.html#logformats --- modules/common_log_format.lua | 1 + src/test/lua/lpeg_clf.lua | 2 ++ 2 files changed, 3 insertions(+) diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua index 3264820..5d40952 100644 --- a/modules/common_log_format.lua +++ b/modules/common_log_format.lua @@ -222,6 +222,7 @@ local apache_format_arguments = { , p = port_format , P = pid_format , t = time_format + , x = function(a) return string.gsub(string.lower(a), "-", "_") end -- already prefixed by SSL_ } local function apache_lookup_variable(var) diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index 4d6ef7f..49c7be4 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -264,6 +264,8 @@ local function apache_formats() ,{"%{%s}t" ,"time" ,"1392050801" ,1392050801000000000} ,{'"%r"' , "request" ,'"test \\"item\\""' ,'test \\"item\\"'} ,{'"%{Test-item}o"' ,"sent_http_test_item" ,'"test \\"item\\""' ,'test \\"item\\"'} + ,{"%{SSL_PROTOCOL}x" ,"ssl_protocol" ,"TLSv1.2" ,"TLSv1.2"} + ,{'"%{SSL_CIPHER}x"' ,"ssl_cipher" ,'"ECDHE-RSA-AES256-SHA"' ,"ECDHE-RSA-AES256-SHA"} } if os.date("%c"):find("^%d") then -- windows From faadb4130e5a27bbd142fbf085efe5aeecbd15b7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 27 Apr 2016 09:38:54 -0700 Subject: [PATCH 139/235] Add a logging context and message matcher module for the test GUI --- CMakeLists.txt | 4 +- docs/sandbox.md | 18 +-- include/luasandbox.h | 4 +- include/luasandbox/error.h | 14 +- include/luasandbox/heka/message_matcher.h | 5 +- include/luasandbox/heka/sandbox.h | 18 +-- include/luasandbox/util/heka_message.h | 8 +- sandboxes/heka/output/heka_tcp_matcher.lua | 115 ++++++++++++++++ src/heka/message_matcher.c | 153 ++++++++++++++++++++- src/heka/message_matcher_impl.h | 30 ++++ src/heka/rapidjson_impl.h | 4 +- src/heka/sandbox.c | 68 ++++++--- src/heka/stream_reader.c | 2 +- src/heka/stream_reader_impl.h | 7 +- src/heka/test/lua/output.lua | 19 ++- src/heka/test/test_sandbox.c | 67 +++++---- src/luasandbox.c | 70 ++++++---- src/test/test_luasandbox.c | 28 ++-- src/util/heka_message.c | 32 +++-- src/util/test/test_heka_message.c | 11 +- 20 files changed, 524 insertions(+), 153 deletions(-) create mode 100644 sandboxes/heka/output/heka_tcp_matcher.lua create mode 100644 src/heka/message_matcher_impl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index d54ac38..3a1b9af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 17) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_MINOR 18) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/docs/sandbox.md b/docs/sandbox.md index fdb19bc..b873814 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -11,16 +11,16 @@ Sandbox API * **cpath** - The path used by require to search for a C loader (same notes as above) * **disabled_modules** - Hash specifying which modules should be completely inaccessible. The existence of the key in the table will disable the module. - ```lua - disabled_modules = {io = 1} - ``` +```lua +disabled_modules = {io = 1} +``` * **remove_entries** - Hash specifying which functions within a module should be inaccessible. - ```lua - remove_entries = { - os = {"getenv", "execute"}, - string = {"dump"} - } - ``` +```lua +remove_entries = { + os = {"getenv", "execute"}, + string = {"dump"} +} +``` * **log_level** - Integer specifying the syslog severity level, when set to debug (7) the print function will be wired to the specified logger * *user defined* any other variable (string, bool, number, table) is passed through as-is and available via [read_config](#read_config) diff --git a/include/luasandbox.h b/include/luasandbox.h index 092781e..c631a0a 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -94,12 +94,12 @@ LSB_EXPORT extern lsb_err_id LSB_ERR_TERMINATED; * @param cfg Lua structure defining the full sandbox restrictions (may contain * optional host configuration options, everything is available to * the sandbox through the read_config API. - * @param logger Optional logger function pointer for error reporting + * @param logger Struct for error reporting/debug printing (NULL to disable) * @return lsb_lua_sandbox Sandbox pointer or NULL on failure. */ LSB_EXPORT lsb_lua_sandbox* lsb_create(void *parent, const char *lua_file, const char *cfg, - lsb_logger logger); + lsb_logger *logger); /** * Initializes the Lua sandbox and loads/runs the Lua script that was specified diff --git a/include/luasandbox/error.h b/include/luasandbox/error.h index 39b1a60..f1d0d61 100644 --- a/include/luasandbox/error.h +++ b/include/luasandbox/error.h @@ -15,9 +15,15 @@ typedef const char lsb_err_id[]; typedef const char *lsb_err_value; #define lsb_err_string(s) s ? s : "" -typedef void (*lsb_logger)(const char *component, - int level, - const char *fmt, - ...); +typedef void (*lsb_logger_cb)(void *context, + const char *component, + int level, + const char *fmt, + ...); + +typedef struct lsb_logger { + void *context; + lsb_logger_cb cb; +} lsb_logger; #endif diff --git a/include/luasandbox/heka/message_matcher.h b/include/luasandbox/heka/message_matcher.h index 4b6ed18..e5fa683 100644 --- a/include/luasandbox/heka/message_matcher.h +++ b/include/luasandbox/heka/message_matcher.h @@ -18,7 +18,8 @@ typedef struct lsb_message_matcher lsb_message_matcher; typedef struct lsb_message_match_builder lsb_message_match_builder; #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /** @@ -71,7 +72,7 @@ LSB_HEKA_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); * @return bool True if the message is a match */ LSB_HEKA_EXPORT bool lsb_eval_message_matcher(lsb_message_matcher *mm, - lsb_heka_message *m); + lsb_heka_message *m); #ifdef __cplusplus } diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 815d27c..d0b88a3 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -121,8 +121,8 @@ typedef int (*lsb_heka_update_checkpoint)(void *parent, void *sequence_id); * (NULL if no preservation is required) * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb * defaults) - * @param logger Logger call back (NULL to disable logging) - * @param im inject_message call back + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param im inject_message callback * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ LSB_HEKA_EXPORT @@ -130,7 +130,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, const char *lua_file, const char *state_file, const char *lsb_cfg, - lsb_logger logger, + lsb_logger *logger, lsb_heka_im_input im); /** @@ -163,8 +163,8 @@ int lsb_heka_pm_input(lsb_heka_sandbox *hsb, * (NULL if no preservation is required) * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb * defaults) - * @param logger Logger call back (NULL to disable logging) - * @param im inject_message call back + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param im inject_message callback * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ LSB_HEKA_EXPORT @@ -172,7 +172,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, const char *lua_file, const char *state_file, const char *lsb_cfg, - lsb_logger logger, + lsb_logger *logger, lsb_heka_im_analysis im); /** @@ -202,8 +202,8 @@ int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, * (NULL if no preservation is required) * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb * defaults) - * @param logger Logger call back (NULL to disable logging) - * @param upc checkpoint_updated call back when using batch or async output + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param upc checkpoint_updated callback when using batch or async output * * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ @@ -212,7 +212,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, const char *lua_file, const char *state_file, const char *lsb_cfg, - lsb_logger logger, + lsb_logger *logger, lsb_heka_update_checkpoint ucp); /** diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index cb8657d..0af3158 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -148,7 +148,7 @@ LSB_UTIL_EXPORT void lsb_clear_heka_message(lsb_heka_message *m); * @param ib Input buffer * @param decode True if the framed message should be protobuf decoded * @param discarded_bytes Used to track stream corruption - * @param logger Logger call back (can be set to NULL to disable logging) + * @param logger Logger structure (can be set to NULL to disable logging) * * @return bool True on success */ @@ -156,7 +156,7 @@ LSB_UTIL_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, lsb_input_buffer *ib, bool decode, size_t *discarded_bytes, - lsb_logger logger); + lsb_logger *logger); /** * Decodes an array of bytes into a Heka message. The message structure is @@ -165,7 +165,7 @@ LSB_UTIL_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, * @param m Heka message structure * @param buf Protobuf array * @param len Length of the protobuf array - * @param logger Logger call back (can be set to NULL to disable logging) + * @param logger Logger structure (can be set to NULL to disable logging) * * @return bool True on success * @@ -173,7 +173,7 @@ LSB_UTIL_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, LSB_UTIL_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, const char *buf, size_t len, - lsb_logger logger); + lsb_logger *logger); /** * Reads a dynamic field from the Heka message diff --git a/sandboxes/heka/output/heka_tcp_matcher.lua b/sandboxes/heka/output/heka_tcp_matcher.lua new file mode 100644 index 0000000..de46d00 --- /dev/null +++ b/sandboxes/heka/output/heka_tcp_matcher.lua @@ -0,0 +1,115 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Heka compatible TCP output with dynamic matching. Used for generating a debug +stream of messages for as long as the connection is maintained. + +-- .cfg +filename = "heka_tcp_matcher.lua" +instruction_limit = 0 +messsage_matcher = "TRUE" +ticker_interval = 1 + +address = "127.0.0.1" +port = 5566 + +ssl_params = { + mode = "server", + protocol = "tlsv1", + key = "/etc/hindsight/certs/serverkey.pem", + certificate = "/etc/hindsight/certs/server.pem", + cafile = "/etc/hindsight/certs/CA.pem", + verify = {"peer", "fail_if_no_peer_cert"}, + options = {"all", "no_sslv3"} +} + +--]] + +require "string" +local socket = require "socket" +require "table" +local mmb = require "heka_message_match_builder".new() + +local address = read_config("address") or "127.0.0.1" +local port = read_config("port") or 5566 +local ssl_params = read_config("ssl_params") +local ssl_ctx = nil +if ssl_params then + require "ssl" + ssl_ctx = assert(ssl.newcontext(ssl_params)) +end +local server = assert(socket.bind(address, port)) +server:settimeout(0) +local sockets = {server} +local subscribers = {} + +local function send_message(socket, msg, i) + local len, err, i = socket:send(msg, i) + if not len then + if err == "timeout" or err == "closed" then + return false + end + return send_message(msg, i) + end + return true +end + + +local num_subs = 0 +function process_message() + local msg + for i = num_subs, 1, -1 do + local sub = subscribers[i] + if sub[2]:eval() then + if not msg then msg = read_message("framed") end + if not send_message(sub[1], msg, 1) then + sub[1]:close() + table.remove(subscribers, i) + num_subs = num_subs - 1 + end + end + end + return 0 +end + + +function timer_event(ns, shutdown) + if shutdown then + for i,v in ipairs(subscribers) do + v[1]:close() + end + return + end + + local ready = socket.select(sockets, nil, 0) + if ready then + for _, s in ipairs(ready) do + local client = s:accept() + if client then + if ssl_ctx then + client = ssl.wrap(client, ssl_ctx) + client:dohandshake() + end + local exp, err = client:receive("*l") + local ok, mm = pcall(mmb.create_matcher, mmb, exp) + if ok then + num_subs = num_subs + 1 + subscribers[num_subs] = {client, mm} + else + print("bad matcher ", exp, mm) + local msg = encode_message( + { + Type = "bad matcher", + Payload = string.format("'%s' %s", tostring(exp), tostring(mm)) + }, + true) + send_message(client, msg, 1) + client:close() + end + end + end + end +end + diff --git a/src/heka/message_matcher.c b/src/heka/message_matcher.c index 6d492ca..544ca94 100644 --- a/src/heka/message_matcher.c +++ b/src/heka/message_matcher.c @@ -6,6 +6,7 @@ /** @brief Hindsight/Heka message matcher implementation @file */ +#include "message_matcher_impl.h" #include "luasandbox/heka/message_matcher.h" #include @@ -17,6 +18,7 @@ #include "luasandbox/lua.h" #include "luasandbox/lualib.h" #include "luasandbox/util/string_matcher.h" +#include "sandbox_impl.h" static const char *grammar = "local l = require 'lpeg'\n" @@ -454,7 +456,7 @@ lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath) } #endif - lsb_message_match_builder *mmb = malloc(sizeof *mmb); + lsb_message_match_builder *mmb = malloc(sizeof*mmb); mmb->parser = luaL_newstate(); if (!mmb->parser) { free(mmb); @@ -492,11 +494,13 @@ void lsb_destroy_message_match_builder(lsb_message_match_builder *mmb) lsb_message_matcher* lsb_create_message_matcher(const lsb_message_match_builder *mmb, - const char *exp) + const char *exp) { if (!mmb || !exp) { return NULL; } + lua_pop(mmb->parser, lua_gettop(mmb->parser)); + lua_getglobal(mmb->parser, "parse"); if (!lua_isfunction(mmb->parser, -1)) { return NULL; @@ -580,3 +584,148 @@ bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m) { return eval_tree(mm->nodes, m); } + +// add a lua wrapper so outputs can do some dynamic filtering +const char *mozsvc_heka_message_match_builder = + "mozsvc.heka_message_match_builder"; +const char *mozsvc_heka_message_match_builder_table = + "heka_message_match_builder"; +const char *mozsvc_heka_message_matcher = "mozsvc.heka_message_matcher"; + +static lsb_message_match_builder* check_mmb(lua_State *lua) +{ + lsb_message_match_builder **pmmb = + luaL_checkudata(lua, 1, mozsvc_heka_message_match_builder); + return *pmmb; +} + + +static lsb_message_matcher* check_mm(lua_State *lua) +{ + lsb_message_matcher **pmm = luaL_checkudata(lua, 1, + mozsvc_heka_message_matcher); + return *pmm; +} + + +static int mmb_new(lua_State *lua) +{ + const char *path = NULL; + const char *cpath = NULL; + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + lua_getfield(lua, -1, LSB_LUA_PATH); + path = lua_tostring(lua, -1); + lua_getfield(lua, -2, LSB_LUA_CPATH); + cpath = lua_tostring(lua, -1); + } else { + return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); + } + + lsb_message_match_builder **pmmb = lua_newuserdata(lua, sizeof**pmmb); + luaL_getmetatable(lua, mozsvc_heka_message_match_builder); + lua_setmetatable(lua, -2); + + + *pmmb = lsb_create_message_match_builder(path, cpath); + if (!*pmmb) { + return luaL_error(lua, "message_match_builder could not be created"); + } + return 1; +} + + +static int mmb_create_matcher(lua_State *lua) +{ + lsb_message_match_builder *mmb = check_mmb(lua); + const char *exp = luaL_checkstring(lua, 2); + + lsb_message_matcher **pmm = lua_newuserdata(lua, sizeof**pmm); + luaL_getmetatable(lua, mozsvc_heka_message_matcher); + lua_setmetatable(lua, -2); + + *pmm = lsb_create_message_matcher(mmb, exp); + if (!*pmm) { + return luaL_error(lua, "invalid message matcher expression"); + } + return 1; +} + + +static int mmb_gc(lua_State *lua) +{ + lsb_message_match_builder *mmb = check_mmb(lua); + lsb_destroy_message_match_builder(mmb); + return 0; +} + + +static int mm_eval(lua_State *lua) +{ + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "%s() invalid lightuserdata", __func__); + } + lsb_message_matcher *mm = check_mm(lua); + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + if (!hsb->msg || !hsb->msg->raw.s) { + return luaL_error(lua, "no active message"); + } + lua_pushboolean(lua, lsb_eval_message_matcher(mm, hsb->msg)); + return 1; +} + + +static int mm_gc(lua_State *lua) +{ + lsb_message_matcher *mm = check_mm(lua); + lsb_destroy_message_matcher(mm); + return 0; +} + + +static const struct luaL_reg mmlib_m[] = +{ + { "__gc", mm_gc }, + { NULL, NULL } +}; + +static const struct luaL_reg mmblib_f[] = +{ + { "new", mmb_new }, + { NULL, NULL } +}; + + +static const struct luaL_reg mmblib_m[] = +{ + { "create_matcher", mmb_create_matcher }, + { "__gc", mmb_gc }, + { NULL, NULL } +}; + + +int luaopen_heka_message_match_builder(lua_State *lua) +{ + luaL_newmetatable(lua, mozsvc_heka_message_matcher); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, mmlib_m); + // special case parse_message since it needs access to the sandbox + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) != LUA_TTABLE) { + return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); + } + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + lua_pushlightuserdata(lua, (void *)lsb); + lua_pushcclosure(lua, mm_eval, 1); + lua_setfield(lua, -3, "eval"); + lua_pop(lua, 2); // remove LSB_CONFIG_TABLE and metatable + + luaL_newmetatable(lua, mozsvc_heka_message_match_builder); + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, mmblib_m); + luaL_register(lua, mozsvc_heka_message_match_builder_table, mmblib_f); + return 1; +} diff --git a/src/heka/message_matcher_impl.h b/src/heka/message_matcher_impl.h new file mode 100644 index 0000000..c52e343 --- /dev/null +++ b/src/heka/message_matcher_impl.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight Heka message matcher Lua wrapper @file */ + +#ifndef luasandbox_heka_message_matcher_impl_h_ +#define luasandbox_heka_message_matcher_impl_h_ + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "luasandbox/lua.h" + +extern const char *mozsvc_heka_message_match_builder; +extern const char *mozsvc_heka_message_match_builder_table; + +extern const char *mozsvc_heka_message_matcher; + +int luaopen_heka_message_match_builder(lua_State *lua); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/heka/rapidjson_impl.h b/src/heka/rapidjson_impl.h index 3e9003f..6e4055c 100644 --- a/src/heka/rapidjson_impl.h +++ b/src/heka/rapidjson_impl.h @@ -6,8 +6,8 @@ /** Hindsight Heka JSON structures @file */ -#ifndef luasandbox_heka_json_h_ -#define luasandbox_heka_json_h_ +#ifndef luasandbox_heka_rapidjson_impl_h_ +#define luasandbox_heka_rapidjson_impl_h_ #ifdef __cplusplus extern "C" diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 542824e..f18d9f7 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -18,9 +18,10 @@ #include "luasandbox/util/running_stats.h" #include "luasandbox_output.h" #include "message_impl.h" +#include "message_matcher_impl.h" +#include "rapidjson_impl.h" #include "sandbox_impl.h" #include "stream_reader_impl.h" -#include "rapidjson_impl.h" #ifdef HAVE_KAFKA #include "kafka_impl.h" #endif @@ -347,23 +348,29 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, const char *lua_file, const char *state_file, const char *lsb_cfg, - lsb_logger logger, + lsb_logger *logger, lsb_heka_im_input im) { if (!lua_file) { - if (logger) logger(__func__, 3, "lua_file must be specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } return NULL; } if (!im) { - if (logger) logger(__func__, 3, "inject_message callback must be " - "specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "inject_message callback must " + "be specified"); + } return NULL; } lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__func__, 3, "memory allocation failed"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } return NULL; } @@ -408,7 +415,9 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lua_pop(lua, 1); // remove the preloaded table if (lsb_init(hsb->lsb, state_file)) { - if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); + if (logger && logger->cb) { + logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); + } lsb_destroy(hsb->lsb); free(hsb->hostname); free(hsb->name); @@ -548,24 +557,30 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, const char *lua_file, const char *state_file, const char *lsb_cfg, - lsb_logger logger, + lsb_logger *logger, lsb_heka_im_analysis im) { if (!lua_file) { - if (logger) logger(__func__, 3, "lua_file must be specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } return NULL; } if (!im) { - if (logger) logger(__func__, 3, "inject_message callback must be " - "specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "inject_message callback must " + "be specified"); + } return NULL; } lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__func__, 3, "memory allocation failed"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } return NULL; } @@ -596,7 +611,9 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lua_setglobal(lua, "output"); if (lsb_init(hsb->lsb, state_file)) { - if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); + if (logger && logger->cb) { + logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); + } lsb_destroy(hsb->lsb); free(hsb->hostname); free(hsb->name); @@ -630,24 +647,30 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, const char *lua_file, const char *state_file, const char *lsb_cfg, - lsb_logger logger, + lsb_logger *logger, lsb_heka_update_checkpoint ucp) { if (!lua_file) { - if (logger) logger(__func__, 3, "lua_file must be specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } return NULL; } if (!ucp) { - if (logger) logger(__func__, 3, LSB_HEKA_UPDATE_CHECKPOINT - " callback must be specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, LSB_HEKA_UPDATE_CHECKPOINT + " callback must be specified"); + } return NULL; } lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { - if (logger) logger(__func__, 3, "memory allocation failed"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } return NULL; } @@ -677,6 +700,11 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lua_pushlightuserdata(lua, (void *)hsb->lsb); lua_pushcclosure(lua, luaopen_heka_json, 1); lua_rawset(lua, -3); +// preload Heka message match builder with a pointer back to the sandbox + lua_pushstring(lua, mozsvc_heka_message_match_builder_table); + lua_pushlightuserdata(lua, (void *)hsb->lsb); + lua_pushcclosure(lua, luaopen_heka_message_match_builder, 1); + lua_rawset(lua, -3); #ifdef HAVE_KAFKA // preload the Heka Kafka producer lua_pushstring(lua, mozsvc_heka_kafka_producer_table); @@ -688,7 +716,9 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lua_pop(lua, 1); // remove the preloaded table if (lsb_init(hsb->lsb, state_file)) { - if (logger) logger(hsb->name, 3, lsb_get_error(hsb->lsb)); + if (logger && logger->cb) { + logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); + } lsb_destroy(hsb->lsb); free(hsb->hostname); free(hsb->name); diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c index ae5702e..4ed61ae 100644 --- a/src/heka/stream_reader.c +++ b/src/heka/stream_reader.c @@ -207,7 +207,7 @@ static int hsr_gc(lua_State *lua) free(hsr->name); lsb_free_heka_message(&hsr->msg); lsb_free_input_buffer(&hsr->buf); - return 1; + return 0; } diff --git a/src/heka/stream_reader_impl.h b/src/heka/stream_reader_impl.h index f368435..bc08660 100644 --- a/src/heka/stream_reader_impl.h +++ b/src/heka/stream_reader_impl.h @@ -6,8 +6,8 @@ /** Hindsight Heka stream reader structures @file */ -#ifndef luasandbox_heka_stream_reader_h_ -#define luasandbox_heka_stream_reader_h_ +#ifndef luasandbox_heka_stream_reader_impl_h_ +#define luasandbox_heka_stream_reader_impl_h_ #ifdef __cplusplus extern "C" @@ -21,9 +21,6 @@ extern "C" extern const char *mozsvc_heka_stream_reader; extern const char *mozsvc_heka_stream_reader_table; -extern const char *mozsvc_heka_json; -extern const char *mozsvc_heka_json_table; - typedef struct heka_stream_reader { char *name; diff --git a/src/heka/test/lua/output.lua b/src/heka/test/lua/output.lua index 6f3288f..61e0705 100644 --- a/src/heka/test/lua/output.lua +++ b/src/heka/test/lua/output.lua @@ -30,7 +30,17 @@ function process_message(sequence_id) sid = sequence_id return -5 else - local ok, err = pcall(update_checkpoint, sequence_id) + require "heka_message_match_builder" + local mmb = heka_message_match_builder.new() + local mm = mmb:create_matcher("Type == 'type'") + assert(mm:eval()) + mm = mmb:create_matcher("Type == 'foo'") + assert(not mm:eval()) + + local ok, err = pcall(mmb.create_matcher, mmb, "Widget == 'foo'") + assert(not ok) + + ok, err = pcall(update_checkpoint, sequence_id) assert(not ok) return -3 -- retry end @@ -38,5 +48,10 @@ function process_message(sequence_id) return 0 end -function timer_event(ns) +function timer_event(ns, shutdown) + require "heka_message_match_builder" + local mmb = heka_message_match_builder.new() + local mm = mmb:create_matcher("Type == 'foo'") + ok, err = pcall(mm.eval, mm) + assert(err == "no active message") end diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index e3a7888..782fc4a 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -21,9 +21,9 @@ static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x char *e = NULL; -void dlog(const char *component, int level, const char *fmt, ...) +void dlog(void *context, const char *component, int level, const char *fmt, ...) { - + (void)context; va_list args; va_start(args, fmt); fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, @@ -32,6 +32,7 @@ void dlog(const char *component, int level, const char *fmt, ...) fwrite("\n", 1, 1, stderr); va_end(args); } +static lsb_logger logger = { .context = NULL, .cb = dlog }; static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, @@ -165,7 +166,7 @@ static int aim(void *parent, const char *pb, size_t pb_len) } else { lsb_heka_message m; lsb_init_heka_message(&m, 2); - bool rv = lsb_decode_heka_message(&m, pb, pb_len, dlog); + bool rv = lsb_decode_heka_message(&m, pb, pb_len, &logger); lsb_free_heka_message(&m); if (!rv) return 1; } @@ -200,9 +201,9 @@ static char* test_api_assertion() mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); lsb_heka_sandbox * isb,*asb,*osb; - isb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); - asb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, aim); - osb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); + isb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); + asb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); + osb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, &logger, ucp); mu_assert(isb, "lsb_heka_create_input failed"); mu_assert_rv(1, lsb_heka_pm_input(NULL, 0, NULL, false)); @@ -237,7 +238,7 @@ static char* test_create_input_sandbox() { static const char *lua_file = "lua/input.lua"; lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, iim); + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); const char *lfn = lsb_heka_get_lua_file(hsb); mu_assert(strcmp(lua_file, lfn) == 0, "expected %s received %s", lua_file, @@ -255,8 +256,7 @@ static char* test_create_input_sandbox() mu_assert(!hsb, "lsb_heka_create_input succeeded"); e = lsb_heka_destroy_sandbox(hsb); // test NULL - hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, dlog, - iim); + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); lsb_heka_terminate_sandbox(hsb, "boom"); const char *err = lsb_heka_get_error(hsb); @@ -270,8 +270,7 @@ static char* test_create_input_sandbox() static char* test_create_analysis_sandbox() { lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, - aim); + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); mu_assert(hsb, "lsb_heka_create_analysis failed"); e = lsb_heka_destroy_sandbox(hsb); @@ -291,7 +290,7 @@ static char* test_create_analysis_sandbox() static char* test_create_output_sandbox() { lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_destroy_sandbox(hsb); @@ -310,8 +309,7 @@ static char* test_create_output_sandbox() static char* test_timer_event() { lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, - aim); + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); mu_assert(hsb, "lsb_heka_create_analysis failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(0 < stats.mem_cur, "received %llu", stats.mem_cur); @@ -362,7 +360,7 @@ static char* test_stop_input() static const char *state_file = "stop.data"; remove(state_file); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, dlog, iim); + hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); lsb_heka_stop_sandbox(hsb); e = lsb_heka_destroy_sandbox(hsb); @@ -389,7 +387,7 @@ static char* test_pm_input() }; lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); @@ -432,7 +430,7 @@ static char* test_pm_error() lsb_heka_sandbox *hsb; for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ - hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, dlog, iim); + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); const char *err = lsb_heka_get_error(hsb); @@ -452,8 +450,7 @@ static char* test_pm_analysis() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, dlog, - aim); + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); mu_assert(hsb, "lsb_heka_create_analysis failed"); mu_assert_rv(0, lsb_heka_pm_analysis(hsb, &m, false)); const char *err = lsb_heka_get_error(hsb); @@ -475,8 +472,7 @@ static char* test_pm_no_return() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, dlog, - aim); + hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, &logger, aim); mu_assert_rv(1, lsb_heka_pm_analysis(hsb, &m, false)); const char *err = lsb_heka_get_error(hsb); const char *eerr = "process_message() must return a numeric status code"; @@ -491,8 +487,11 @@ static char* test_pm_output() { lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, dlog, ucp); + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n", &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); mu_assert_rv(0, lsb_heka_pm_output(hsb, &m, NULL, false)); @@ -516,6 +515,8 @@ static char* test_pm_output() mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); + mu_assert_rv(0, lsb_heka_timer_event(hsb, 0, false)); + e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; @@ -527,8 +528,7 @@ static char* test_im_input() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/iim.lua", NULL, "path = [[" TEST_LUA_PATH "]]\n" - "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); - + "cpath = [[" TEST_LUA_CPATH "]]\n", &logger, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(9 == stats.im_cnt, "received %llu", stats.im_cnt); mu_assert(440 == stats.im_bytes, "received %llu", stats.im_bytes); @@ -542,8 +542,8 @@ static char* test_im_analysis() { lsb_heka_sandbox *hsb; hsb = lsb_heka_create_analysis(NULL, "lua/aim.lua", NULL, - "Hostname = 'foo.com';Logger = 'aim'", dlog, - aim); + "Hostname = 'foo.com';Logger = 'aim'", + &logger, aim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(3 == stats.im_cnt, "expected %llu", stats.im_cnt); mu_assert(232 == stats.im_bytes, "expected %llu", stats.im_bytes); @@ -559,7 +559,7 @@ static char* test_encode_message() hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n" - "Hostname = 'sh';Logger = 'sl'", dlog, ucp); + "Hostname = 'sh';Logger = 'sl'", &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(172 == stats.out_max, "received %llu", stats.out_max); @@ -573,8 +573,7 @@ static char* test_decode_message() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_output(NULL, "lua/decode_message.lua", NULL, NULL, - dlog, ucp); + hsb = lsb_heka_create_output(NULL, "lua/decode_message.lua", NULL, NULL, &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); @@ -586,11 +585,10 @@ static char* test_read_message() { lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); - mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, dlog), "failed"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, dlog, - aim); + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); mu_assert(hsb, "lsb_heka_create_analysist failed"); int rv = lsb_heka_pm_analysis(hsb, &m, false); mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, @@ -610,7 +608,7 @@ static char* test_heka_json() #endif "max_message_size = 8196\n" "path = [[" TEST_LUA_PATH "]]\n" - "cpath = [[" TEST_LUA_CPATH "]]\n", dlog, iim); + "cpath = [[" TEST_LUA_CPATH "]]\n", &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -625,8 +623,7 @@ static char* benchmark_decode_message() mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); lsb_heka_sandbox *hsb; hsb = lsb_heka_create_output(NULL, "lua/decode_message_benchmark.lua", NULL, - NULL, dlog, ucp); - + NULL, &logger, ucp); clock_t t = clock(); for (int x = 0; x < iter; ++x) { mu_assert(0 == lsb_heka_pm_output(hsb, &m, NULL, false), "%s", diff --git a/src/luasandbox.c b/src/luasandbox.c index 6ab593a..162754e 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -144,7 +144,7 @@ static int output_print(lua_State *lua) lsb->output.pos = 0; // clear the buffer int n = lua_gettop(lua); - if (!lsb->logger || n == 0) { + if (!lsb->logger.cb || n == 0) { return 0; } @@ -185,7 +185,7 @@ static int output_print(lua_State *lua) } } - lsb->logger(component, 7, "%s", lsb->output.buf); + lsb->logger.cb(lsb->logger.context, component, 7, "%s", lsb->output.buf); lsb->output.pos = 0; return 0; } @@ -275,11 +275,12 @@ static int check_int(lua_State *L, int idx, const char *name, int val) } -static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) +static lua_State* load_sandbox_config(const char *cfg, lsb_logger *logger) { lua_State *L = luaL_newstate(); if (!L) { - if (logger) logger(__func__, 3, "lua_State creation failed"); + if (logger->cb) logger->cb(logger->context, __func__, 3, + "lua_State creation failed"); return NULL; } @@ -305,8 +306,9 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) cleanup: if (ret) { - if (logger) { - logger(__func__, 3, "config error: %s", lua_tostring(L, -1)); + if (logger->cb) { + logger->cb(logger->context, __func__, 3, "config error: %s", + lua_tostring(L, -1)); } lua_close(L); return NULL; @@ -315,7 +317,7 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger logger) } -static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) +static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger *logger) { lua_newtable(sb); lua_pushnil(cfg); @@ -360,17 +362,17 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger logger) copy_table(sb, cfg, logger); break; default: - if (logger) { - logger(__func__, 4, "skipping config value type: %s", - lua_typename(cfg, vt)); + if (logger->cb) { + logger->cb(logger->context, __func__, 4, + "skipping config value type: %s", lua_typename(cfg, vt)); } break; } break; default: - if (logger) { - logger(__func__, 4, "skipping config key type: %s", - lua_typename(cfg, kt)); + if (logger->cb) { + logger->cb(logger->context, __func__, 4, "skipping config key type: %s", + lua_typename(cfg, kt)); } break; } @@ -434,41 +436,53 @@ static void set_random_seed() lsb_lua_sandbox* lsb_create(void *parent, const char *lua_file, const char *cfg, - lsb_logger logger) + lsb_logger *logger) { if (!lua_file) { - if (logger) logger(__func__, 3, "lua_file must be specified"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } return NULL; } if (!set_tz()) { - if (logger) logger(__func__, 3, "fail to set the TZ to UTC"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "fail to set the TZ to UTC"); + } return NULL; } set_random_seed(); - lsb_lua_sandbox *lsb = malloc(sizeof*lsb); + lsb_lua_sandbox *lsb = calloc(1, sizeof(*lsb)); if (!lsb) { - if (logger) logger(__func__, 3, "memory allocation failed"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } return NULL; } - memset(lsb->usage, 0, sizeof(lsb->usage)); #ifdef LUA_JIT lsb->lua = luaL_newstate(); #else lsb->lua = lua_newstate(memory_manager, lsb); #endif + if (logger) { + lsb->logger = *logger; + } if (!lsb->lua) { - if (logger) logger(__func__, 3, "lua state creation failed"); + if (lsb->logger.cb) { + lsb->logger.cb(lsb->logger.context, __func__, 3, "lua state creation " + "failed"); + } free(lsb); return NULL; } + // add the config to the lsb_config registry table - lua_State *lua_cfg = load_sandbox_config(cfg, logger); + lua_State *lua_cfg = load_sandbox_config(cfg, &lsb->logger); if (!lua_cfg) { lua_close(lsb->lua); free(lsb); @@ -476,7 +490,7 @@ lsb_lua_sandbox* lsb_create(void *parent, } lua_pushnil(lua_cfg); lua_pushvalue(lua_cfg, LUA_GLOBALSINDEX); - copy_table(lsb->lua, lua_cfg, logger); + copy_table(lsb->lua, lua_cfg, &lsb->logger); lua_pop(lua_cfg, 2); lua_close(lua_cfg); size_t ml = get_int(lsb->lua, -1, "memory_limit"); @@ -499,12 +513,16 @@ lsb_lua_sandbox* lsb_create(void *parent, lsb->error_message[0] = 0; lsb->lua_file = malloc(strlen(lua_file) + 1); lsb->state_file = NULL; - lsb->logger = log_level == 7 ? logger : NULL; // give the sandbox access to - // the logger (print) when - // debugging + if (log_level != 7) { + lsb->logger.cb = NULL; // only give the sandbox access to the logger (print) + // when debugging + } if (!lsb->lua_file || lsb_init_output_buffer(&lsb->output, ol)) { - if (logger) logger(__func__, 3, "memory allocation failed failed"); + if (lsb->logger.cb) { + lsb->logger.cb(lsb->logger.context, __func__, 3, "memory allocation " + "failed"); + } lsb_free_output_buffer(&lsb->output); free(lsb->lua_file); lua_close(lsb->lua); diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 287f72b..1e0c7aa 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -24,7 +24,7 @@ char *e = NULL; const char *written_data = NULL; size_t written_data_len = 0; -static char print_out[2048] = {0}; +static char print_out[2048] = { 0 }; #ifdef _WIN32 @@ -60,27 +60,31 @@ int file_exists(const char *fn) return 0; } -void dlog(const char *component, int level, const char *fmt, ...) +void dlog(void *context, const char *component, int level, const char *fmt, ...) { + (void)context; va_list args; - va_start(args, fmt); fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, component ? component : "unnamed"); + va_start(args, fmt); vfprintf(stderr, fmt, args); - fwrite("\n", 1, 1, stderr); va_end(args); + fwrite("\n", 1, 1, stderr); } +static lsb_logger logger = { .context = NULL, .cb = dlog }; -void print(const char *component, int level, const char *fmt, ...) +void print(void *context, const char *component, int level, const char *fmt, ...) { + (void)context; va_list args; - va_start(args, fmt); int n = snprintf(print_out, sizeof print_out, "%d %s ", level, - component ? component : "unnamed"); + component ? component : "unnamed"); + va_start(args, fmt); n = vsnprintf(print_out + n, sizeof print_out - n, fmt, args); va_end(args); } +static lsb_logger printer = { .context = NULL, .cb = print }; int process(lsb_lua_sandbox *lsb, double ts) @@ -223,7 +227,7 @@ static char* test_api_assertion() static char* test_create() { static char *cfg = "function foo() return 0 end\nt = {[true] = 1}\n"; - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", cfg, dlog); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", cfg, &logger); mu_assert(sb, "lsb_create() failed"); lsb_destroy(sb); sb = lsb_create(NULL, "lua/counter.lua", cfg, NULL); @@ -253,7 +257,7 @@ static char* test_create_error() sb = lsb_create(NULL, "lua/counter.lua", "cpath = 1", NULL); mu_assert(!sb, "lsb_create() invalid config"); - sb = lsb_create(NULL, "lua/counter.lua", "test = {", dlog); + sb = lsb_create(NULL, "lua/counter.lua", "test = {", &logger); mu_assert(!sb, "lsb_create() invalid config"); return NULL; @@ -1318,7 +1322,7 @@ static char* test_print() , "" , NULL }; - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;", print); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;", &printer); mu_assert(sb, "lsb_create() received: NULL"); lsb_err_value ret = lsb_init(sb, NULL); @@ -1345,7 +1349,7 @@ static char* test_print_disabled() , "" , NULL }; - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 6;", print); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 6;", &printer); mu_assert(sb, "lsb_create() received: NULL"); lsb_err_value ret = lsb_init(sb, NULL); @@ -1372,7 +1376,7 @@ static char* test_print_logger() , "7 test.print foo \t10\ttrue" , NULL }; - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;Logger = 'test.print';", print); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;Logger = 'test.print';", &printer); mu_assert(sb, "lsb_create() received: NULL"); lsb_err_value ret = lsb_init(sb, NULL); diff --git a/src/util/heka_message.c b/src/util/heka_message.c index ad8d5e4..70848f9 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -200,11 +200,11 @@ process_fields(lsb_heka_field *f, const char *p, const char *e) bool lsb_decode_heka_message(lsb_heka_message *m, const char *buf, size_t len, - lsb_logger logger) + lsb_logger *logger) { if (!m || !buf || len == 0) { - if (logger) { - logger(__func__, 4, LSB_ERR_UTIL_NULL); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, LSB_ERR_UTIL_NULL); } return false; } @@ -274,7 +274,9 @@ bool lsb_decode_heka_message(lsb_heka_message *m, lsb_heka_field *tmp = realloc(m->fields, m->fields_size * sizeof(lsb_heka_field)); if (!tmp) { - if (logger) logger(__func__, 0, "fields reallocation failed"); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 0, "fields reallocation failed"); + } return false; } m->fields = tmp; @@ -292,20 +294,24 @@ bool lsb_decode_heka_message(lsb_heka_message *m, } while (cp && cp < ep); if (!cp) { - if (logger) { - logger(__func__, 4, "tag:%d wiretype:%d position:%d", tag, - wiretype, lp - buf); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "tag:%d wiretype:%d position:%d", + tag, wiretype, lp - buf); } return false; } if (!m->uuid.s) { - if (logger) logger(__func__, 4, "missing " LSB_UUID); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "missing " LSB_UUID); + } return false; } if (!timestamp) { - if (logger) logger(__func__, 4, "missing " LSB_TIMESTAMP); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "missing " LSB_TIMESTAMP); + } return false; } @@ -319,11 +325,11 @@ bool lsb_find_heka_message(lsb_heka_message *m, lsb_input_buffer *ib, bool decode, size_t *discarded_bytes, - lsb_logger logger) + lsb_logger *logger) { if (!m || !ib || !discarded_bytes) { - if (logger) { - logger(__func__, 4, LSB_ERR_UTIL_NULL); + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, LSB_ERR_UTIL_NULL); } return false; } @@ -511,7 +517,7 @@ lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) static const size_t needed = 18; ob->pos = 0; // writing a uuid will always clear the buffer as it is the - // start of a new message + // start of a new message lsb_err_value ret = lsb_expand_output_buffer(ob, needed); if (ret) return ret; diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 9b24f70..2a8895b 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -29,10 +29,12 @@ struct log_message { char msg[1024]; }; -static struct log_message lm = { .component = NULL, .severity = 0, .msg = {0} }; +static struct log_message lm = { .component = NULL, .severity = 0, .msg = { 0 } }; -void logger(const char *component, int severity, const char *fmt, ...) +void log_cb(void *context, const char *component, int severity, const char *fmt, + ...) { + (void)context; lm.component = component; lm.severity = severity; va_list args; @@ -40,6 +42,7 @@ void logger(const char *component, int severity, const char *fmt, ...) vsnprintf(lm.msg, sizeof lm.msg, fmt, args); va_end(args); } +static lsb_logger logger = { .context = NULL, .cb = log_cb }; static char* test_stub() { @@ -89,7 +92,7 @@ static char* test_decode() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed"); for (unsigned i = 0; i < sizeof tests / sizeof(lsb_const_string); ++i){ - bool ok = lsb_decode_heka_message(&m, tests[i].s, tests[i].len, logger); + bool ok = lsb_decode_heka_message(&m, tests[i].s, tests[i].len, &logger); mu_assert(ok, "test: %d failed err: %s", i, lm.msg); } mu_assert(!lsb_decode_heka_message(NULL, NULL, 0, NULL), "succeeded"); @@ -140,7 +143,7 @@ static char* test_decode_failure() lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 10), "failed"); for (unsigned i = 0; i < sizeof(tests) / sizeof(struct decode_failure); ++i) { - bool ok = lsb_decode_heka_message(&m, tests[i].s, strlen(tests[i].s), logger); + bool ok = lsb_decode_heka_message(&m, tests[i].s, strlen(tests[i].s), &logger); mu_assert(!ok, "test: %u no error generated", i); mu_assert(!strcmp(lm.msg, tests[i].e), "test: %u expected: %s received: %s", i, tests[i].e, lm.msg); From 42b844e0d240fc26a3b3068d60f08f1eff526ec6 Mon Sep 17 00:00:00 2001 From: robbie Date: Sun, 17 Apr 2016 10:30:42 -0700 Subject: [PATCH 140/235] Adding Percona support Adding Percona support, now with working tests Adding Percona support, now with working tests Adding Percona support, now with working tests --- modules/mysql.lua | 94 ++++++++++++++++++++++++------ src/test/lua/lpeg_mysql.lua | 112 +++++++++++++++++++++++++++++++++--- 2 files changed, 182 insertions(+), 24 deletions(-) diff --git a/modules/mysql.lua b/modules/mysql.lua index 6b33f3c..115b674 100644 --- a/modules/mysql.lua +++ b/modules/mysql.lua @@ -6,35 +6,44 @@ local l = require "lpeg" l.locale(l) local tonumber = tonumber +local ip = require "ip_address" local M = {} setfenv(1, M) -- Remove external access to contain everything in the module +local pct_encoded = l.P"%" * l.xdigit * l.xdigit +local unreserved = l.alnum + l.S"-._~" +local sub_delims = l.S"!$&'()*+,;=" + local space = l.space^1 local sep = l.P"\n" local sql_end = l.P";" * (l.P"\n" + -1) local line = (l.P(1) - sep)^0 * sep -local float = l.digit^1 * "." * l.digit^1 +local float = l.digit^1 * "." * l.digit^1 / tonumber +local integer = l.P"-"^-1 * l.digit^1 / tonumber +local host = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) + + l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation")) + + l.Ct(l.Cg((unreserved + pct_encoded + sub_delims)^1, "value") * l.Cg(l.Cc"hostname", "representation")) local time = l.P"# Time: " * line local user_legal = (1 - l.S"[]") local host_legal = (l.alnum + l.S".-") -local user_name = user_legal^1 * "[" * l.Cg(user_legal^1, "Username") * "]" +local user_name = user_legal^0 * "[" * l.Cg(user_legal^0, "Username") * "]" local host_name = host_legal^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep -local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Query_time") -local lock_time = l.P"Lock_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Lock_time") -local rows_sent = l.P"Rows_sent: " * l.Cg(l.digit^1 / tonumber, "Rows_sent") -local rows_examined = l.P"Rows_examined: " * l.Cg(l.digit^1 / tonumber, "Rows_examined") +local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Query_time") +local lock_time = l.P"Lock_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Lock_time") +local rows_sent = l.P"Rows_sent: " * l.Cg(integer, "Rows_sent") +local rows_examined = l.P"Rows_examined: " * l.Cg(integer, "Rows_examined") local query = query_time * space * lock_time * space * rows_sent * space * rows_examined * sep local use_db = l.P"use " * line -local last_insert = l.P"last_insert_id=" * l.digit^1 * "," -local insert = l.P"insert_id=" * l.digit^1 * "," -local timestamp = l.P"timestamp=" * l.Cg((l.digit^1 / "%0000000000"), "Timestamp") +local last_insert = l.P"last_insert_id=" * l.Cg(integer, "Last_insert") * "," +local insert = l.P"insert_id=" * l.Cg(integer, "Insert_id") * "," +local timestamp = l.P"timestamp=" * l.Cg(integer / "%0000000000", "Timestamp") local set = l.P"SET " * last_insert^0 * insert^0 * timestamp * ";" * sep local admin = l.P"# administrator command: " * line @@ -43,22 +52,73 @@ local sql = l.Cg((l.P(1) - sql_end)^0 * sql_end, "Payload") -- Maria DB extensions local yes_no = l.C(l.P"Yes" + "No") -local thread_id = l.P"# Thread_id: " * l.Cg(l.digit^1 / tonumber, "Thread_id") - * l.P" Schema: " * l.Cg(l.alnum^0, "Schema") +local thread_id = l.P"# Thread_id: " * l.Cg(integer, "Thread_id") + * l.P" Schema: " * l.Cg(unreserved^0, "Schema") * l.P" QC_hit: " * l.Cg(yes_no, "QC_hit") * sep local full_scan = l.P"# Full_scan: " * l.Cg(yes_no, "Full_scan") - * l.P" Full_join: " * l.Cg(yes_no, "Full_join") - * l.P" Tmp_table: " * l.Cg(yes_no, "Tmp_table") - * l.P" Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep + * " Full_join: " * l.Cg(yes_no, "Full_join") + * " Tmp_table: " * l.Cg(yes_no, "Tmp_table") + * " Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep * l.P"# Filesort: " * l.Cg(yes_no, "Filesort") * " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk") - * " Merge_passes: " * l.Cg(l.digit^1 / tonumber, "Merge_passes") * sep + * " Merge_passes: " * l.Cg(integer, "Merge_passes") * sep + +-- Percona extensions +local percona_user = "# User@Host: " * user_name * space * "@" * space * host_name * sep^0 +local conn_id = " Id:" * space * l.Cg(integer, "Connection_id") * sep + +local info = l.P(("# Thread_id: " * l.Cg(integer, "Thread_id") + * " Schema: " * l.Cg(unreserved^0, "Schema") + * " Last_errno: " * l.Cg(integer, "Last_errno") + * " Killed: " * l.Cg(integer, "Killed")) + + ("# Schema: " * l.Cg(unreserved^0, "Schema") + * " Last_errno: " * l.Cg(integer, "Last_errno") + * " Killed: " * l.Cg(integer, "Killed"))) * sep + +local percona_query = "# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") + * l.Cg(l.Cc"s", "representation")), "Query_time") + * " Lock_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") + * l.Cg(l.Cc"s", "representation")), "Lock_time") + * " Rows_sent: " * l.Cg(integer, "Rows_sent") + * " Rows_examined: " * l.Cg(integer, "Rows_examined") + * " Rows_affected: " * l.Cg(integer, "Rows_affected") + * (" Rows_read: " * l.Cg(integer, "Rows_read"))^0 * sep + +local memory_footprint = "# Bytes_sent: " * l.Cg(l.Ct(l.Cg(integer, "value") + * l.Cg(l.Cc"B", "representation")), "Bytes_sent") + * (" Tmp_tables: " * l.Cg(integer, "Tmp_tables") + * " Tmp_disk_tables: " * l.Cg(integer, "Tmp_disk_tables") + * " Tmp_table_sizes: " * l.Cg(l.Ct(l.Cg(integer, "value") + * l.Cg(l.Cc"B", "representation")), "Tmp_table_sizes"))^0 * sep + +local stored_routine = "# Stored routine: " * l.Cg((l.alnum + l.S"_.-")^1, "Stored_routine") * sep + +local query_plan_info = "# QC_Hit: " * l.Cg(yes_no, "QC_hit") + * " Full_scan: " * l.Cg(yes_no, "Full_scan") + * " Full_join: " * l.Cg(yes_no, "Full_join") + * " Tmp_table: " * l.Cg(yes_no, "Tmp_table") + * " Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep + * "# Filesort: " * l.Cg(yes_no, "Filesort") + * " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk") + * " Merge_passes: " * l.Cg(integer, "Merge_passes") * sep + +local innodb_usage = "# InnoDB_IO_r_ops: " * l.Cg(integer, "InnoDB_IO_r_ops") + * " InnoDB_IO_r_bytes: " * l.Cg(l.Ct(l.Cg(integer, "value") + * l.Cg(l.Cc"B", "representation")), "InnoDB_IO_r_bytes") + * " InnoDB_IO_r_wait: " * l.Cg(l.Ct(l.Cg(float, "value") + * l.Cg(l.Cc"s", "representation")), "InnoDB_IO_r_wait") * sep + * "# InnoDB_rec_lock_wait: " * l.Cg(l.Ct(l.Cg(float, "value") + * l.Cg(l.Cc"s", "representation")), "InnoDB_rec_lock_wait") + * " InnoDB_queue_wait: " * l.Cg(l.Ct(l.Cg(float, "value") + * l.Cg(l.Cc"s", "representation")), "InnoDB_queue_wait") * sep + * "# InnoDB_pages_distinct: " * l.Cg(integer, "InnoDB_pages_distinct") * sep slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) mariadb_slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) +percona_slow_query_grammar = l.Ct(time^0 * percona_user * l.Cg(l.Ct(conn_id^0 * info^0 * percona_query * memory_footprint^0 * stored_routine^0 * query_plan_info^0 * innodb_usage^0), "Fields") * use_db^0 * set * admin^0 * sql) -short_slow_query_grammar = l.Ct(l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) -mariadb_short_slow_query_grammar= l.Ct(l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) +short_slow_query_grammar = l.Ct(l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) +mariadb_short_slow_query_grammar = l.Ct(l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) return M diff --git a/src/test/lua/lpeg_mysql.lua b/src/test/lua/lpeg_mysql.lua index 2e12022..e1f2b86 100644 --- a/src/test/lua/lpeg_mysql.lua +++ b/src/test/lua/lpeg_mysql.lua @@ -84,11 +84,42 @@ WHERE id = 10; ]] } +local percona_slow_query_log = { +[[ +# User@Host: syncrw[syncrw] @ db-01.example.com [127.0.0.1] Id: 42 +# Schema: imdb Last_errno: 0 Killed: 0 +# Query_time: 7.725616 Lock_time: 0.000328 Rows_sent: 4 Rows_examined: 1543720 Rows_affected: 0 +# Bytes_sent: 272 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +SET timestamp=1399500744; +/* [queryName=FIND_ITEMS] */ SELECT * +FROM widget +WHERE id = 10; +]], +[[ +# Time: 130601 8:01:06 +# User@Host: syncrw[syncrw] @ db-01.example.com [127.0.0.1] Id: 42 +# Schema: imdb Last_errno: 0 Killed: 0 +# Query_time: 7.725616 Lock_time: 0.000328 Rows_sent: 4 Rows_examined: 1543720 Rows_affected: 0 Rows_read: 30 +# Bytes_sent: 272 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 +# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No +# Filesort: No Filesort_on_disk: No Merge_passes: 0 +# InnoDB_IO_r_ops: 6415 InnoDB_IO_r_bytes: 105103360 InnoDB_IO_r_wait: 0.001279 +# InnoDB_rec_lock_wait: 0.000000 InnoDB_queue_wait: 0.000000 +# InnoDB_pages_distinct: 6430 +SET timestamp=1399500744; +/* [queryName=FIND_ITEMS] */ SELECT * +FROM widget +WHERE id = 10; +]] +} + local fields = { {"Query_time", 2.964652, "s"}, {"Rows_examined", 9773}, {"Lock_time", 0.000050, "s"}, - {"Rows_sent", 251}, + {"Rows_sent", 251} } local mariadb_fields = { @@ -118,6 +149,59 @@ local mariadb_verbose_fields = { {"Merge_passes", 3} } +local percona_fields = { + {"Schema", "imdb"}, + {"Last_errno", 0}, + {"Killed", 0}, + {"Query_time", 7.725616, "s"}, + {"Lock_time", 0.000328, "s"}, + {"Rows_sent", 4}, + {"Rows_examined", 1543720}, + {"Rows_affected", 0}, + {"Bytes_sent", 272, "B"}, + {"Tmp_tables", 0}, + {"Tmp_disk_tables", 0}, + {"Tmp_table_sizes", 0, "B"}, + {"QC_hit", "No"}, + {"Full_scan", "Yes"}, + {"Full_join", "No"}, + {"Tmp_table", "No"}, + {"Tmp_table_on_disk", "No"}, + {"Filesort", "No"}, + {"Filesort_on_disk", "No"}, + {"Merge_passes", 0} +} + +local percona_verbose_fields = { + {"Schema", "imdb"}, + {"Last_errno", 0}, + {"Killed", 0}, + {"Query_time", 7.725616, "s"}, + {"Lock_time", 0.000328, "s"}, + {"Rows_sent", 4}, + {"Rows_examined", 1543720}, + {"Rows_affected", 0}, + {"Rows_read", 30}, + {"Bytes_sent", 272, "B"}, + {"Tmp_tables", 0}, + {"Tmp_disk_tables", 0}, + {"Tmp_table_sizes", 0, "B"}, + {"QC_hit", "No"}, + {"Full_scan", "Yes"}, + {"Full_join", "No"}, + {"Tmp_table", "No"}, + {"Tmp_table_on_disk", "No"}, + {"Filesort", "No"}, + {"Filesort_on_disk", "No"}, + {"Merge_passes", 0}, + {"InnoDB_IO_r_ops", 6415}, + {"InnoDB_IO_r_bytes", 105103360, "B"}, + {"InnoDB_IO_r_wait", 0.001279, "s"}, + {"InnoDB_rec_lock_wait", 0, "s"}, + {"InnoDB_queue_wait", 0, "s"}, + {"InnoDB_pages_distinct", 6430} +} + local function validate(fields, t) if t.Timestamp ~= "1399500744000000000" then return error("Timestamp:" .. t.Timestamp) end if t.Payload ~= "/* [queryName=FIND_ITEMS] */ SELECT *\nFROM widget\nWHERE id = 10;\n" then return error("Payload:" .. t.Payload) end @@ -131,6 +215,7 @@ local function validate(fields, t) end end else + print("Output:" .. v[1]) if t.Fields[v[1]] ~= v[2] then return error(string.format("field:%s:", v[1]) .. t.Fields[v[1]], 2) end end end @@ -139,45 +224,56 @@ end local function header() local t = mysql.slow_query_grammar:match(slow_query_log[1]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end validate(fields, t) end local function standard() local t = mysql.slow_query_grammar:match(slow_query_log[2]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end validate(fields, t) end local function short() local t = mysql.short_slow_query_grammar:match(slow_query_log[3]) if not t then return error("no match") end - if t.Hostname then return error("Hostname:" .. t.Hostname) end validate(fields, t) end local function mariadb_standard() local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[1]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end validate(mariadb_fields, t) end local function mariadb_short() local t = mysql.mariadb_short_slow_query_grammar:match(mariadb_slow_query_log[2]) if not t then return error("no match") end - if t.Hostname then return error("Hostname:" .. t.Hostname) end validate(mariadb_fields, t) end local function mariadb_verbose() local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[3]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end validate(mariadb_verbose_fields, t) end +local function percona_standard() + local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[1]) + if not t then return error("no match") end + if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + validate(percona_fields, t) +end + +local function percona_verbose() + local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[2]) + if not t then return error("no match") end + if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + validate(percona_verbose_fields, t) +end function process(tc) header() @@ -186,6 +282,8 @@ function process(tc) mariadb_standard() mariadb_short() mariadb_verbose() + percona_standard() + percona_verbose() return 0 end From 1b1590fbb7fbffb1ee9578903aac97ae9062c798 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 8 May 2016 21:46:12 -0700 Subject: [PATCH 141/235] Issue #131 create a standalone message matcher parser and move it to util --- CMakeLists.txt | 2 +- docs/heka/message_matcher.md | 14 +- docs/heka/output.md | 16 + include/luasandbox/heka/message_matcher.h | 81 - .../luasandbox/util/heka_message_matcher.h | 55 + include/luasandbox/util/util.h | 10 + sandboxes/heka/output/heka_tcp_matcher.lua | 3 +- src/heka/CMakeLists.txt | 1 - src/heka/message_matcher.c | 731 ------ src/heka/message_matcher_impl.h | 30 - src/heka/sandbox.c | 71 +- src/heka/test/CMakeLists.txt | 4 - src/heka/test/lua/output.lua | 17 +- src/heka/test/test_sandbox.c | 3 +- src/luasandbox.c | 17 +- src/util/CMakeLists.txt | 2 + src/util/heka_message_matcher.c | 202 ++ src/util/heka_message_matcher_impl.h | 61 + src/util/heka_message_matcher_parser.c | 2213 +++++++++++++++++ src/util/heka_message_matcher_parser.leg | 418 ++++ src/util/protobuf.c | 4 +- src/util/test/CMakeLists.txt | 4 + .../test/test_heka_message_matcher.c} | 86 +- src/util/test/test_protobuf.c | 4 + src/util/test/test_util.c | 15 + src/util/util.c | 20 + 26 files changed, 3153 insertions(+), 931 deletions(-) delete mode 100644 include/luasandbox/heka/message_matcher.h create mode 100644 include/luasandbox/util/heka_message_matcher.h delete mode 100644 src/heka/message_matcher.c delete mode 100644 src/heka/message_matcher_impl.h create mode 100644 src/util/heka_message_matcher.c create mode 100644 src/util/heka_message_matcher_impl.h create mode 100644 src/util/heka_message_matcher_parser.c create mode 100644 src/util/heka_message_matcher_parser.leg rename src/{heka/test/test_message_matcher.c => util/test/test_heka_message_matcher.c} (74%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a1b9af..40059f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 18) +set(CPACK_PACKAGE_VERSION_MINOR 19) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") diff --git a/docs/heka/message_matcher.md b/docs/heka/message_matcher.md index 14fe727..c73211e 100644 --- a/docs/heka/message_matcher.md +++ b/docs/heka/message_matcher.md @@ -63,14 +63,20 @@ All message variables must be on the left hand side of the relational comparison * Fields[_field_name_] - shorthand for Field[_field_name_][0][0] * Fields[_field_name_][_field_index_] - shorthand for Field[_field_name_][_field_index_][0] -* Fields[_field_name_][_field_index_][_array_index_] +* Fields[_field_name_][_field_index_][_array_index_] the indices are restricted to 0-255 * If a field type is mis-match for the relational comparison, false will be returned e.g., Fields[foo] == 6 where "foo" is a string #### Quoted String -* single or double quoted strings are allowed +* Single or double quoted strings are allowed +* The maximum string length is 255 bytes #### Lua Pattern Matching Expression -* see [Lua Patterns](http://www.lua.org/manual/5.1/manual.html#5.4.1) -* capture groups are ignored +* Patterns are quoted string values +* See [Lua Patterns](http://www.lua.org/manual/5.1/manual.html#5.4.1) +* Capture groups are ignored + +#### Additional Restrictions + +* Message matchers are restricted to 128 tests diff --git a/docs/heka/output.md b/docs/heka/output.md index 52636e5..c67041f 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -78,6 +78,22 @@ be added by infrastructure using the corresponding configuration setting. *Return* * heka_pb (string) - Heka protobuf binary string, framed as specified or an error is thrown +#### create_message_matcher + +Returns a Heka protocol buffer message matcher; used to dynamic filter messages sent to the output plugin. + +*Arguments* +* message_matcher [message matcher](message_matcher.md) + +*Return* +* message_matcher (userdata) - or an error is thrown + The message matcher object has one method `eval` that returns true if the current message matches, false + if it does not. + +##### Example + +See: [heka_tcp_matcher.lua](../../sandboxes/heka/output/heka_tcp_matcher.lua) + #### update_checkpoint ##### Batch Mode diff --git a/include/luasandbox/heka/message_matcher.h b/include/luasandbox/heka/message_matcher.h deleted file mode 100644 index e5fa683..0000000 --- a/include/luasandbox/heka/message_matcher.h +++ /dev/null @@ -1,81 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Hindsight/Heka message matcher @file */ - -#ifndef luasandbox_heka_message_matcher_h_ -#define luasandbox_heka_message_matcher_h_ - -#include - -#include "../util/heka_message.h" -#include "sandbox.h" - -typedef struct lsb_message_matcher lsb_message_matcher; -typedef struct lsb_message_match_builder lsb_message_match_builder; - -#ifdef __cplusplus -extern "C" -{ -#endif - -/** - * Initializes a Lua state containing the message matcher PEG. - * - * @param lua_path Path to the Lua based modules (date_time) - * @param lua_cpath Path to the C based modules (lpeg) - * @return NULL if the message match builder cannot be created - * - * @todo Remove the dependency on the sandbox and create a standalone - * match parser in the luasandboxutil lib - * - */ -LSB_HEKA_EXPORT lsb_message_match_builder* -lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath); - -/** - * Frees all memory associated with the message matcher builder - * - * @param mmb Message matcher builder - */ -LSB_HEKA_EXPORT void -lsb_destroy_message_match_builder(lsb_message_match_builder *mmb); - -/** - * Parsers a message matcher expression and returns the matcher - * - * @param mmb Message matcher builder - * @param exp Expression to parse into a matcher - * - * @return lsb_message_matcher* - */ -LSB_HEKA_EXPORT lsb_message_matcher* -lsb_create_message_matcher(const lsb_message_match_builder *mmb, - const char *exp); - -/** - * Frees all memory associated with a message matcher instance - * - * @param mm Message matcher - */ -LSB_HEKA_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); - -/** - * Evaluates the message matcher against the provided message - * - * @param mm Message matcher - * @param m Heka message - * - * @return bool True if the message is a match - */ -LSB_HEKA_EXPORT bool lsb_eval_message_matcher(lsb_message_matcher *mm, - lsb_heka_message *m); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/include/luasandbox/util/heka_message_matcher.h b/include/luasandbox/util/heka_message_matcher.h new file mode 100644 index 0000000..b0c74bb --- /dev/null +++ b/include/luasandbox/util/heka_message_matcher.h @@ -0,0 +1,55 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher @file */ + +#ifndef luasandbox_util_heka_message_matcher_h_ +#define luasandbox_util_heka_message_matcher_h_ + +#include + +#include "heka_message.h" + +typedef struct lsb_message_matcher lsb_message_matcher; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Parses a message matcher expression and returns the matcher + * + * @param exp Expression to parse into a matcher + * + * @return lsb_message_matcher* + */ +LSB_UTIL_EXPORT lsb_message_matcher* +lsb_create_message_matcher(const char *exp); + +/** + * Frees all memory associated with a message matcher instance + * + * @param mm Message matcher + */ +LSB_UTIL_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); + +/** + * Evaluates the message matcher against the provided message + * + * @param mm Message matcher + * @param m Heka message + * + * @return bool True if the message is a match + */ +LSB_UTIL_EXPORT bool +lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index f55fcfe..36fff96 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -9,6 +9,7 @@ #ifndef luasandbox_util_util_h_ #define luasandbox_util_util_h_ +#include #include #include "../error.h" @@ -62,6 +63,15 @@ LSB_UTIL_EXPORT char* lsb_read_file(const char *fn); */ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); +/** + * Sets the timezone environment variable for the time conversion functions + * + * @param tz Timezone string (if NULL uses UTC) + * + * @return bool True if the environment variable is successfully set + */ +LSB_UTIL_EXPORT bool lsb_set_tz(const char *tz); + #ifdef HAVE_ZLIB /** diff --git a/sandboxes/heka/output/heka_tcp_matcher.lua b/sandboxes/heka/output/heka_tcp_matcher.lua index de46d00..5023b47 100644 --- a/sandboxes/heka/output/heka_tcp_matcher.lua +++ b/sandboxes/heka/output/heka_tcp_matcher.lua @@ -30,7 +30,6 @@ ssl_params = { require "string" local socket = require "socket" require "table" -local mmb = require "heka_message_match_builder".new() local address = read_config("address") or "127.0.0.1" local port = read_config("port") or 5566 @@ -93,7 +92,7 @@ function timer_event(ns, shutdown) client:dohandshake() end local exp, err = client:receive("*l") - local ok, mm = pcall(mmb.create_matcher, mmb, exp) + local ok, mm = pcall(create_message_matcher, exp) if ok then num_subs = num_subs + 1 subscribers[num_subs] = {client, mm} diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index 1e99f1b..2fed407 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -4,7 +4,6 @@ set(HEKA_SRC message.c -message_matcher.c rapidjson.cpp sandbox.c stream_reader.c diff --git a/src/heka/message_matcher.c b/src/heka/message_matcher.c deleted file mode 100644 index 544ca94..0000000 --- a/src/heka/message_matcher.c +++ /dev/null @@ -1,731 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Hindsight/Heka message matcher implementation @file */ - -#include "message_matcher_impl.h" -#include "luasandbox/heka/message_matcher.h" - -#include -#include -#include - -#include "luasandbox.h" -#include "luasandbox/lauxlib.h" -#include "luasandbox/lua.h" -#include "luasandbox/lualib.h" -#include "luasandbox/util/string_matcher.h" -#include "sandbox_impl.h" - -static const char *grammar = - "local l = require 'lpeg'\n" - "local dt = require 'date_time'\n" - "l.locale(l)\n" - "\n" - "local sp = l.space^0\n" - "local eqne = l.Cg(l.P'==' + '!=', 'op')\n" - "local string_match = l.Cg(l.P'=~' + '!~', 'op')\n" - "local relational = l.Cg(eqne + '>=' + '>' + '<=' + '<', 'op')\n" - "local op_and = l.C(l.P'&&')\n" - "local op_or = l.C(l.P'||')\n" - "\n" - "local string_vars = l.Cg(l.P'" LSB_TYPE - "' + '" LSB_LOGGER - "' + '" LSB_HOSTNAME - "' + '" LSB_ENV_VERSION - "' + '" LSB_PAYLOAD - "' + '" LSB_UUID - "', 'variable')\n" - "local numeric_vars = l.Cg(l.P'" LSB_SEVERITY - "' + '" LSB_PID - "', 'variable')\n" - "local boolean = l.Cg(l.P'TRUE' / function() return true end + l.P'FALSE' / function() return false end, 'value')\n" - "local null = l.P'NIL'\n" - "local index = l.P'[' * (l.digit^1 / tonumber) * l.P']' + l.Cc'0' / tonumber\n" - "local fields = l.P'Fields[' * l.Cg((l.P(1) - ']')^0, 'variable') * ']' * l.Cg(index, 'fi') * l.Cg(index, 'ai')\n" - "\n" - "local string_value = l.Cg(l.P'\"' * l.Cs(((l.P(1) - l.S'\\\\\"') + l.P'\\\\\"' / '\"')^0) * '\"'\n" - " + l.P'\\'' * l.Cs(((l.P(1) - l.S'\\\\\\'') + l.P'\\\\\\'' / '\\'')^0) * '\\'', 'value')\n" - "local string_test = l.Ct(string_vars * sp * (relational * sp * string_value + string_match * sp * string_value))\n" - "\n" - "local sign = l.S'+-'^-1\n" - "local number = l.P'0' + l.R'19' * l.digit^0\n" - "local decimal = (l.P'.'^-1 * l.digit^1)^-1\n" - "local exponent = (l.S'eE' * sign * l.digit^1)^-1\n" - "local numeric_value = l.Cg((sign * number * decimal * exponent) / tonumber, 'value')\n" - "local numeric_test = l.Ct(numeric_vars * sp * relational * sp * numeric_value)\n" - "\n" - "local ts_value = l.Cg(dt.rfc3339 / dt.time_to_ns, 'value')\n" - "local ts_quoted = l.P'\"' * ts_value * '\"' + l.P\"'\" * ts_value * \"'\"\n" - "local ts_test = l.Ct(l.Cg(l.P'Timestamp', 'variable') * sp * relational * sp * (numeric_value + ts_quoted))\n" - "\n" - "local field_test = l.Ct(fields * sp * (relational * sp * (string_value + numeric_value)\n" - " + string_match * sp * string_value\n" - " + eqne * sp * (boolean + null)))\n" - "\n" - "local Exp, Term, Test = l.V'Exp', l.V'Term', l.V'Test'\n" - "g = l.P{\n" - "Exp,\n" - "Exp = l.Ct(Term * (op_or * sp * Term)^0);\n" - "Term = l.Ct(Test * (op_and * sp * Test)^0) * sp;\n" - "Test = (string_test + ts_test + numeric_test + field_test + l.Ct(l.Cg(boolean, 'op'))) * sp + l.P'(' * Exp * ')' * sp;\n" - "}\n" - "\n" - "local function postfix(t, output, stack, stack_ptr)\n" - " local sp = stack_ptr\n" - " for j,v in ipairs(t) do\n" - " if type(v) == 'string' then\n" - " stack_ptr = stack_ptr + 1\n" - " stack[stack_ptr] = v\n" - " elseif type(v) == 'table' then\n" - " if v.op then\n" - " output[#output+1] = v\n" - " else\n" - " postfix(v, output, stack, stack_ptr)\n" - " end\n" - " end\n" - " end\n" - " for i=stack_ptr, sp+1, -1 do\n" - " output[#output+1] = stack[i]\n" - " end\n" - " stack_ptr = sp\n" - " return output\n" - "end\n" - "\n" - "local grammar = l.Ct(sp * g * -1)\n" - "\n" - "function parse(s)\n" - " local output = {}\n" - " local stack = {}\n" - " local stack_ptr = 0\n" - " local t = grammar:match(s)\n" - " local len = 0\n" - " if t then\n" - " t = postfix(t, output, stack, stack_ptr)\n" - " len = #t\n" - " end\n" - " return t, len -- node array and length\n" - "end\n"; - - -typedef enum { - OP_EQ, - OP_NE, - OP_GTE, - OP_GT, - OP_LTE, - OP_LT, - OP_RE, - OP_NRE, - OP_TRUE, - OP_FALSE, - OP_OR, - OP_AND -} match_operation; - - -typedef enum { - TYPE_NIL, - TYPE_STRING, - TYPE_NUMERIC, - TYPE_BOOLEAN, -} match_type; - - -typedef struct match_node { - struct match_node *left; - struct match_node *right; - - char *variable; - size_t variable_len; - - union { - char *s; - double d; - } value; - size_t value_len; - match_type value_type; - - match_operation op; - int id; - int fi; - int ai; -} match_node; - - -struct lsb_message_match_builder { - lua_State *parser; -}; - - -struct lsb_message_matcher { - int nodes_size; - match_node nodes[]; -}; - - -static bool string_test(match_node *mn, lsb_const_string *val) -{ - switch (mn->op) { - case OP_EQ: - if (val->len != mn->value_len) return false; - return strncmp(val->s, mn->value.s, val->len) == 0; - case OP_NE: - if (val->len != mn->value_len) return true; - return strncmp(val->s, mn->value.s, val->len) != 0; - case OP_LT: - { - int cmp = strncmp(val->s, mn->value.s, val->len); - if (cmp == 0) { - return val->len < mn->value_len; - } - return cmp < 0; - } - case OP_LTE: - return strncmp(val->s, mn->value.s, val->len) <= 0; - case OP_GT: - { - int cmp = strncmp(val->s, mn->value.s, val->len); - if (cmp == 0) { - return val->len > mn->value_len; - } - return cmp > 0; - } - case OP_GTE: - { - int cmp = strncmp(val->s, mn->value.s, val->len); - if (cmp == 0) { - return val->len == mn->value_len; - } - return cmp > 0; - } - case OP_RE: - return lsb_string_match(val->s, val->len, mn->value.s); - case OP_NRE: - return !lsb_string_match(val->s, val->len, mn->value.s); - default: - break; - } - return false; -} - - -static bool numeric_test(match_node *mn, double val) -{ - switch (mn->op) { - case OP_EQ: - return val == mn->value.d; - case OP_NE: - return val != mn->value.d; - case OP_LT: - return val < mn->value.d; - case OP_LTE: - return val <= mn->value.d; - case OP_GT: - return val > mn->value.d; - case OP_GTE: - return val >= mn->value.d; - default: - break; - } - return false; -} - - -static bool eval_node(match_node *mn, lsb_heka_message *m) -{ - switch (mn->op) { - case OP_TRUE: - return true; - case OP_FALSE: - return false; - default: - switch (mn->id) { - case LSB_PB_TIMESTAMP: - return numeric_test(mn, (double)m->timestamp); - case LSB_PB_TYPE: - return string_test(mn, &m->type); - case LSB_PB_LOGGER: - return string_test(mn, &m->logger); - case LSB_PB_SEVERITY: - return numeric_test(mn, m->severity); - case LSB_PB_PAYLOAD: - return string_test(mn, &m->payload); - case LSB_PB_ENV_VERSION: - return string_test(mn, &m->env_version); - case LSB_PB_PID: - return numeric_test(mn, m->pid); - case LSB_PB_HOSTNAME: - return string_test(mn, &m->hostname); - case LSB_PB_UUID: - return string_test(mn, &m->uuid); - default: - { - lsb_read_value val; - lsb_const_string variable = { .s = mn->variable, - .len = mn->variable_len }; - - switch (mn->value_type) { - case TYPE_STRING: - if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) - && val.type == LSB_READ_STRING) { - return string_test(mn, &val.u.s); - } - break; - case TYPE_NUMERIC: - case TYPE_BOOLEAN: - if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) - && (val.type == LSB_READ_NUMERIC || val.type == LSB_READ_BOOL)) { - return numeric_test(mn, val.u.d); - } - break; - case TYPE_NIL: - if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val)) { - return mn->op == OP_NE; - } - return mn->op == OP_EQ; - } - } - break; - } - break; - } - return false; -} - - -static bool eval_tree(match_node *mn, lsb_heka_message *m) -{ - bool match; - if (mn->left) { - match = eval_tree(mn->left, m); - } else { - match = eval_node(mn, m); - } - - if (match && mn->op == OP_OR) { - return match; // short circuit - } - - if (!match && mn->op == OP_AND) { - return match; // short circuit - } - - if (mn->right) { - match = eval_tree(mn->right, m); - } - return match; -} - - -static void load_op_node(lua_State *L, match_node *mn) -{ - const char *op = lua_tostring(L, -1); - if (strcmp(op, "||") == 0) { - mn->op = OP_OR; - } else if (strcmp(op, "&&") == 0) { - mn->op = OP_AND; - } else { - fprintf(stderr, "message_matcher unknown op: %s", op); - exit(EXIT_FAILURE); - } -} - - -static void load_expression_node(lua_State *L, match_node *mn) -{ - size_t len; - const char *tmp; - lua_getfield(L, -1, "op"); - tmp = lua_tolstring(L, -1, &len); - if (strcmp(tmp, "==") == 0) { - mn->op = OP_EQ; - } else if (strcmp(tmp, "!=") == 0) { - mn->op = OP_NE; - } else if (strcmp(tmp, ">=") == 0) { - mn->op = OP_GTE; - } else if (strcmp(tmp, ">") == 0) { - mn->op = OP_GT; - } else if (strcmp(tmp, "<=") == 0) { - mn->op = OP_LTE; - } else if (strcmp(tmp, "<") == 0) { - mn->op = OP_LT; - } else if (strcmp(tmp, "=~") == 0) { - mn->op = OP_RE; - } else if (strcmp(tmp, "!~") == 0) { - mn->op = OP_NRE; - } else if (strcmp(tmp, "TRUE") == 0) { - mn->op = OP_TRUE; - lua_pop(L, 1); - return; // no other args - } else if (strcmp(tmp, "FALSE") == 0) { - mn->op = OP_FALSE; // no other args - lua_pop(L, 1); - return; - } else { - fprintf(stderr, "message_matcher invalid op: %s", tmp); - exit(EXIT_FAILURE); - } - lua_pop(L, 1); - - lua_getfield(L, -1, "fi"); - if (lua_type(L, -1) == LUA_TNUMBER) { - mn->id = LSB_PB_FIELDS; - } - mn->fi = (int)lua_tointeger(L, -1); - lua_pop(L, 1); - - lua_getfield(L, -1, "ai"); - mn->ai = (int)lua_tointeger(L, -1); - lua_pop(L, 1); - - lua_getfield(L, -1, "variable"); - tmp = lua_tolstring(L, -1, &len); - if (mn->id == LSB_PB_FIELDS) { - mn->variable = malloc(len + 1); - mn->variable_len = len; - memcpy(mn->variable, tmp, len + 1); - } else { - if (strcmp(tmp, LSB_TIMESTAMP) == 0) { - mn->id = LSB_PB_TIMESTAMP; - } else if (strcmp(tmp, LSB_TYPE) == 0) { - mn->id = LSB_PB_TYPE; - } else if (strcmp(tmp, LSB_LOGGER) == 0) { - mn->id = LSB_PB_LOGGER; - } else if (strcmp(tmp, LSB_SEVERITY) == 0) { - mn->id = LSB_PB_SEVERITY; - } else if (strcmp(tmp, LSB_PAYLOAD) == 0) { - mn->id = LSB_PB_PAYLOAD; - } else if (strcmp(tmp, LSB_ENV_VERSION) == 0) { - mn->id = LSB_PB_ENV_VERSION; - } else if (strcmp(tmp, LSB_PID) == 0) { - mn->id = LSB_PB_PID; - } else if (strcmp(tmp, LSB_HOSTNAME) == 0) { - mn->id = LSB_PB_HOSTNAME; - } else if (strcmp(tmp, LSB_UUID) == 0) { - mn->id = LSB_PB_UUID; - } else { - fprintf(stderr, "broken message_matcher grammar: invalid header"); - exit(EXIT_FAILURE); - } - } - lua_pop(L, 1); - - lua_getfield(L, -1, "value"); - switch (lua_type(L, -1)) { - case LUA_TSTRING: - tmp = lua_tolstring(L, -1, &len); - mn->value_type = TYPE_STRING; - mn->value_len = len; - mn->value.s = malloc(len + 1); - memcpy(mn->value.s, tmp, len + 1); - break; - case LUA_TNUMBER: - mn->value_type = TYPE_NUMERIC; - mn->value.d = lua_tonumber(L, -1); - break; - case LUA_TBOOLEAN: - mn->value_type = TYPE_BOOLEAN; - mn->value.d = lua_toboolean(L, -1); - break; - case LUA_TNIL: - mn->value_type = TYPE_NIL; - break; - default: - fprintf(stderr, "message_matcher invalid value"); - exit(EXIT_FAILURE); - } - lua_pop(L, 1); -} - - -lsb_message_match_builder* -lsb_create_message_match_builder(const char *lua_path, const char *lua_cpath) -{ - -#if _WIN32 - if (_putenv("TZ=UTC") != 0) { - return NULL; - } -#else - if (setenv("TZ", "UTC", 1) != 0) { - return NULL; - } -#endif - - lsb_message_match_builder *mmb = malloc(sizeof*mmb); - mmb->parser = luaL_newstate(); - if (!mmb->parser) { - free(mmb); - return NULL; - } - - lua_createtable(mmb->parser, 0, 1); - lua_pushstring(mmb->parser, lua_path); - lua_setfield(mmb->parser, -2, "path"); - lua_pushstring(mmb->parser, lua_cpath); - lua_setfield(mmb->parser, -2, "cpath"); - lua_setfield(mmb->parser, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); - - luaL_openlibs(mmb->parser); - - if (luaL_dostring(mmb->parser, grammar)) { - free(mmb); - return NULL; - } - return mmb; -} - - -void lsb_destroy_message_match_builder(lsb_message_match_builder *mmb) -{ - if (!mmb) return; - - if (mmb->parser) { - lua_close(mmb->parser); - mmb->parser = NULL; - } - free(mmb); -} - - -lsb_message_matcher* -lsb_create_message_matcher(const lsb_message_match_builder *mmb, - const char *exp) -{ - if (!mmb || !exp) { - return NULL; - } - lua_pop(mmb->parser, lua_gettop(mmb->parser)); - - lua_getglobal(mmb->parser, "parse"); - if (!lua_isfunction(mmb->parser, -1)) { - return NULL; - } - lua_pushstring(mmb->parser, exp); - if (lua_pcall(mmb->parser, 1, 2, 0)) { - return NULL; - } - - if (lua_type(mmb->parser, 1) != LUA_TTABLE) { - return NULL; - } - int size = (int)lua_tointeger(mmb->parser, 2); - lsb_message_matcher *mm = calloc(sizeof(lsb_message_matcher) + - (sizeof(match_node) * size), 1); - if (!mm) { - return NULL; - } - mm->nodes_size = size; - - // load in reverse order so the root node will be first - for (int i = size, j = 0; i > 0; --i, ++j) { - lua_rawgeti(mmb->parser, 1, i); - switch (lua_type(mmb->parser, -1)) { - case LUA_TSTRING: - load_op_node(mmb->parser, &mm->nodes[j]); - break; - case LUA_TTABLE: - load_expression_node(mmb->parser, &mm->nodes[j]); - break; - default: - free(mm); - return NULL; - } - lua_pop(mmb->parser, 1); - } - lua_pop(mmb->parser, 2); - - // turn the postfix stack into an executable tree - match_node **stack = calloc(sizeof(match_node *) * size, 1); - if (!stack) { - free(mm); - return NULL; - } - - int top = 0; - for (int i = size - 1; i >= 0; --i) { - if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { - stack[top++] = &mm->nodes[i]; - } else { - mm->nodes[i].right = stack[--top]; - mm->nodes[i].left = stack[--top]; - stack[top++] = &mm->nodes[i]; - } - } - free(stack); - return mm; -} - - -void lsb_destroy_message_matcher(lsb_message_matcher *mm) -{ - if (!mm) return; - - for (int i = 0; i < mm->nodes_size; ++i) { - free(mm->nodes[i].variable); - switch (mm->nodes[i].value_type) { - case TYPE_STRING: - free(mm->nodes[i].value.s); - break; - default: - // no action required - break; - } - } - free(mm); -} - - -bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m) -{ - return eval_tree(mm->nodes, m); -} - -// add a lua wrapper so outputs can do some dynamic filtering -const char *mozsvc_heka_message_match_builder = - "mozsvc.heka_message_match_builder"; -const char *mozsvc_heka_message_match_builder_table = - "heka_message_match_builder"; -const char *mozsvc_heka_message_matcher = "mozsvc.heka_message_matcher"; - -static lsb_message_match_builder* check_mmb(lua_State *lua) -{ - lsb_message_match_builder **pmmb = - luaL_checkudata(lua, 1, mozsvc_heka_message_match_builder); - return *pmmb; -} - - -static lsb_message_matcher* check_mm(lua_State *lua) -{ - lsb_message_matcher **pmm = luaL_checkudata(lua, 1, - mozsvc_heka_message_matcher); - return *pmm; -} - - -static int mmb_new(lua_State *lua) -{ - const char *path = NULL; - const char *cpath = NULL; - lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); - if (lua_type(lua, -1) == LUA_TTABLE) { - lua_getfield(lua, -1, LSB_LUA_PATH); - path = lua_tostring(lua, -1); - lua_getfield(lua, -2, LSB_LUA_CPATH); - cpath = lua_tostring(lua, -1); - } else { - return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); - } - - lsb_message_match_builder **pmmb = lua_newuserdata(lua, sizeof**pmmb); - luaL_getmetatable(lua, mozsvc_heka_message_match_builder); - lua_setmetatable(lua, -2); - - - *pmmb = lsb_create_message_match_builder(path, cpath); - if (!*pmmb) { - return luaL_error(lua, "message_match_builder could not be created"); - } - return 1; -} - - -static int mmb_create_matcher(lua_State *lua) -{ - lsb_message_match_builder *mmb = check_mmb(lua); - const char *exp = luaL_checkstring(lua, 2); - - lsb_message_matcher **pmm = lua_newuserdata(lua, sizeof**pmm); - luaL_getmetatable(lua, mozsvc_heka_message_matcher); - lua_setmetatable(lua, -2); - - *pmm = lsb_create_message_matcher(mmb, exp); - if (!*pmm) { - return luaL_error(lua, "invalid message matcher expression"); - } - return 1; -} - - -static int mmb_gc(lua_State *lua) -{ - lsb_message_match_builder *mmb = check_mmb(lua); - lsb_destroy_message_match_builder(mmb); - return 0; -} - - -static int mm_eval(lua_State *lua) -{ - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __func__); - } - lsb_message_matcher *mm = check_mm(lua); - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); - if (!hsb->msg || !hsb->msg->raw.s) { - return luaL_error(lua, "no active message"); - } - lua_pushboolean(lua, lsb_eval_message_matcher(mm, hsb->msg)); - return 1; -} - - -static int mm_gc(lua_State *lua) -{ - lsb_message_matcher *mm = check_mm(lua); - lsb_destroy_message_matcher(mm); - return 0; -} - - -static const struct luaL_reg mmlib_m[] = -{ - { "__gc", mm_gc }, - { NULL, NULL } -}; - -static const struct luaL_reg mmblib_f[] = -{ - { "new", mmb_new }, - { NULL, NULL } -}; - - -static const struct luaL_reg mmblib_m[] = -{ - { "create_matcher", mmb_create_matcher }, - { "__gc", mmb_gc }, - { NULL, NULL } -}; - - -int luaopen_heka_message_match_builder(lua_State *lua) -{ - luaL_newmetatable(lua, mozsvc_heka_message_matcher); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, mmlib_m); - // special case parse_message since it needs access to the sandbox - lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); - if (lua_type(lua, -1) != LUA_TTABLE) { - return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); - } - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - lua_pushlightuserdata(lua, (void *)lsb); - lua_pushcclosure(lua, mm_eval, 1); - lua_setfield(lua, -3, "eval"); - lua_pop(lua, 2); // remove LSB_CONFIG_TABLE and metatable - - luaL_newmetatable(lua, mozsvc_heka_message_match_builder); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, mmblib_m); - luaL_register(lua, mozsvc_heka_message_match_builder_table, mmblib_f); - return 1; -} diff --git a/src/heka/message_matcher_impl.h b/src/heka/message_matcher_impl.h deleted file mode 100644 index c52e343..0000000 --- a/src/heka/message_matcher_impl.h +++ /dev/null @@ -1,30 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Hindsight Heka message matcher Lua wrapper @file */ - -#ifndef luasandbox_heka_message_matcher_impl_h_ -#define luasandbox_heka_message_matcher_impl_h_ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include "luasandbox/lua.h" - -extern const char *mozsvc_heka_message_match_builder; -extern const char *mozsvc_heka_message_match_builder_table; - -extern const char *mozsvc_heka_message_matcher; - -int luaopen_heka_message_match_builder(lua_State *lua); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index f18d9f7..f46ed63 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -13,12 +13,11 @@ #include #include "luasandbox.h" -#include "luasandbox/heka/message_matcher.h" +#include "luasandbox/util/heka_message_matcher.h" #include "luasandbox/util/protobuf.h" #include "luasandbox/util/running_stats.h" #include "luasandbox_output.h" #include "message_impl.h" -#include "message_matcher_impl.h" #include "rapidjson_impl.h" #include "sandbox_impl.h" #include "stream_reader_impl.h" @@ -37,12 +36,13 @@ lsb_err_id LSB_ERR_HEKA_INPUT = "invalid input"; static const char *pm_func_name = "process_message"; static const char *im_func_name = "inject_message"; +static const char *mozsvc_heka_message_matcher = "mozsvc.heka_message_matcher"; static int read_message(lua_State *lua) { lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); if (NULL == lsb) { - return luaL_error(lua, "read_message() invalid lightuserdata"); + return luaL_error(lua, "%s() invalid lightuserdata", __func__); } lsb_heka_sandbox *hsb = lsb_get_parent(lsb); @@ -54,6 +54,65 @@ static int read_message(lua_State *lua) } +static lsb_message_matcher* mm_check(lua_State *lua) +{ + lsb_message_matcher **ppmm = luaL_checkudata(lua, 1, + mozsvc_heka_message_matcher); + return *ppmm; +} + + +static int mm_gc(lua_State *lua) +{ + lsb_message_matcher *mm = mm_check(lua); + lsb_destroy_message_matcher(mm); + return 0; +} + + +static int mm_eval(lua_State *lua) +{ + lsb_message_matcher *mm = mm_check(lua); + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (!lsb) { + return luaL_error(lua, "%s() invalid lightuserdata", __func__); + } + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + if (!hsb->msg || !hsb->msg->raw.s) { + return luaL_error(lua, "no active message"); + } + lua_pushboolean(lua, lsb_eval_message_matcher(mm, hsb->msg)); + return 1; +} + + +static int mm_create(lua_State *lua) +{ + const char *exp = luaL_checkstring(lua, 1); + lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); + if (NULL == lsb) { + return luaL_error(lua, "%() invalid lightuserdata", __func__); + } + + lsb_message_matcher **ppmm = lua_newuserdata(lua, sizeof*ppmm); + if (luaL_newmetatable(lua, mozsvc_heka_message_matcher) == 1) { + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + lua_pushcclosure(lua, mm_gc, 0); + lua_setfield(lua, -2, "__gc"); + lua_pushlightuserdata(lua, lsb); + lua_pushcclosure(lua, mm_eval, 1); + lua_setfield(lua, -2, "eval"); + } + lua_setmetatable(lua, -2); + + *ppmm = lsb_create_message_matcher(exp); + if (!*ppmm) { + return luaL_error(lua, "invalid message matcher expression"); + } + return 1; +} + static int inject_message_input(lua_State *lua) { lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); @@ -694,17 +753,13 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); lsb_add_function(hsb->lsb, update_checkpoint, LSB_HEKA_UPDATE_CHECKPOINT); + lsb_add_function(hsb->lsb, mm_create, "create_message_matcher"); // preload Heka JSON with a pointer back to the sandbox luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); lua_pushstring(lua, mozsvc_heka_json_table); lua_pushlightuserdata(lua, (void *)hsb->lsb); lua_pushcclosure(lua, luaopen_heka_json, 1); lua_rawset(lua, -3); -// preload Heka message match builder with a pointer back to the sandbox - lua_pushstring(lua, mozsvc_heka_message_match_builder_table); - lua_pushlightuserdata(lua, (void *)hsb->lsb); - lua_pushcclosure(lua, luaopen_heka_message_match_builder, 1); - lua_rawset(lua, -3); #ifdef HAVE_KAFKA // preload the Heka Kafka producer lua_pushstring(lua, mozsvc_heka_kafka_producer_table); diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt index 0f23ca1..3fa403c 100644 --- a/src/heka/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -12,10 +12,6 @@ add_executable(test_heka_sandbox test_sandbox.c) target_link_libraries(test_heka_sandbox luasandboxheka luasandbox) add_test(NAME test_heka_sandbox COMMAND test_heka_sandbox) -add_executable(test_message_matcher test_message_matcher.c) -target_link_libraries(test_message_matcher luasandboxheka luasandboxutil) -add_test(NAME test_message_matcher COMMAND test_message_matcher) - if(LIBRTKAFKA_LIBRARY) add_executable(test_kafka test_kafka.c) target_link_libraries(test_kafka luasandboxheka luasandbox) diff --git a/src/heka/test/lua/output.lua b/src/heka/test/lua/output.lua index 61e0705..80f9337 100644 --- a/src/heka/test/lua/output.lua +++ b/src/heka/test/lua/output.lua @@ -30,18 +30,20 @@ function process_message(sequence_id) sid = sequence_id return -5 else - require "heka_message_match_builder" - local mmb = heka_message_match_builder.new() - local mm = mmb:create_matcher("Type == 'type'") + local ok, mm = pcall(create_message_matcher, "Type == 'type'") + assert(ok, mm) assert(mm:eval()) - mm = mmb:create_matcher("Type == 'foo'") + + ok, mm = pcall(create_message_matcher, "Type == 'foo'") + assert(ok, mm) assert(not mm:eval()) - local ok, err = pcall(mmb.create_matcher, mmb, "Widget == 'foo'") + ok, err = pcall(create_message_matcher, "Widget == 'foo'") assert(not ok) ok, err = pcall(update_checkpoint, sequence_id) assert(not ok) + return -3 -- retry end end @@ -49,9 +51,8 @@ function process_message(sequence_id) end function timer_event(ns, shutdown) - require "heka_message_match_builder" - local mmb = heka_message_match_builder.new() - local mm = mmb:create_matcher("Type == 'foo'") + local ok, mm = pcall(create_message_matcher, "Type == 'foo'"); + assert(ok, mm) ok, err = pcall(mm.eval, mm) assert(err == "no active message") end diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 782fc4a..2dfc4b0 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -508,8 +508,7 @@ static char* test_pm_output() mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); - mu_assert_rv(-3, lsb_heka_pm_output(hsb, &m, (void *)100, - false)); + mu_assert_rv(-3, lsb_heka_pm_output(hsb, &m, (void *)100, false)); mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); stats = lsb_heka_get_stats(hsb); mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); diff --git a/src/luasandbox.c b/src/luasandbox.c index 162754e..a09d6b4 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -390,21 +390,6 @@ static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger *logger) } -static bool set_tz() -{ -#if _WIN32 - if (_putenv("TZ=UTC") != 0) { - return false; - } -#else - if (setenv("TZ", "UTC", 1) != 0) { - return false; - } -#endif - return true; -} - - static void set_random_seed() { bool seeded = false; @@ -445,7 +430,7 @@ lsb_lua_sandbox* lsb_create(void *parent, return NULL; } - if (!set_tz()) { + if (!lsb_set_tz(NULL)) { if (logger && logger->cb) { logger->cb(logger->context, __func__, 3, "fail to set the TZ to UTC"); } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index ae96542..ef6f7f9 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -4,6 +4,8 @@ set(UTIL_SRC heka_message.c +heka_message_matcher.c +heka_message_matcher_parser.c input_buffer.c output_buffer.c protobuf.c diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c new file mode 100644 index 0000000..538898c --- /dev/null +++ b/src/util/heka_message_matcher.c @@ -0,0 +1,202 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight/Heka message matcher implementation @file */ + +#include "heka_message_matcher_impl.h" + +#include +#include +#include + +#include "luasandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox/lualib.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" +#include "luasandbox/util/string.h" +#include "luasandbox/util/string_matcher.h" + + +static bool string_test(match_node *mn, lsb_const_string *val) +{ + switch (mn->op) { + case OP_EQ: + if (val->len != mn->value_len) return false; + return strncmp(val->s, mn->value.s, val->len) == 0; + case OP_NE: + if (val->len != mn->value_len) return true; + return strncmp(val->s, mn->value.s, val->len) != 0; + case OP_LT: + { + int cmp = strncmp(val->s, mn->value.s, val->len); + if (cmp == 0) { + return val->len < mn->value_len; + } + return cmp < 0; + } + case OP_LTE: + return strncmp(val->s, mn->value.s, val->len) <= 0; + case OP_GT: + { + int cmp = strncmp(val->s, mn->value.s, val->len); + if (cmp == 0) { + return val->len > mn->value_len; + } + return cmp > 0; + } + case OP_GTE: + { + int cmp = strncmp(val->s, mn->value.s, val->len); + if (cmp == 0) { + return val->len == mn->value_len; + } + return cmp > 0; + } + case OP_RE: + return lsb_string_match(val->s, val->len, mn->value.s); + case OP_NRE: + return !lsb_string_match(val->s, val->len, mn->value.s); + default: + break; + } + return false; +} + + +static bool numeric_test(match_node *mn, double val) +{ + switch (mn->op) { + case OP_EQ: + return val == mn->value.d; + case OP_NE: + return val != mn->value.d; + case OP_LT: + return val < mn->value.d; + case OP_LTE: + return val <= mn->value.d; + case OP_GT: + return val > mn->value.d; + case OP_GTE: + return val >= mn->value.d; + default: + break; + } + return false; +} + + +static bool eval_node(match_node *mn, lsb_heka_message *m) +{ + switch (mn->op) { + case OP_TRUE: + return true; + case OP_FALSE: + return false; + default: + switch (mn->id) { + case LSB_PB_TIMESTAMP: + return numeric_test(mn, (double)m->timestamp); + case LSB_PB_TYPE: + return string_test(mn, &m->type); + case LSB_PB_LOGGER: + return string_test(mn, &m->logger); + case LSB_PB_SEVERITY: + return numeric_test(mn, m->severity); + case LSB_PB_PAYLOAD: + return string_test(mn, &m->payload); + case LSB_PB_ENV_VERSION: + return string_test(mn, &m->env_version); + case LSB_PB_PID: + return numeric_test(mn, m->pid); + case LSB_PB_HOSTNAME: + return string_test(mn, &m->hostname); + case LSB_PB_UUID: + return string_test(mn, &m->uuid); + default: + { + lsb_read_value val; + lsb_const_string variable = { .s = mn->variable, + .len = mn->variable_len }; + + switch (mn->value_type) { + case TYPE_STRING: + if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) + && val.type == LSB_READ_STRING) { + return string_test(mn, &val.u.s); + } + break; + case TYPE_NUMERIC: + case TYPE_BOOLEAN: + if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) + && (val.type == LSB_READ_NUMERIC || val.type == LSB_READ_BOOL)) { + return numeric_test(mn, val.u.d); + } + break; + case TYPE_NIL: + if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val)) { + return mn->op == OP_NE; + } + return mn->op == OP_EQ; + } + } + break; + } + break; + } + return false; +} + + +static bool eval_tree(match_node *root, match_node *mn, lsb_heka_message *m) +{ + bool match; + if (mn->id == 0 && mn->fi) { + match = eval_tree(root, root + mn->fi, m); + } else { + match = eval_node(mn, m); + } + + if (match && mn->op == OP_OR) { + return match; // short circuit + } + + if (!match && mn->op == OP_AND) { + return match; // short circuit + } + + if (mn->id == 0 && mn->ai) { + match = eval_tree(root, root + mn->ai, m); + } + return match; +} + + +void lsb_destroy_message_matcher(lsb_message_matcher *mm) +{ + if (!mm) return; + + for (int i = 0; i < mm->nodes[0].size; ++i) { + free(mm->nodes[i].variable); + switch (mm->nodes[i].value_type) { + case TYPE_STRING: + free(mm->nodes[i].value.s); + break; + default: + // no action required + break; + } + } + free(mm->nodes); + free(mm); +} + + +bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m) +{ + return eval_tree(mm->nodes, mm->nodes, m); +} diff --git a/src/util/heka_message_matcher_impl.h b/src/util/heka_message_matcher_impl.h new file mode 100644 index 0000000..57d9c9b --- /dev/null +++ b/src/util/heka_message_matcher_impl.h @@ -0,0 +1,61 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher implementation header @file */ + +#ifndef luasandbox_util_heka_message_matcher_impl_h_ +#define luasandbox_util_heka_message_matcher_impl_h_ + +#include + +typedef enum { + OP_EQ, + OP_NE, + OP_GTE, + OP_GT, + OP_LTE, + OP_LT, + OP_RE, + OP_NRE, + OP_TRUE, + OP_FALSE, + OP_OPEN, + OP_OR, + OP_AND +} match_operation; + + +typedef enum { + TYPE_NIL, + TYPE_STRING, + TYPE_NUMERIC, + TYPE_BOOLEAN, +} match_type; + + +typedef struct match_node { + unsigned char id; + unsigned char op; + unsigned char size; + unsigned char value_type; + unsigned char value_len; + unsigned char variable_len; + unsigned char fi; // left node index for logical op + unsigned char ai; // right node index for logical op + char *variable; + + union { + char *s; + double d; + } value; +} match_node; + + +struct lsb_message_matcher { + match_node *nodes; +}; + +#endif diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c new file mode 100644 index 0000000..85f0105 --- /dev/null +++ b/src/util/heka_message_matcher_parser.c @@ -0,0 +1,2213 @@ +/* A recursive-descent parser generated by peg 0.1.15 */ + +#include +#include +#include +#define YYRULECOUNT 54 + +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight message matcher parser @file */ + +#include +#include +#include +#include +#include + +#include "heka_message_matcher_impl.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" + +#ifndef _MSC_VER +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-function" +#endif + +typedef struct match_node_array { + match_node *a; + int pos; + int size; +} match_node_array; + + +typedef struct input_string { + const char *s; + size_t pos; + size_t size; +} input_string; + + +typedef struct context { + match_node_array out; + match_node_array ops; + match_node mn; + struct tm tms; + int cond_cnt; + input_string is; +} context; + + +#define YY_PARSE(T) static T +#define YY_CTX_LOCAL 1 +#define YY_CTX_MEMBERS \ + context ctx; + +#define YY_INPUT(yy, buf, result, max_size) \ +{ \ + input_string *is = &yy->ctx.is; \ + result = is->size - is->pos; \ + if (result > 0) { \ + if (max_size >= result) { \ + memcpy(buf, &is->s[is->pos], result); \ + is->pos += result; \ + } else { \ + memcpy(buf, &is->s[is->pos], max_size); \ + is->pos += max_size; \ + result = max_size; \ + } \ + } \ +} + + +static void init_match_node(match_node *mn) +{ + memset(mn, 0, sizeof(match_node)); +} + + +static void move_match_node(match_node *dest, match_node *src) +{ + memcpy(dest, src, sizeof(match_node)); + init_match_node(src); // dest now owns the memory, wipe the pointers +} + + +static void realloc_mna(match_node_array *mna) +{ + size_t bytes = sizeof(match_node) * ++mna->size; + match_node *tmp = realloc(mna->a, bytes); + if (tmp) { + mna->a = tmp; + init_match_node(&mna->a[mna->size - 1]); + } else { + fprintf(stderr, "realloc failed\n"); + exit(1); + } +} + + +static void push_output(context *ctx, match_node *mn) +{ + if (!ctx->out.a || ctx->out.pos == ctx->out.size) { + realloc_mna(&ctx->out); + } + move_match_node(&ctx->out.a[ctx->out.pos++], mn); +} + + +static void push_op(context *ctx, match_operation op) +{ + if (!ctx->ops.a) { + realloc_mna(&ctx->ops); + ctx->ops.a[ctx->ops.pos++].op = op; + return; + } + + if (op == OP_OPEN || op > ctx->ops.a[ctx->ops.pos - 1].op) { + if (ctx->ops.pos == ctx->ops.size) { + realloc_mna(&ctx->ops); + } + ctx->ops.a[ctx->ops.pos++].op = op; + } else { + push_output(ctx, &ctx->ops.a[ctx->ops.pos - 1]); + ctx->ops.a[ctx->ops.pos - 1].op = op; + } +} + + +static void pop_to_paren(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) break; + push_output(ctx, op); + } +} + + +static void pop_all_ops(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) continue; + push_output(ctx, op); + } +} + + +static void update_date(context *ctx, int year, int mon, int day) +{ + ctx->tms.tm_isdst = -1; + ctx->tms.tm_year = year - 1900; + ctx->tms.tm_mon = mon - 1; + ctx->tms.tm_mday = day; +} + + +static void update_time(context *ctx, int hour, int minute, int sec) +{ + ctx->tms.tm_hour = hour; + ctx->tms.tm_min = minute; + ctx->tms.tm_sec = sec; +} + + +static void update_offset(context *ctx, char sign, int hour, int minute) +{ + ctx->mn.value.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); +} + + +static void set_field(context *ctx, char *name) +{ + ctx->mn.id = LSB_PB_FIELDS; + ctx->mn.variable_len = strlen(name); + ctx->mn.variable = malloc(ctx->mn.variable_len + 1); + if (!ctx->mn.variable) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.variable, name, ctx->mn.variable_len + 1); +} + + +static void set_timestamp(context *ctx) +{ + ctx->mn.id = LSB_PB_TIMESTAMP; + ctx->mn.value_type = TYPE_NUMERIC; + if (ctx->tms.tm_isdst == -1) { + ctx->mn.value.d += mktime(&ctx->tms); + ctx->mn.value.d *= 1e9; + } + memset(&ctx->tms, 0, sizeof(struct tm)); +} + + +static void set_numeric_value(context *ctx, char *s) +{ + ctx->mn.value_type = TYPE_NUMERIC; + ctx->mn.value.d = strtod(s, NULL); +} + + +static void set_string_value(context *ctx, char *s) +{ + ctx->mn.value_type = TYPE_STRING; + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + ++i; + } + s[j] = s[i]; + } + s[j] = 0; + + ctx->mn.value_len = j; + ctx->mn.value.s = malloc(j + 1); + if (!ctx->mn.value.s) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.value.s, s, j + 1); +} + + +static bool check_string_len(char *s) +{ + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + ++i; + } + } + return (j > UCHAR_MAX) ? false : true; +} + + +static void set_boolean_value(context *ctx, bool b) +{ + ctx->mn.value_type = TYPE_BOOLEAN; + ctx->mn.value.d = b; +} + + +static int cond_cnt(context *ctx) +{ + return (++ctx->cond_cnt * 2 + 1 > UCHAR_MAX) ? 0 : 1; +} + + +#ifndef YY_MALLOC +#define YY_MALLOC(C, N) malloc(N) +#endif +#ifndef YY_REALLOC +#define YY_REALLOC(C, P, N) realloc(P, N) +#endif +#ifndef YY_FREE +#define YY_FREE(C, P) free(P) +#endif +#ifndef YY_LOCAL +#define YY_LOCAL(T) static T +#endif +#ifndef YY_ACTION +#define YY_ACTION(T) static T +#endif +#ifndef YY_RULE +#define YY_RULE(T) static T +#endif +#ifndef YY_PARSE +#define YY_PARSE(T) T +#endif +#ifndef YYPARSE +#define YYPARSE yyparse +#endif +#ifndef YYPARSEFROM +#define YYPARSEFROM yyparsefrom +#endif +#ifndef YYRELEASE +#define YYRELEASE yyrelease +#endif +#ifndef YY_BEGIN +#define YY_BEGIN ( yy->__begin= yy->__pos, 1) +#endif +#ifndef YY_END +#define YY_END ( yy->__end= yy->__pos, 1) +#endif +#ifdef YY_DEBUG +# define yyprintf(args) fprintf args +#else +# define yyprintf(args) +#endif +#ifndef YYSTYPE +#define YYSTYPE int +#endif +#ifndef YY_STACK_SIZE +#define YY_STACK_SIZE 128 +#endif + +#ifndef YY_BUFFER_SIZE +#define YY_BUFFER_SIZE 1024 +#endif + +#ifndef YY_PART + +typedef struct _yycontext yycontext; +typedef void (*yyaction)(yycontext *yy, char *yytext, int yyleng); +typedef struct _yythunk { int begin, end; yyaction action; struct _yythunk *next; } yythunk; + +struct _yycontext { + char *__buf; + int __buflen; + int __pos; + int __limit; + char *__text; + int __textlen; + int __begin; + int __end; + int __textmax; + yythunk *__thunks; + int __thunkslen; + int __thunkpos; + YYSTYPE __; + YYSTYPE *__val; + YYSTYPE *__vals; + int __valslen; +#ifdef YY_CTX_MEMBERS + YY_CTX_MEMBERS +#endif +}; + +#ifdef YY_CTX_LOCAL +#define YY_CTX_PARAM_ yycontext *yyctx, +#define YY_CTX_PARAM yycontext *yyctx +#define YY_CTX_ARG_ yyctx, +#define YY_CTX_ARG yyctx +#ifndef YY_INPUT +#define YY_INPUT(yy, buf, result, max_size) \ + { \ + int yyc= getchar(); \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ + yyprintf((stderr, "<%c>", yyc)); \ + } +#endif +#else +#define YY_CTX_PARAM_ +#define YY_CTX_PARAM +#define YY_CTX_ARG_ +#define YY_CTX_ARG +yycontext _yyctx= { 0, 0 }; +yycontext *yyctx= &_yyctx; +#ifndef YY_INPUT +#define YY_INPUT(buf, result, max_size) \ + { \ + int yyc= getchar(); \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ + yyprintf((stderr, "<%c>", yyc)); \ + } +#endif +#endif + +YY_LOCAL(int) yyrefill(yycontext *yy) +{ + int yyn; + while (yy->__buflen - yy->__pos < 512) + { + yy->__buflen *= 2; + yy->__buf= (char *)YY_REALLOC(yy, yy->__buf, yy->__buflen); + } +#ifdef YY_CTX_LOCAL + YY_INPUT(yy, (yy->__buf + yy->__pos), yyn, (yy->__buflen - yy->__pos)); +#else + YY_INPUT((yy->__buf + yy->__pos), yyn, (yy->__buflen - yy->__pos)); +#endif + if (!yyn) return 0; + yy->__limit += yyn; + return 1; +} + +YY_LOCAL(int) yymatchDot(yycontext *yy) +{ + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + ++yy->__pos; + return 1; +} + +YY_LOCAL(int) yymatchChar(yycontext *yy, int c) +{ + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + if ((unsigned char)yy->__buf[yy->__pos] == c) + { + ++yy->__pos; + yyprintf((stderr, " ok yymatchChar(yy, %c) @ %s\n", c, yy->__buf+yy->__pos)); + return 1; + } + yyprintf((stderr, " fail yymatchChar(yy, %c) @ %s\n", c, yy->__buf+yy->__pos)); + return 0; +} + +YY_LOCAL(int) yymatchString(yycontext *yy, const char *s) +{ + int yysav= yy->__pos; + while (*s) + { + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + if (yy->__buf[yy->__pos] != *s) + { + yy->__pos= yysav; + return 0; + } + ++s; + ++yy->__pos; + } + return 1; +} + +YY_LOCAL(int) yymatchClass(yycontext *yy, unsigned char *bits) +{ + int c; + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + c= (unsigned char)yy->__buf[yy->__pos]; + if (bits[c >> 3] & (1 << (c & 7))) + { + ++yy->__pos; + yyprintf((stderr, " ok yymatchClass @ %s\n", yy->__buf+yy->__pos)); + return 1; + } + yyprintf((stderr, " fail yymatchClass @ %s\n", yy->__buf+yy->__pos)); + return 0; +} + +YY_LOCAL(void) yyDo(yycontext *yy, yyaction action, int begin, int end) +{ + while (yy->__thunkpos >= yy->__thunkslen) + { + yy->__thunkslen *= 2; + yy->__thunks= (yythunk *)YY_REALLOC(yy, yy->__thunks, sizeof(yythunk) * yy->__thunkslen); + } + yy->__thunks[yy->__thunkpos].begin= begin; + yy->__thunks[yy->__thunkpos].end= end; + yy->__thunks[yy->__thunkpos].action= action; + ++yy->__thunkpos; +} + +YY_LOCAL(int) yyText(yycontext *yy, int begin, int end) +{ + int yyleng= end - begin; + if (yyleng <= 0) + yyleng= 0; + else + { + while (yy->__textlen < (yyleng + 1)) + { + yy->__textlen *= 2; + yy->__text= (char *)YY_REALLOC(yy, yy->__text, yy->__textlen); + } + memcpy(yy->__text, yy->__buf + begin, yyleng); + } + yy->__text[yyleng]= '\0'; + return yyleng; +} + +YY_LOCAL(void) yyDone(yycontext *yy) +{ + int pos; + for (pos= 0; pos < yy->__thunkpos; ++pos) + { + yythunk *thunk= &yy->__thunks[pos]; + int yyleng= thunk->end ? yyText(yy, thunk->begin, thunk->end) : thunk->begin; + yyprintf((stderr, "DO [%d] %p %s\n", pos, thunk->action, yy->__text)); + thunk->action(yy, yy->__text, yyleng); + } + yy->__thunkpos= 0; +} + +YY_LOCAL(void) yyCommit(yycontext *yy) +{ + if ((yy->__limit -= yy->__pos)) + { + memmove(yy->__buf, yy->__buf + yy->__pos, yy->__limit); + } + yy->__begin -= yy->__pos; + yy->__end -= yy->__pos; + yy->__pos= yy->__thunkpos= 0; +} + +YY_LOCAL(int) yyAccept(yycontext *yy, int tp0) +{ + if (tp0) + { + fprintf(stderr, "accept denied at %d\n", tp0); + return 0; + } + else + { + yyDone(yy); + yyCommit(yy); + } + return 1; +} + +YY_LOCAL(void) yyPush(yycontext *yy, char *text, int count) +{ + yy->__val += count; + while (yy->__valslen <= yy->__val - yy->__vals) + { + long offset= yy->__val - yy->__vals; + int olen = yy->__valslen; + yy->__valslen *= 2; + yy->__vals= (YYSTYPE *)YY_REALLOC(yy, yy->__vals, sizeof(YYSTYPE) * yy->__valslen); + yy->__val= yy->__vals + offset; + memset(yy->__vals + olen, 0, sizeof(YYSTYPE) * olen); + } +} +YY_LOCAL(void) yyPop(yycontext *yy, char *text, int count) { yy->__val -= count; } +YY_LOCAL(void) yySet(yycontext *yy, char *text, int count) { yy->__val[count]= yy->__; } + +#endif /* YY_PART */ + +#define YYACCEPT yyAccept(yy, yythunkpos0) + +YY_RULE(int) yy_second_frac(yycontext *yy); /* 54 */ +YY_RULE(int) yy_second(yycontext *yy); /* 53 */ +YY_RULE(int) yy_minute(yycontext *yy); /* 52 */ +YY_RULE(int) yy_hour(yycontext *yy); /* 51 */ +YY_RULE(int) yy_timeoffset(yycontext *yy); /* 50 */ +YY_RULE(int) yy_partialtime(yycontext *yy); /* 49 */ +YY_RULE(int) yy_fulltime(yycontext *yy); /* 48 */ +YY_RULE(int) yy_day(yycontext *yy); /* 47 */ +YY_RULE(int) yy_month(yycontext *yy); /* 46 */ +YY_RULE(int) yy_year(yycontext *yy); /* 45 */ +YY_RULE(int) yy_fulldate(yycontext *yy); /* 44 */ +YY_RULE(int) yy_rfc3339(yycontext *yy); /* 43 */ +YY_RULE(int) yy_ts_quoted(yycontext *yy); /* 42 */ +YY_RULE(int) yy_zero_to_255(yycontext *yy); /* 41 */ +YY_RULE(int) yy_index(yycontext *yy); /* 40 */ +YY_RULE(int) yy_nil(yycontext *yy); /* 39 */ +YY_RULE(int) yy_fields(yycontext *yy); /* 38 */ +YY_RULE(int) yy_exponent(yycontext *yy); /* 37 */ +YY_RULE(int) yy_decimal(yycontext *yy); /* 36 */ +YY_RULE(int) yy_number(yycontext *yy); /* 35 */ +YY_RULE(int) yy_sign(yycontext *yy); /* 34 */ +YY_RULE(int) yy_numeric_value(yycontext *yy); /* 33 */ +YY_RULE(int) yy_numeric_headers(yycontext *yy); /* 32 */ +YY_RULE(int) yy_string_match(yycontext *yy); /* 31 */ +YY_RULE(int) yy_string_value(yycontext *yy); /* 30 */ +YY_RULE(int) yy_string_headers(yycontext *yy); /* 29 */ +YY_RULE(int) yy_boolean(yycontext *yy); /* 28 */ +YY_RULE(int) yy_false(yycontext *yy); /* 27 */ +YY_RULE(int) yy_true(yycontext *yy); /* 26 */ +YY_RULE(int) yy_relational(yycontext *yy); /* 25 */ +YY_RULE(int) yy_op_lt(yycontext *yy); /* 24 */ +YY_RULE(int) yy_op_lte(yycontext *yy); /* 23 */ +YY_RULE(int) yy_op_gt(yycontext *yy); /* 22 */ +YY_RULE(int) yy_op_gte(yycontext *yy); /* 21 */ +YY_RULE(int) yy_op_sne(yycontext *yy); /* 20 */ +YY_RULE(int) yy_op_seq(yycontext *yy); /* 19 */ +YY_RULE(int) yy_op_ne(yycontext *yy); /* 18 */ +YY_RULE(int) yy_op_eq(yycontext *yy); /* 17 */ +YY_RULE(int) yy_boolean_test(yycontext *yy); /* 16 */ +YY_RULE(int) yy_ts_test(yycontext *yy); /* 15 */ +YY_RULE(int) yy_field_test(yycontext *yy); /* 14 */ +YY_RULE(int) yy_numeric_test(yycontext *yy); /* 13 */ +YY_RULE(int) yy_string_test(yycontext *yy); /* 12 */ +YY_RULE(int) yy_close(yycontext *yy); /* 11 */ +YY_RULE(int) yy_open(yycontext *yy); /* 10 */ +YY_RULE(int) yy_test(yycontext *yy); /* 9 */ +YY_RULE(int) yy_and(yycontext *yy); /* 8 */ +YY_RULE(int) yy_expr(yycontext *yy); /* 7 */ +YY_RULE(int) yy_or(yycontext *yy); /* 6 */ +YY_RULE(int) yy_anded(yycontext *yy); /* 5 */ +YY_RULE(int) yy_eol(yycontext *yy); /* 4 */ +YY_RULE(int) yy_ored(yycontext *yy); /* 3 */ +YY_RULE(int) yy_sp(yycontext *yy); /* 2 */ +YY_RULE(int) yy_match(yycontext *yy); /* 1 */ + +YY_ACTION(void) yy_1_nil(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_nil\n")); + { + yy->ctx.mn.value_type = TYPE_NIL;; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_second_frac(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_second_frac\n")); + { + yy->ctx.mn.value.d += strtod(yytext, NULL); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_second(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_second\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_minute(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_minute\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_hour(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_hour\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_timeoffset(yycontext *yy, char *yytext, int yyleng) +{ +#define m yy->__val[-1] +#define h yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_timeoffset\n")); + { + update_offset(&yy->ctx, yytext[0], h, m); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef m +#undef h +} +YY_ACTION(void) yy_1_timeoffset(yycontext *yy, char *yytext, int yyleng) +{ +#define m yy->__val[-1] +#define h yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_timeoffset\n")); + { + update_offset(&yy->ctx, '+', 0, 0); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef m +#undef h +} +YY_ACTION(void) yy_1_partialtime(yycontext *yy, char *yytext, int yyleng) +{ +#define s yy->__val[-1] +#define m yy->__val[-2] +#define h yy->__val[-3] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_partialtime\n")); + { + update_time(&yy->ctx, h, m, s); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef s +#undef m +#undef h +} +YY_ACTION(void) yy_1_day(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_day\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_month(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_month\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_year(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_year\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) +{ +#define d yy->__val[-1] +#define m yy->__val[-2] +#define y yy->__val[-3] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_fulldate\n")); + { + update_date(&yy->ctx, y, m, d); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef d +#undef m +#undef y +} +YY_ACTION(void) yy_1_ts_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_ts_test\n")); + { + set_timestamp(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_index(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_index\n")); + { + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_3_fields(yycontext *yy, char *yytext, int yyleng) +{ +#define a yy->__val[-1] +#define f yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_3_fields\n")); + { + yy->ctx.mn.ai = a; + } +#undef yythunkpos +#undef yypos +#undef yy +#undef a +#undef f +} +YY_ACTION(void) yy_2_fields(yycontext *yy, char *yytext, int yyleng) +{ +#define a yy->__val[-1] +#define f yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_fields\n")); + { + yy->ctx.mn.fi = f; + } +#undef yythunkpos +#undef yypos +#undef yy +#undef a +#undef f +} +YY_ACTION(void) yy_1_fields(yycontext *yy, char *yytext, int yyleng) +{ +#define a yy->__val[-1] +#define f yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_fields\n")); + { + set_field(&yy->ctx, yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef a +#undef f +} +YY_ACTION(void) yy_1_numeric_value(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_numeric_value\n")); + { + set_numeric_value(&yy->ctx, yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_numeric_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_numeric_headers\n")); + { + yy->ctx.mn.id = LSB_PB_PID; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_numeric_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_numeric_headers\n")); + { + yy->ctx.mn.id = LSB_PB_SEVERITY; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_value\n")); + { + set_string_value(&yy->ctx, yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_6_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_6_string_headers\n")); + { + yy->ctx.mn.id = LSB_PB_UUID; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_5_string_headers\n")); + { + yy->ctx.mn.id = LSB_PB_PAYLOAD; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_4_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_4_string_headers\n")); + { + yy->ctx.mn.id = LSB_PB_ENV_VERSION; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_3_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_3_string_headers\n")); + { + yy->ctx.mn.id = LSB_PB_HOSTNAME; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_string_headers\n")); + { + yy->ctx.mn.id = LSB_PB_LOGGER; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_headers\n")); + { + yy->ctx.mn.id = LSB_PB_TYPE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_close\n")); + { + pop_to_paren(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_open(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_open\n")); + { + push_op(&yy->ctx, OP_OPEN); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_or(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_or\n")); + { + push_op(&yy->ctx, OP_OR); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_and(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_and\n")); + { + push_op(&yy->ctx, OP_AND); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_boolean(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_boolean\n")); + { + set_boolean_value(&yy->ctx, 0); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_boolean(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_boolean\n")); + { + set_boolean_value(&yy->ctx, 1); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_boolean_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_boolean_test\n")); + { + yy->ctx.mn.op = OP_FALSE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_boolean_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_boolean_test\n")); + { + yy->ctx.mn.op = OP_TRUE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_lt(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_lt\n")); + { + yy->ctx.mn.op = OP_LT; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_lte(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_lte\n")); + { + yy->ctx.mn.op = OP_LTE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_gt(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_gt\n")); + { + yy->ctx.mn.op = OP_GT; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_gte(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_gte\n")); + { + yy->ctx.mn.op = OP_GTE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_sne(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_sne\n")); + { + yy->ctx.mn.op = OP_NRE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_seq(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_seq\n")); + { + yy->ctx.mn.op = OP_RE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_ne(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_ne\n")); + { + yy->ctx.mn.op = OP_NE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_eq(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_eq\n")); + { + yy->ctx.mn.op = OP_EQ; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_test\n")); + { + push_output(&yy->ctx, &yy->ctx.mn); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_match(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_match\n")); + { + pop_all_ops(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} + +YY_RULE(int) yy_second_frac(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "second_frac")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l1; +#undef yytext +#undef yyleng + } if (!yy_decimal(yy)) goto l1; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l1; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_second_frac, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "second_frac", yy->__buf+yy->__pos)); + return 1; + l1:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "second_frac", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_second(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "second")); + { int yypos3= yy->__pos, yythunkpos3= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l4; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l4; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l4; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l4; +#undef yytext +#undef yyleng + } goto l3; + l4:; yy->__pos= yypos3; yy->__thunkpos= yythunkpos3; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l2; +#undef yytext +#undef yyleng + } if (!yymatchString(yy, "60")) goto l2; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l2; +#undef yytext +#undef yyleng + } + } + l3:; yyDo(yy, yy_1_second, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "second", yy->__buf+yy->__pos)); + return 1; + l2:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "second", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_minute(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "minute")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l5; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l5; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l5; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l5; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_minute, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "minute", yy->__buf+yy->__pos)); + return 1; + l5:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "minute", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_hour(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "hour")); + { int yypos7= yy->__pos, yythunkpos7= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l8; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l8; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l8; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l8; +#undef yytext +#undef yyleng + } goto l7; + l8:; yy->__pos= yypos7; yy->__thunkpos= yythunkpos7; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l6; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '2')) goto l6; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\017\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l6; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l6; +#undef yytext +#undef yyleng + } + } + l7:; yyDo(yy, yy_1_hour, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "hour", yy->__buf+yy->__pos)); + return 1; + l6:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "hour", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_timeoffset(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 2, 0); + yyprintf((stderr, "%s\n", "timeoffset")); + { int yypos10= yy->__pos, yythunkpos10= yy->__thunkpos; if (!yymatchChar(yy, 'Z')) goto l11; yyDo(yy, yy_1_timeoffset, yy->__begin, yy->__end); goto l10; + l11:; yy->__pos= yypos10; yy->__thunkpos= yythunkpos10; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l9; +#undef yytext +#undef yyleng + } if (!yy_sign(yy)) goto l9; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l9; +#undef yytext +#undef yyleng + } if (!yy_hour(yy)) goto l9; yyDo(yy, yySet, -2, 0); if (!yy_minute(yy)) goto l9; yyDo(yy, yySet, -1, 0); yyDo(yy, yy_2_timeoffset, yy->__begin, yy->__end); + } + l10:; + yyprintf((stderr, " ok %s @ %s\n", "timeoffset", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 2, 0); + return 1; + l9:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "timeoffset", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_partialtime(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 3, 0); + yyprintf((stderr, "%s\n", "partialtime")); if (!yy_hour(yy)) goto l12; yyDo(yy, yySet, -3, 0); if (!yymatchChar(yy, ':')) goto l12; if (!yy_minute(yy)) goto l12; yyDo(yy, yySet, -2, 0); if (!yymatchChar(yy, ':')) goto l12; if (!yy_second(yy)) goto l12; yyDo(yy, yySet, -1, 0); + { int yypos13= yy->__pos, yythunkpos13= yy->__thunkpos; if (!yy_second_frac(yy)) goto l13; goto l14; + l13:; yy->__pos= yypos13; yy->__thunkpos= yythunkpos13; + } + l14:; yyDo(yy, yy_1_partialtime, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "partialtime", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 3, 0); + return 1; + l12:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "partialtime", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_fulltime(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "fulltime")); if (!yy_partialtime(yy)) goto l15; if (!yy_timeoffset(yy)) goto l15; + yyprintf((stderr, " ok %s @ %s\n", "fulltime", yy->__buf+yy->__pos)); + return 1; + l15:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "fulltime", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_day(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "day")); + { int yypos17= yy->__pos, yythunkpos17= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l18; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l18; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l18; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l18; +#undef yytext +#undef yyleng + } goto l17; + l18:; yy->__pos= yypos17; yy->__thunkpos= yythunkpos17; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l19; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '0')) goto l19; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l19; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l19; +#undef yytext +#undef yyleng + } goto l17; + l19:; yy->__pos= yypos17; yy->__thunkpos= yythunkpos17; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l16; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '3')) goto l16; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l16; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l16; +#undef yytext +#undef yyleng + } + } + l17:; yyDo(yy, yy_1_day, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "day", yy->__buf+yy->__pos)); + return 1; + l16:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "day", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_month(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "month")); + { int yypos21= yy->__pos, yythunkpos21= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l22; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '0')) goto l22; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l22; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l22; +#undef yytext +#undef yyleng + } goto l21; + l22:; yy->__pos= yypos21; yy->__thunkpos= yythunkpos21; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l20; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '1')) goto l20; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l20; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l20; +#undef yytext +#undef yyleng + } + } + l21:; yyDo(yy, yy_1_month, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "month", yy->__buf+yy->__pos)); + return 1; + l20:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "month", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_year(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "year")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l23; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l23; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_year, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "year", yy->__buf+yy->__pos)); + return 1; + l23:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "year", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_fulldate(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 3, 0); + yyprintf((stderr, "%s\n", "fulldate")); if (!yy_year(yy)) goto l24; yyDo(yy, yySet, -3, 0); if (!yymatchChar(yy, '-')) goto l24; if (!yy_month(yy)) goto l24; yyDo(yy, yySet, -2, 0); if (!yymatchChar(yy, '-')) goto l24; if (!yy_day(yy)) goto l24; yyDo(yy, yySet, -1, 0); yyDo(yy, yy_1_fulldate, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "fulldate", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 3, 0); + return 1; + l24:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "fulldate", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_rfc3339(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "rfc3339")); if (!yy_fulldate(yy)) goto l25; if (!yymatchChar(yy, 'T')) goto l25; if (!yy_fulltime(yy)) goto l25; + yyprintf((stderr, " ok %s @ %s\n", "rfc3339", yy->__buf+yy->__pos)); + return 1; + l25:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "rfc3339", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_ts_quoted(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "ts_quoted")); + { int yypos27= yy->__pos, yythunkpos27= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l28; if (!yy_rfc3339(yy)) goto l28; if (!yymatchChar(yy, '"')) goto l28; goto l27; + l28:; yy->__pos= yypos27; yy->__thunkpos= yythunkpos27; if (!yymatchChar(yy, '\'')) goto l26; if (!yy_rfc3339(yy)) goto l26; if (!yymatchChar(yy, '\'')) goto l26; + } + l27:; + yyprintf((stderr, " ok %s @ %s\n", "ts_quoted", yy->__buf+yy->__pos)); + return 1; + l26:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "ts_quoted", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_zero_to_255(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "zero_to_255")); + { int yypos30= yy->__pos, yythunkpos30= yy->__thunkpos; if (!yymatchChar(yy, '2')) goto l31; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l31; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l31; goto l30; + l31:; yy->__pos= yypos30; yy->__thunkpos= yythunkpos30; if (!yymatchChar(yy, '1')) goto l32; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l32; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l32; goto l30; + l32:; yy->__pos= yypos30; yy->__thunkpos= yythunkpos30; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l33; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l33; goto l30; + l33:; yy->__pos= yypos30; yy->__thunkpos= yythunkpos30; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l29; + } + l30:; + yyprintf((stderr, " ok %s @ %s\n", "zero_to_255", yy->__buf+yy->__pos)); + return 1; + l29:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "zero_to_255", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_index(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "index")); if (!yymatchChar(yy, '[')) goto l34; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l34; +#undef yytext +#undef yyleng + } if (!yy_zero_to_255(yy)) goto l34; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l34; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, ']')) goto l34; yyDo(yy, yy_1_index, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "index", yy->__buf+yy->__pos)); + return 1; + l34:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "index", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_nil(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "nil")); if (!yymatchString(yy, "NIL")) goto l35; yyDo(yy, yy_1_nil, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l35; + yyprintf((stderr, " ok %s @ %s\n", "nil", yy->__buf+yy->__pos)); + return 1; + l35:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "nil", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_fields(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 2, 0); + yyprintf((stderr, "%s\n", "fields")); if (!yymatchString(yy, "Fields[")) goto l36; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l36; +#undef yytext +#undef yyleng + } + l37:; + { int yypos38= yy->__pos, yythunkpos38= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\377\377\377\377\377\377\377\377\377\377\377\337\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377")) goto l38; goto l37; + l38:; yy->__pos= yypos38; yy->__thunkpos= yythunkpos38; + } yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l36; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, ']')) goto l36; yyDo(yy, yy_1_fields, yy->__begin, yy->__end); + { int yypos39= yy->__pos, yythunkpos39= yy->__thunkpos; if (!yy_index(yy)) goto l39; yyDo(yy, yySet, -2, 0); goto l40; + l39:; yy->__pos= yypos39; yy->__thunkpos= yythunkpos39; + } + l40:; yyDo(yy, yy_2_fields, yy->__begin, yy->__end); + { int yypos41= yy->__pos, yythunkpos41= yy->__thunkpos; if (!yy_index(yy)) goto l41; yyDo(yy, yySet, -1, 0); goto l42; + l41:; yy->__pos= yypos41; yy->__thunkpos= yythunkpos41; + } + l42:; yyDo(yy, yy_3_fields, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "fields", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 2, 0); + return 1; + l36:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "fields", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_exponent(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "exponent")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\000\000\040\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l43; + { int yypos44= yy->__pos, yythunkpos44= yy->__thunkpos; if (!yy_sign(yy)) goto l44; goto l45; + l44:; yy->__pos= yypos44; yy->__thunkpos= yythunkpos44; + } + l45:; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l43; + l46:; + { int yypos47= yy->__pos, yythunkpos47= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l47; goto l46; + l47:; yy->__pos= yypos47; yy->__thunkpos= yythunkpos47; + } + yyprintf((stderr, " ok %s @ %s\n", "exponent", yy->__buf+yy->__pos)); + return 1; + l43:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "exponent", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_decimal(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "decimal")); if (!yymatchChar(yy, '.')) goto l48; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l48; + l49:; + { int yypos50= yy->__pos, yythunkpos50= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l50; goto l49; + l50:; yy->__pos= yypos50; yy->__thunkpos= yythunkpos50; + } + yyprintf((stderr, " ok %s @ %s\n", "decimal", yy->__buf+yy->__pos)); + return 1; + l48:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "decimal", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_number(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "number")); + { int yypos52= yy->__pos, yythunkpos52= yy->__thunkpos; if (!yymatchChar(yy, '0')) goto l53; goto l52; + l53:; yy->__pos= yypos52; yy->__thunkpos= yythunkpos52; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l51; + l54:; + { int yypos55= yy->__pos, yythunkpos55= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l55; goto l54; + l55:; yy->__pos= yypos55; yy->__thunkpos= yythunkpos55; + } + } + l52:; + yyprintf((stderr, " ok %s @ %s\n", "number", yy->__buf+yy->__pos)); + return 1; + l51:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "number", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_sign(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "sign")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\050\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l56; + yyprintf((stderr, " ok %s @ %s\n", "sign", yy->__buf+yy->__pos)); + return 1; + l56:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "sign", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_numeric_value(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "numeric_value")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l57; +#undef yytext +#undef yyleng + } + { int yypos58= yy->__pos, yythunkpos58= yy->__thunkpos; if (!yy_sign(yy)) goto l58; goto l59; + l58:; yy->__pos= yypos58; yy->__thunkpos= yythunkpos58; + } + l59:; if (!yy_number(yy)) goto l57; + { int yypos60= yy->__pos, yythunkpos60= yy->__thunkpos; if (!yy_decimal(yy)) goto l60; goto l61; + l60:; yy->__pos= yypos60; yy->__thunkpos= yythunkpos60; + } + l61:; + { int yypos62= yy->__pos, yythunkpos62= yy->__thunkpos; if (!yy_exponent(yy)) goto l62; goto l63; + l62:; yy->__pos= yypos62; yy->__thunkpos= yythunkpos62; + } + l63:; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l57; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_numeric_value, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "numeric_value", yy->__buf+yy->__pos)); + return 1; + l57:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "numeric_value", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_numeric_headers(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "numeric_headers")); + { int yypos65= yy->__pos, yythunkpos65= yy->__thunkpos; if (!yymatchString(yy, "Severity")) goto l66; yyDo(yy, yy_1_numeric_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l66; goto l65; + l66:; yy->__pos= yypos65; yy->__thunkpos= yythunkpos65; if (!yymatchString(yy, "Pid")) goto l64; yyDo(yy, yy_2_numeric_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l64; + } + l65:; + yyprintf((stderr, " ok %s @ %s\n", "numeric_headers", yy->__buf+yy->__pos)); + return 1; + l64:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "numeric_headers", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_match(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_match")); + { int yypos68= yy->__pos, yythunkpos68= yy->__thunkpos; if (!yy_op_seq(yy)) goto l69; goto l68; + l69:; yy->__pos= yypos68; yy->__thunkpos= yythunkpos68; if (!yy_op_sne(yy)) goto l67; + } + l68:; + yyprintf((stderr, " ok %s @ %s\n", "string_match", yy->__buf+yy->__pos)); + return 1; + l67:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_match", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_value(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_value")); + { int yypos71= yy->__pos, yythunkpos71= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l72; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l72; +#undef yytext +#undef yyleng + } + l73:; + { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; + { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l76; goto l75; + l76:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; + { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l77; goto l74; + l77:; yy->__pos= yypos77; yy->__thunkpos= yythunkpos77; + } if (!yymatchDot(yy)) goto l74; + } + l75:; goto l73; + l74:; yy->__pos= yypos74; yy->__thunkpos= yythunkpos74; + } yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l72; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '"')) goto l72; goto l71; + l72:; yy->__pos= yypos71; yy->__thunkpos= yythunkpos71; if (!yymatchChar(yy, '\'')) goto l70; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l70; +#undef yytext +#undef yyleng + } + l78:; + { int yypos79= yy->__pos, yythunkpos79= yy->__thunkpos; + { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l81; goto l80; + l81:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; + { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l82; goto l79; + l82:; yy->__pos= yypos82; yy->__thunkpos= yythunkpos82; + } if (!yymatchDot(yy)) goto l79; + } + l80:; goto l78; + l79:; yy->__pos= yypos79; yy->__thunkpos= yythunkpos79; + } yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l70; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '\'')) goto l70; + } + l71:; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(check_string_len(yytext))) goto l70; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_string_value, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "string_value", yy->__buf+yy->__pos)); + return 1; + l70:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_value", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_headers(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_headers")); + { int yypos84= yy->__pos, yythunkpos84= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l85; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l85; goto l84; + l85:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Logger")) goto l86; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l86; goto l84; + l86:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Hostname")) goto l87; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l87; goto l84; + l87:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "EnvVersion")) goto l88; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l88; goto l84; + l88:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Payload")) goto l89; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l89; goto l84; + l89:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Uuid")) goto l83; yyDo(yy, yy_6_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l83; + } + l84:; + yyprintf((stderr, " ok %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); + return 1; + l83:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_boolean(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "boolean")); + { int yypos91= yy->__pos, yythunkpos91= yy->__thunkpos; if (!yy_true(yy)) goto l92; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l91; + l92:; yy->__pos= yypos91; yy->__thunkpos= yythunkpos91; if (!yy_false(yy)) goto l90; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); + } + l91:; + yyprintf((stderr, " ok %s @ %s\n", "boolean", yy->__buf+yy->__pos)); + return 1; + l90:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "boolean", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_false(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l93; if (!yy_sp(yy)) goto l93; + yyprintf((stderr, " ok %s @ %s\n", "false", yy->__buf+yy->__pos)); + return 1; + l93:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "false", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_true(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l94; if (!yy_sp(yy)) goto l94; + yyprintf((stderr, " ok %s @ %s\n", "true", yy->__buf+yy->__pos)); + return 1; + l94:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "true", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_relational(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "relational")); + { int yypos96= yy->__pos, yythunkpos96= yy->__thunkpos; if (!yy_op_eq(yy)) goto l97; goto l96; + l97:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_ne(yy)) goto l98; goto l96; + l98:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_gte(yy)) goto l99; goto l96; + l99:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_gt(yy)) goto l100; goto l96; + l100:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_lte(yy)) goto l101; goto l96; + l101:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_lt(yy)) goto l95; + } + l96:; + yyprintf((stderr, " ok %s @ %s\n", "relational", yy->__buf+yy->__pos)); + return 1; + l95:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "relational", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_lt(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l102; if (!yy_sp(yy)) goto l102; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); + return 1; + l102:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_lte(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l103; if (!yy_sp(yy)) goto l103; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); + return 1; + l103:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_gt(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l104; if (!yy_sp(yy)) goto l104; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); + return 1; + l104:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_gte(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); + return 1; + l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_sne(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); + return 1; + l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_seq(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); + return 1; + l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_ne(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); + return 1; + l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_eq(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l109; if (!yy_sp(yy)) goto l109; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); + return 1; + l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_boolean_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "boolean_test")); + { int yypos111= yy->__pos, yythunkpos111= yy->__thunkpos; if (!yy_true(yy)) goto l112; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l111; + l112:; yy->__pos= yypos111; yy->__thunkpos= yythunkpos111; if (!yy_false(yy)) goto l110; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); + } + l111:; + yyprintf((stderr, " ok %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); + return 1; + l110:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_ts_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "ts_test")); if (!yymatchString(yy, "Timestamp")) goto l113; if (!yy_sp(yy)) goto l113; if (!yy_relational(yy)) goto l113; if (!yy_sp(yy)) goto l113; + { int yypos114= yy->__pos, yythunkpos114= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l115; goto l114; + l115:; yy->__pos= yypos114; yy->__thunkpos= yythunkpos114; if (!yy_ts_quoted(yy)) goto l113; + } + l114:; yyDo(yy, yy_1_ts_test, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); + return 1; + l113:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_field_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l116; if (!yy_sp(yy)) goto l116; + { int yypos117= yy->__pos, yythunkpos117= yy->__thunkpos; if (!yy_relational(yy)) goto l118; if (!yy_sp(yy)) goto l118; + { int yypos119= yy->__pos, yythunkpos119= yy->__thunkpos; if (!yy_string_value(yy)) goto l120; goto l119; + l120:; yy->__pos= yypos119; yy->__thunkpos= yythunkpos119; if (!yy_numeric_value(yy)) goto l118; + } + l119:; goto l117; + l118:; yy->__pos= yypos117; yy->__thunkpos= yythunkpos117; if (!yy_string_match(yy)) goto l121; if (!yy_sp(yy)) goto l121; if (!yy_string_value(yy)) goto l121; goto l117; + l121:; yy->__pos= yypos117; yy->__thunkpos= yythunkpos117; + { int yypos122= yy->__pos, yythunkpos122= yy->__thunkpos; if (!yy_op_eq(yy)) goto l123; goto l122; + l123:; yy->__pos= yypos122; yy->__thunkpos= yythunkpos122; if (!yy_op_ne(yy)) goto l116; + } + l122:; if (!yy_sp(yy)) goto l116; + { int yypos124= yy->__pos, yythunkpos124= yy->__thunkpos; if (!yy_boolean(yy)) goto l125; goto l124; + l125:; yy->__pos= yypos124; yy->__thunkpos= yythunkpos124; if (!yy_nil(yy)) goto l116; + } + l124:; + } + l117:; + yyprintf((stderr, " ok %s @ %s\n", "field_test", yy->__buf+yy->__pos)); + return 1; + l116:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "field_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_numeric_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "numeric_test")); if (!yy_numeric_headers(yy)) goto l126; if (!yy_sp(yy)) goto l126; if (!yy_relational(yy)) goto l126; if (!yy_sp(yy)) goto l126; if (!yy_numeric_value(yy)) goto l126; + yyprintf((stderr, " ok %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); + return 1; + l126:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_test")); if (!yy_string_headers(yy)) goto l127; if (!yy_sp(yy)) goto l127; + { int yypos128= yy->__pos, yythunkpos128= yy->__thunkpos; if (!yy_relational(yy)) goto l129; if (!yy_sp(yy)) goto l129; if (!yy_string_value(yy)) goto l129; goto l128; + l129:; yy->__pos= yypos128; yy->__thunkpos= yythunkpos128; if (!yy_string_match(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_string_value(yy)) goto l127; + } + l128:; + yyprintf((stderr, " ok %s @ %s\n", "string_test", yy->__buf+yy->__pos)); + return 1; + l127:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_close(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "close")); if (!yymatchChar(yy, ')')) goto l130; yyDo(yy, yy_1_close, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l130; + yyprintf((stderr, " ok %s @ %s\n", "close", yy->__buf+yy->__pos)); + return 1; + l130:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "close", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_open(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "open")); if (!yymatchChar(yy, '(')) goto l131; yyDo(yy, yy_1_open, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l131; + yyprintf((stderr, " ok %s @ %s\n", "open", yy->__buf+yy->__pos)); + return 1; + l131:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "open", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "test")); + { int yypos133= yy->__pos, yythunkpos133= yy->__thunkpos; if (!yy_string_test(yy)) goto l134; goto l133; + l134:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_numeric_test(yy)) goto l135; goto l133; + l135:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_field_test(yy)) goto l136; goto l133; + l136:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_ts_test(yy)) goto l137; goto l133; + l137:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_boolean_test(yy)) goto l132; + } + l133:; yyDo(yy, yy_1_test, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l132; + yyprintf((stderr, " ok %s @ %s\n", "test", yy->__buf+yy->__pos)); + return 1; + l132:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_and(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "and")); if (!yymatchString(yy, "&&")) goto l138; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(cond_cnt(&yy->ctx))) goto l138; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_and, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l138; + yyprintf((stderr, " ok %s @ %s\n", "and", yy->__buf+yy->__pos)); + return 1; + l138:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "and", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_expr(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "expr")); + { int yypos140= yy->__pos, yythunkpos140= yy->__thunkpos; if (!yy_test(yy)) goto l141; goto l140; + l141:; yy->__pos= yypos140; yy->__thunkpos= yythunkpos140; if (!yy_open(yy)) goto l139; if (!yy_ored(yy)) goto l139; if (!yy_close(yy)) goto l139; + } + l140:; + yyprintf((stderr, " ok %s @ %s\n", "expr", yy->__buf+yy->__pos)); + return 1; + l139:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "expr", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_or(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "or")); if (!yymatchString(yy, "||")) goto l142; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(cond_cnt(&yy->ctx))) goto l142; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_or, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l142; + yyprintf((stderr, " ok %s @ %s\n", "or", yy->__buf+yy->__pos)); + return 1; + l142:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "or", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_anded(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "anded")); if (!yy_expr(yy)) goto l143; + l144:; + { int yypos145= yy->__pos, yythunkpos145= yy->__thunkpos; if (!yy_and(yy)) goto l145; if (!yy_expr(yy)) goto l145; goto l144; + l145:; yy->__pos= yypos145; yy->__thunkpos= yythunkpos145; + } if (!yy_sp(yy)) goto l143; + yyprintf((stderr, " ok %s @ %s\n", "anded", yy->__buf+yy->__pos)); + return 1; + l143:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "anded", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_eol(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "eol")); + { int yypos147= yy->__pos, yythunkpos147= yy->__thunkpos; if (!yymatchDot(yy)) goto l147; goto l146; + l147:; yy->__pos= yypos147; yy->__thunkpos= yythunkpos147; + } + yyprintf((stderr, " ok %s @ %s\n", "eol", yy->__buf+yy->__pos)); + return 1; + l146:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "eol", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_ored(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "ored")); if (!yy_anded(yy)) goto l148; + l149:; + { int yypos150= yy->__pos, yythunkpos150= yy->__thunkpos; if (!yy_or(yy)) goto l150; if (!yy_anded(yy)) goto l150; goto l149; + l150:; yy->__pos= yypos150; yy->__thunkpos= yythunkpos150; + } if (!yy_sp(yy)) goto l148; + yyprintf((stderr, " ok %s @ %s\n", "ored", yy->__buf+yy->__pos)); + return 1; + l148:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "ored", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_sp(yycontext *yy) +{ + yyprintf((stderr, "%s\n", "sp")); + l152:; + { int yypos153= yy->__pos, yythunkpos153= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\002\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l153; goto l152; + l153:; yy->__pos= yypos153; yy->__thunkpos= yythunkpos153; + } + yyprintf((stderr, " ok %s @ %s\n", "sp", yy->__buf+yy->__pos)); + return 1; +} +YY_RULE(int) yy_match(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "match")); if (!yy_sp(yy)) goto l154; if (!yy_ored(yy)) goto l154; if (!yy_eol(yy)) goto l154; yyDo(yy, yy_1_match, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "match", yy->__buf+yy->__pos)); + return 1; + l154:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "match", yy->__buf+yy->__pos)); + return 0; +} + +#ifndef YY_PART + +typedef int (*yyrule)(yycontext *yy); + +YY_PARSE(int) YYPARSEFROM(YY_CTX_PARAM_ yyrule yystart) +{ + int yyok; + if (!yyctx->__buflen) + { + yyctx->__buflen= YY_BUFFER_SIZE; + yyctx->__buf= (char *)YY_MALLOC(yyctx, yyctx->__buflen); + yyctx->__textlen= YY_BUFFER_SIZE; + yyctx->__text= (char *)YY_MALLOC(yyctx, yyctx->__textlen); + yyctx->__thunkslen= YY_STACK_SIZE; + yyctx->__thunks= (yythunk *)YY_MALLOC(yyctx, sizeof(yythunk) * yyctx->__thunkslen); + yyctx->__valslen= YY_STACK_SIZE; + yyctx->__vals= (YYSTYPE *)YY_MALLOC(yyctx, sizeof(YYSTYPE) * yyctx->__valslen); + memset(yyctx->__vals, 0, sizeof(YYSTYPE) * yyctx->__valslen); + yyctx->__begin= yyctx->__end= yyctx->__pos= yyctx->__limit= yyctx->__thunkpos= 0; + } + yyctx->__begin= yyctx->__end= yyctx->__pos; + yyctx->__thunkpos= 0; + yyctx->__val= yyctx->__vals; + yyok= yystart(yyctx); + if (yyok) yyDone(yyctx); + yyCommit(yyctx); + return yyok; +} + +YY_PARSE(int) YYPARSE(YY_CTX_PARAM) +{ + return YYPARSEFROM(YY_CTX_ARG_ yy_match); +} + +YY_PARSE(yycontext *) YYRELEASE(yycontext *yyctx) +{ + if (yyctx->__buflen) + { + yyctx->__buflen= 0; + YY_FREE(yyctx, yyctx->__buf); + YY_FREE(yyctx, yyctx->__text); + YY_FREE(yyctx, yyctx->__thunks); + YY_FREE(yyctx, yyctx->__vals); + } + return yyctx; +} + +#endif + + + +lsb_message_matcher* lsb_create_message_matcher(const char *exp) +{ + if (!exp) { + return NULL; + } + + lsb_message_matcher *mm = NULL; + yycontext yy; + memset(&yy, 0, sizeof(yy)); + yy.ctx.is.s = exp; + yy.ctx.is.size = strlen(exp); + int ret = yyparse(&yy); + if (!ret) { + free(yy.ctx.out.a); + goto cleanup; + } + + // reverse the order so the root node will be first + match_node *s = yy.ctx.out.a; + match_node *e = yy.ctx.out.a + yy.ctx.out.pos; + for (--e; s < e; ++s, --e) { + move_match_node(&yy.ctx.mn, s); + move_match_node(s, e); + move_match_node(e, &yy.ctx.mn); + } + + mm = malloc(sizeof(lsb_message_matcher)); + if (!mm) { + goto cleanup; + } + mm->nodes = yy.ctx.out.a; + mm->nodes[0].size = yy.ctx.out.pos; + + // turn the postfix stack into an executable tree + match_node **stack = calloc(sizeof(match_node *) * mm->nodes[0].size, 1); + if (!stack) { + free(mm); + mm = NULL; + goto cleanup; + } + + int top = 0; + for (int i = mm->nodes[0].size - 1; i >= 0; --i) { + if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { + stack[top++] = &mm->nodes[i]; + } else { + mm->nodes[i].ai = stack[--top] - mm->nodes; + mm->nodes[i].fi = stack[--top] - mm->nodes; + stack[top++] = &mm->nodes[i]; + } + } + free(stack); + +cleanup: + free(yy.ctx.ops.a); + yyrelease(&yy); + return mm; +} + diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg new file mode 100644 index 0000000..2f5eb6a --- /dev/null +++ b/src/util/heka_message_matcher_parser.leg @@ -0,0 +1,418 @@ +%{ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight message matcher parser @file */ + +#include +#include +#include +#include +#include + +#include "heka_message_matcher_impl.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" + +#ifndef _MSC_VER +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-function" +#endif + +typedef struct match_node_array { + match_node *a; + int pos; + int size; +} match_node_array; + + +typedef struct input_string { + const char *s; + size_t pos; + size_t size; +} input_string; + + +typedef struct context { + match_node_array out; + match_node_array ops; + match_node mn; + struct tm tms; + int cond_cnt; + input_string is; +} context; + + +#define YY_PARSE(T) static T +#define YY_CTX_LOCAL 1 +#define YY_CTX_MEMBERS \ + context ctx; + +#define YY_INPUT(yy, buf, result, max_size) \ +{ \ + input_string *is = &yy->ctx.is; \ + result = is->size - is->pos; \ + if (result > 0) { \ + if (max_size >= result) { \ + memcpy(buf, &is->s[is->pos], result); \ + is->pos += result; \ + } else { \ + memcpy(buf, &is->s[is->pos], max_size); \ + is->pos += max_size; \ + result = max_size; \ + } \ + } \ +} + + +static void init_match_node(match_node *mn) +{ + memset(mn, 0, sizeof(match_node)); +} + + +static void move_match_node(match_node *dest, match_node *src) +{ + memcpy(dest, src, sizeof(match_node)); + init_match_node(src); // dest now owns the memory, wipe the pointers +} + + +static void realloc_mna(match_node_array *mna) +{ + size_t bytes = sizeof(match_node) * ++mna->size; + match_node *tmp = realloc(mna->a, bytes); + if (tmp) { + mna->a = tmp; + init_match_node(&mna->a[mna->size - 1]); + } else { + fprintf(stderr, "realloc failed\n"); + exit(1); + } +} + + +static void push_output(context *ctx, match_node *mn) +{ + if (!ctx->out.a || ctx->out.pos == ctx->out.size) { + realloc_mna(&ctx->out); + } + move_match_node(&ctx->out.a[ctx->out.pos++], mn); +} + + +static void push_op(context *ctx, match_operation op) +{ + if (!ctx->ops.a) { + realloc_mna(&ctx->ops); + ctx->ops.a[ctx->ops.pos++].op = op; + return; + } + + if (op == OP_OPEN || op > ctx->ops.a[ctx->ops.pos - 1].op) { + if (ctx->ops.pos == ctx->ops.size) { + realloc_mna(&ctx->ops); + } + ctx->ops.a[ctx->ops.pos++].op = op; + } else { + push_output(ctx, &ctx->ops.a[ctx->ops.pos - 1]); + ctx->ops.a[ctx->ops.pos - 1].op = op; + } +} + + +static void pop_to_paren(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) break; + push_output(ctx, op); + } +} + + +static void pop_all_ops(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) continue; + push_output(ctx, op); + } +} + + +static void update_date(context *ctx, int year, int mon, int day) +{ + ctx->tms.tm_isdst = -1; + ctx->tms.tm_year = year - 1900; + ctx->tms.tm_mon = mon - 1; + ctx->tms.tm_mday = day; +} + + +static void update_time(context *ctx, int hour, int minute, int sec) +{ + ctx->tms.tm_hour = hour; + ctx->tms.tm_min = minute; + ctx->tms.tm_sec = sec; +} + + +static void update_offset(context *ctx, char sign, int hour, int minute) +{ + ctx->mn.value.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); +} + + +static void set_field(context *ctx, char *name) +{ + ctx->mn.id = LSB_PB_FIELDS; + ctx->mn.variable_len = strlen(name); + ctx->mn.variable = malloc(ctx->mn.variable_len + 1); + if (!ctx->mn.variable) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.variable, name, ctx->mn.variable_len + 1); +} + + +static void set_timestamp(context *ctx) +{ + ctx->mn.id = LSB_PB_TIMESTAMP; + ctx->mn.value_type = TYPE_NUMERIC; + if (ctx->tms.tm_isdst == -1) { + ctx->mn.value.d += mktime(&ctx->tms); + ctx->mn.value.d *= 1e9; + } + memset(&ctx->tms, 0, sizeof(struct tm)); +} + + +static void set_numeric_value(context *ctx, char *s) +{ + ctx->mn.value_type = TYPE_NUMERIC; + ctx->mn.value.d = strtod(s, NULL); +} + + +static void set_string_value(context *ctx, char *s) +{ + ctx->mn.value_type = TYPE_STRING; + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + ++i; + } + s[j] = s[i]; + } + s[j] = 0; + + ctx->mn.value_len = j; + ctx->mn.value.s = malloc(j + 1); + if (!ctx->mn.value.s) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.value.s, s, j + 1); +} + + +static bool check_string_len(char *s) +{ + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + ++i; + } + } + return (j > UCHAR_MAX) ? false : true; +} + + +static void set_boolean_value(context *ctx, bool b) +{ + ctx->mn.value_type = TYPE_BOOLEAN; + ctx->mn.value.d = b; +} + + +static int cond_cnt(context *ctx) +{ + return (++ctx->cond_cnt * 2 + 1 > UCHAR_MAX) ? 0 : 1; +} + +%} + +match = sp ored eol {pop_all_ops(&yy->ctx)} +ored = anded (or anded)* sp +anded = expr (and expr)* sp +expr = test | open ored close +test = ( string_test + | numeric_test + | field_test + | ts_test + | boolean_test + ) {push_output(&yy->ctx, &yy->ctx.mn)} sp + +op_eq = "==" sp {yy->ctx.mn.op = OP_EQ} +op_ne = "!=" sp {yy->ctx.mn.op = OP_NE} +op_seq = "=~" sp {yy->ctx.mn.op = OP_RE} +op_sne = "!~" sp {yy->ctx.mn.op = OP_NRE} +op_gte = ">=" sp {yy->ctx.mn.op = OP_GTE} +op_gt = ">" sp {yy->ctx.mn.op = OP_GT} +op_lte = "<=" sp {yy->ctx.mn.op = OP_LTE} +op_lt = "<" sp {yy->ctx.mn.op = OP_LT} + +relational = op_eq + | op_ne + | op_gte + | op_gt + | op_lte + | op_lt + +boolean_test = true {yy->ctx.mn.op = OP_TRUE} + | false {yy->ctx.mn.op = OP_FALSE} +boolean = true {set_boolean_value(&yy->ctx, 1)} + | false {set_boolean_value(&yy->ctx, 0)} + +and = "&&" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_AND)} sp +or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp +open = '(' {push_op(&yy->ctx, OP_OPEN)} sp +close = ')' {pop_to_paren(&yy->ctx)} sp + +string_test = string_headers sp (relational sp string_value | string_match sp string_value) + +string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} sp + | "Logger" {yy->ctx.mn.id = LSB_PB_LOGGER} sp + | "Hostname" {yy->ctx.mn.id = LSB_PB_HOSTNAME} sp + | "EnvVersion" {yy->ctx.mn.id = LSB_PB_ENV_VERSION} sp + | "Payload" {yy->ctx.mn.id = LSB_PB_PAYLOAD} sp + | "Uuid" {yy->ctx.mn.id = LSB_PB_UUID} sp + +string_value = ( '"' < ('\\\"' | (!'"' .))* > '"' + | "'" < ("\\\'" | (!"'" .))* > "'" + ) &{check_string_len(yytext)} {set_string_value(&yy->ctx, yytext)} + +string_match = op_seq | op_sne + +numeric_headers = "Severity" {yy->ctx.mn.id = LSB_PB_SEVERITY} sp + | "Pid" {yy->ctx.mn.id = LSB_PB_PID} sp + +numeric_test = numeric_headers sp relational sp numeric_value +numeric_value = < sign? number decimal? exponent? > {set_numeric_value(&yy->ctx, yytext)} +sign = [-+] +number = "0" + | [1-9] [0-9]* +decimal = "." [0-9]+ +exponent = [eE] sign? [0-9]+ + +field_test = fields sp ((relational sp (string_value | numeric_value)) + | string_match sp string_value + | (op_eq | op_ne) sp (boolean | nil)) +fields = "Fields[" < [^\]]* > "]" {set_field(&yy->ctx, yytext)} f:index? {yy->ctx.mn.fi = f} a:index? {yy->ctx.mn.ai = a} +index = "[" < zero_to_255 > "]" {$$ = atoi(yytext)} +zero_to_255 = "2" [0-5] [0-5] + | "1" [0-9] [0-9] + | [1-9] [0-9] + | [0-9] + +ts_test = ("Timestamp" sp relational sp (numeric_value | ts_quoted)) {set_timestamp(&yy->ctx)} +ts_quoted = '"' rfc3339 '"' | "'" rfc3339 "'" +fulldate = (y:year "-" m:month "-" d:day) {update_date(&yy->ctx, y, m, d)} +year = < [0-9] [0-9] [0-9] [0-9] > {$$ = atoi(yytext)} +month = ( < "0" [1-9] > + | < "1" [0-2] > + ) {$$ = atoi(yytext)} +day = ( + < [1-2] [0-9] > + | < "0" [1-9] > + | < "3" [0-1] > + ) {$$ = atoi(yytext)} + +rfc3339 = fulldate "T" fulltime +fulltime = partialtime timeoffset +partialtime = h:hour ":" m:minute ":" s:second second_frac? {update_time(&yy->ctx, h, m, s)} +timeoffset = "Z" {update_offset(&yy->ctx, '+', 0, 0)} + | < sign > h:hour m:minute {update_offset(&yy->ctx, yytext[0], h, m)} +hour = ( + < [0-1] [0-9] > + | < "2" [0-3] > + ) {$$ = atoi(yytext)} +minute = < [0-5] [0-9] > {$$ = atoi(yytext)} +second = ( + < [0-5] [0-9] > + | < "60" > + ) {$$ = atoi(yytext)} +second_frac = < decimal > {yy->ctx.mn.value.d += strtod(yytext, NULL)} + +nil = "NIL" {yy->ctx.mn.value_type = TYPE_NIL;} sp +true = "TRUE" sp +false = "FALSE" sp +sp = [ \t]* +eol = !. + +%% + + +lsb_message_matcher* lsb_create_message_matcher(const char *exp) +{ + if (!exp) { + return NULL; + } + + lsb_message_matcher *mm = NULL; + yycontext yy; + memset(&yy, 0, sizeof(yy)); + yy.ctx.is.s = exp; + yy.ctx.is.size = strlen(exp); + int ret = yyparse(&yy); + if (!ret) { + free(yy.ctx.out.a); + goto cleanup; + } + + // reverse the order so the root node will be first + match_node *s = yy.ctx.out.a; + match_node *e = yy.ctx.out.a + yy.ctx.out.pos; + for (--e; s < e; ++s, --e) { + move_match_node(&yy.ctx.mn, s); + move_match_node(s, e); + move_match_node(e, &yy.ctx.mn); + } + + mm = malloc(sizeof(lsb_message_matcher)); + if (!mm) { + goto cleanup; + } + mm->nodes = yy.ctx.out.a; + mm->nodes[0].size = yy.ctx.out.pos; + + // turn the postfix stack into an executable tree + match_node **stack = calloc(sizeof(match_node *) * mm->nodes[0].size, 1); + if (!stack) { + free(mm); + mm = NULL; + goto cleanup; + } + + int top = 0; + for (int i = mm->nodes[0].size - 1; i >= 0; --i) { + if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { + stack[top++] = &mm->nodes[i]; + } else { + mm->nodes[i].ai = stack[--top] - mm->nodes; + mm->nodes[i].fi = stack[--top] - mm->nodes; + stack[top++] = &mm->nodes[i]; + } + } + free(stack); + +cleanup: + free(yy.ctx.ops.a); + yyrelease(&yy); + return mm; +} diff --git a/src/util/protobuf.c b/src/util/protobuf.c index 94a56fa..d510586 100644 --- a/src/util/protobuf.c +++ b/src/util/protobuf.c @@ -23,7 +23,7 @@ const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype) lsb_err_value lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, - unsigned char wiretype) + unsigned char wiretype) { lsb_err_value ret = lsb_expand_output_buffer(ob, 1); if (!ret) { @@ -35,7 +35,7 @@ lsb_err_value lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi) { - if (!p || !e) { + if (!p || !e || !vi) { return NULL; } diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt index 968686c..a50d423 100644 --- a/src/util/test/CMakeLists.txt +++ b/src/util/test/CMakeLists.txt @@ -30,6 +30,10 @@ add_executable(test_heka_message test_heka_message.c) target_link_libraries(test_heka_message luasandboxutil) add_test(NAME test_heka_message COMMAND test_heka_message) +add_executable(test_heka_message_matcher test_heka_message_matcher.c) +target_link_libraries(test_heka_message_matcher luasandboxutil) +add_test(NAME test_heka_message_matcher COMMAND test_heka_message_matcher) + if(WIN32) set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src/util") set_tests_properties(test_input_buffer PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) diff --git a/src/heka/test/test_message_matcher.c b/src/util/test/test_heka_message_matcher.c similarity index 74% rename from src/heka/test/test_message_matcher.c rename to src/util/test/test_heka_message_matcher.c index 187a7e9..926a8db 100644 --- a/src/heka/test/test_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -12,10 +12,9 @@ #include #include -#include "luasandbox/heka/message_matcher.h" +#include "../../test/mu_test.h" #include "luasandbox/util/heka_message.h" - -#include "test.h" +#include "luasandbox/util/heka_message_matcher.h" // {"Logger":"GoSpec","Uuid":"xxx","Pid":32157,"Severity":6,"EnvVersion":"0.8","Fields":[{""value":["bar"],"name":"foo","value_type":0},{"value":[64],"name":"number","value_type":2},{"value":["data"],"name":"bytes","value_type":1},{"value":[999,1024],"name":"int","value_type":2},{"value":[99.9],"name":"double","value_type":3},{"value":[true],"name":"bool","value_type":4},{"value":["alternate"],"name":"foo","value_type":0},{"value":["name=test;type=web;"],"name":"Payload","value_type":0},{"representation":"date-time","value":["Mon Jan 02 15:04:05 -0700 2006"],"name":"Timestamp","value_type":0},{"value":[0],"name":"zero","value_type":2},{"value":["43"],"name":"string","value_type":0}],"Payload":"Test Payload","Timestamp":1.428773426113e+18,"Hostname":"trink-x230","Type":"TEST"} @@ -31,26 +30,39 @@ static char* test_stub() static char* test_api_assertion() { - mu_assert(NULL == lsb_create_message_matcher(NULL, "TRUE"), "not null"); - lsb_destroy_message_match_builder(NULL); + mu_assert(NULL == lsb_create_message_matcher(NULL), "not null"); lsb_destroy_message_matcher(NULL); - - lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); - mu_assert(mmb, "failed to create mmb"); - mu_assert(NULL == lsb_create_message_matcher(mmb, NULL), "not null"); - lsb_destroy_message_match_builder(mmb); return NULL; } +#define T128 "Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.'" + +#define S255 "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" \ +"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" \ +"0123456789012345678901234567890123456789012345678901234" static char* test_true_matcher() { - char* tests[] = { + char *tests[] = { "TRUE" , "Timestamp > 1.428773420000e+18" , "Timestamp < 1.428773426999e18" , "Timestamp == 1428773426113040228" - , "Timestamp > '2015-04-11T17:30:26Z'" + , "Timestamp > '2015-04-11T17:30:26Z'" // 1428773426000000000 + , "Timestamp > '2015-04-11T17:30:26.112Z'" + , "Timestamp < '2015-04-11T17:30:26.114Z'" , "(Severity == 7 || Payload == 'Test Payload') && Type == 'TEST'" , "EnvVersion == \"0.8\"" , "EnvVersion == '0.8'" @@ -105,20 +117,20 @@ static char* test_true_matcher() , "Type =~ 'ST$'" , "Type !~ '^te'" , "Type !~ 'st$'" + , "Fields[foo][255] == NIL" + , "Fields[foo][0][255] == NIL" + , T128 , NULL }; lsb_heka_message m; lsb_init_heka_message(&m, 16); mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); - lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); - mu_assert(mmb, "failed to create mmb"); for (int i = 0; tests[i]; ++i) { - lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); mu_assert(mm, "failed to create the matcher %s", tests[i]); mu_assert(lsb_eval_message_matcher(mm, &m), "%s", tests[i]); lsb_destroy_message_matcher(mm); } - lsb_destroy_message_match_builder(mmb); lsb_free_heka_message(&m); return NULL; } @@ -126,7 +138,7 @@ static char* test_true_matcher() static char* test_false_matcher() { - char* tests[] = { + char *tests[] = { "FALSE" , "Timestamp == 1e9" , "Timestamp > '2015-04-11T17:30:27Z'" @@ -166,20 +178,18 @@ static char* test_false_matcher() , "Type !~ '^TE'" , "Type !~ 'ST$'" , "Logger =~ '.' && Type =~ '^anything'" + , "Type == '" S255 "'" , NULL }; lsb_heka_message m; lsb_init_heka_message(&m, 8); mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); - lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); - mu_assert(mmb, "failed to create mmb"); for (int i = 0; tests[i]; ++i) { - lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); mu_assert(mm, "failed to create the matcher %s", tests[i]); mu_assert(lsb_eval_message_matcher(mm, &m) == false, "%s", tests[i]); lsb_destroy_message_matcher(mm); } - lsb_destroy_message_match_builder(mmb); lsb_free_heka_message(&m); return NULL; } @@ -187,7 +197,7 @@ static char* test_false_matcher() static char* test_malformed_matcher() { - char* tests[] = { + char *tests[] = { "" , "bogus" , "Type = 'test'" // invalid operator @@ -211,48 +221,44 @@ static char* test_malformed_matcher() , "Fields[test] > NIL" // existence check only works with equals and not equals , "TRUE FALSE" // missing operator , "Timestamp == '20150411T173026'" // non rfc3339 timestamp + , T128 " && Type =~ '.'" // too many tests + , "Type == '" S255 "5'" // string too long + , "Fields[test][256] == 1" // field index out of bounds + , "Fields[test][0][256] == 1" // array index out of bounds , NULL }; lsb_heka_message m; lsb_init_heka_message(&m, 8); mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); - lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); - mu_assert(mmb, "failed to create mmb"); for (int i = 0; tests[i]; ++i) { - lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); mu_assert(mm == NULL, "created malformed matcher"); } - lsb_destroy_message_match_builder(mmb); lsb_free_heka_message(&m); return NULL; } - static char* benchmark_matcher_create() { int iter = 100000; - const char* exp = "Type == 'TEST' && Severity == 6"; - - lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); - mu_assert(mmb, "failed to create mmb"); + const char *exp = "Type == 'TEST' && Severity == 6"; clock_t t = clock(); for (int x = 0; x < iter; ++x) { - lsb_message_matcher* mm = lsb_create_message_matcher(mmb, exp); + lsb_message_matcher *mm = lsb_create_message_matcher(exp); mu_assert(mm, "lsb_create_message_matcher failed"); lsb_destroy_message_matcher(mm); } t = clock() - t; - lsb_destroy_message_match_builder(mmb); printf("benchmark_matcher_create: %g\n", ((double)t) / CLOCKS_PER_SEC - / iter); + / iter); return NULL; } static char* benchmark_match() { int iter = 1000000; - char* tests[] = { + char *tests[] = { "Type == 'TEST' && Severity == 6" , "Fields[foo] == 'bar' && Severity == 6" , "Fields[number] == 64 && Severity == 6" @@ -266,11 +272,9 @@ static char* benchmark_match() lsb_heka_message m; lsb_init_heka_message(&m, 8); mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); - lsb_message_match_builder *mmb = lsb_create_message_match_builder(TEST_LUA_PATH, TEST_LUA_CPATH); - mu_assert(mmb, "failed to create mmb"); for (int i = 0; tests[i]; i++) { - lsb_message_matcher* mm = lsb_create_message_matcher(mmb, tests[i]); + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); mu_assert(mm, "lsb_create_message_matcher failed: %s", tests[i]); clock_t t = clock(); for (int x = 0; x < iter; ++x) { @@ -280,9 +284,8 @@ static char* benchmark_match() t = clock() - t; lsb_destroy_message_matcher(mm); printf("matcher: '%s': %g\n", tests[i], ((double)t) / CLOCKS_PER_SEC - / iter); + / iter); } - lsb_destroy_message_match_builder(mmb); lsb_free_heka_message(&m); return NULL; } @@ -304,7 +307,8 @@ static char* all_tests() int main() { - char* result = all_tests(); + lsb_set_tz(NULL); + char *result = all_tests(); if (result) { printf("%s\n", result); } else { diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c index f7a9e2c..41ed3fc 100644 --- a/src/util/test/test_protobuf.c +++ b/src/util/test/test_protobuf.c @@ -107,6 +107,10 @@ static char* test_varint() mu_assert(!lsb_pb_read_varint(ts, ts + sizeof(ts) - 1, &vi), "parsed invalid varint (too short)"); + mu_assert(!lsb_pb_read_varint(NULL, ts + sizeof(ts) - 1, &vi), "null start"); + mu_assert(!lsb_pb_read_varint(ts, NULL, &vi), "null end"); + mu_assert(!lsb_pb_read_varint(ts, ts + sizeof(ts) - 1, NULL), "null ret"); + lsb_output_buffer ob; lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); ob.pos = ob.maxsize; diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index 6ec16db..c541645 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -49,6 +49,20 @@ static char* test_lsb_read_file() } +static char* test_lsb_set_tz() +{ + mu_assert(lsb_set_tz(NULL), "set_tz failed"); + mu_assert(lsb_set_tz("America/Los_Angeles"), "set_tz failed"); + static const char too_long[] = "01234567890123456789012345678901"; +#ifdef WIN32 + mu_assert(!lsb_set_tz(too_long), "set_tz succeeded"); +#else + mu_assert(lsb_set_tz(too_long), "set_tz failed"); +#endif + return NULL; +} + + static char* benchmark_lsb_get_time() { @@ -83,6 +97,7 @@ static char* all_tests() mu_run_test(test_stub); mu_run_test(test_lsb_lp2); mu_run_test(test_lsb_read_file); + mu_run_test(test_lsb_set_tz); mu_run_test(benchmark_lsb_get_time); return NULL; diff --git a/src/util/util.c b/src/util/util.c index 90866f7..28b16b3 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -109,6 +109,26 @@ unsigned long long lsb_get_time() } +bool lsb_set_tz(const char *tz) +{ + if (!tz) { + tz = "UTC"; + } +#if _WIN32 + char s[32]; + int n = snprintf(s, sizeof(s), "TZ=%s", tz); + if (n < 0 || n >= sizeof(s) || _putenv(s) != 0) { + return false; + } +#else + if (setenv("TZ", tz, 1) != 0) { + return false; + } +#endif + return true; +} + + #ifdef HAVE_ZLIB char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len) { From 92b4725d18c0e76e004eebf4772bf7abedededb5 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 9 May 2016 10:59:25 -0700 Subject: [PATCH 142/235] Fixup the OSX build --- CMakeLists.txt | 1 + cmake/externals.cmake | 34 ++++++++++++++++++++++------------ cmake/fixup_lua_sec.cmake | 1 + src/CMakeLists.txt | 4 ---- src/heka/test/CMakeLists.txt | 2 -- src/util/test/CMakeLists.txt | 1 + src/util/test/test_protobuf.c | 4 ++-- 7 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 cmake/fixup_lua_sec.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 40059f9..2228464 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -83,4 +83,5 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake ${CMAKE_CURRENT add_subdirectory(src) add_custom_target(copy_includes COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/include) +add_dependencies(copy_includes ${LUA_PROJECT}) add_dependencies(luasandboxutil copy_includes) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index bf3bff7..026c134 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -110,6 +110,21 @@ externalproject_add( ) add_dependencies(lua_socket ${LUA_PROJECT}) +if (OPENSSL_FOUND) + if (APPLE) + set(SANDBOX_CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_PROJECT_luasec_INCLUDE=${CMAKE_SOURCE_DIR}/cmake/fixup_lua_sec.cmake) + endif() + externalproject_add( + lua_sec + GIT_REPOSITORY https://github.com/LuaDist/luasec.git + GIT_TAG 329a8789f142bbbfef4fd7ed506285a25c3c0e0e + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + INSTALL_ARGS ${INST_ARGS} + ) + add_dependencies(lua_sec ${LUA_PROJECT}) +endif() + # sandbox enhanced modules externalproject_add( @@ -119,6 +134,7 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) +add_dependencies(lua_bloom_filter luasandbox) externalproject_add( lua_circular_buffer @@ -127,6 +143,7 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) +add_dependencies(lua_circular_buffer luasandbox) externalproject_add( lua_hyperloglog @@ -135,6 +152,7 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) +add_dependencies(lua_hyperloglog luasandbox) externalproject_add( lua_cuckoo_filter @@ -143,6 +161,7 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) +add_dependencies(lua_cuckoo_filter luasandbox) externalproject_add( lua_sax @@ -151,23 +170,14 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) +add_dependencies(lua_sax luasandbox) externalproject_add( rapidjson GIT_REPOSITORY https://github.com/miloyip/rapidjson.git GIT_TAG ed7efe6289cfa1fafc354aaa8a29fcd31c1607fd + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "no configure" BUILD_COMMAND ${CMAKE_COMMAND} -E echo "no build" INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "no install" ) -include_directories("${EP_BASE}/Source/rapidjson/include") - -if (OPENSSL_FOUND) - externalproject_add( - lua_sec - GIT_REPOSITORY https://github.com/LuaDist/luasec.git - GIT_TAG 329a8789f142bbbfef4fd7ed506285a25c3c0e0e - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - INSTALL_ARGS ${INST_ARGS} - ) -endif() +include_directories(${EP_BASE}/Source/rapidjson/include) diff --git a/cmake/fixup_lua_sec.cmake b/cmake/fixup_lua_sec.cmake new file mode 100644 index 0000000..95a506b --- /dev/null +++ b/cmake/fixup_lua_sec.cmake @@ -0,0 +1 @@ +include_directories("/opt/local/include") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0e980a5..a078f3d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,10 +14,6 @@ set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) add_dependencies(luasandbox ${LUA_PROJECT}) -add_dependencies(lua_bloom_filter luasandbox) -add_dependencies(lua_circular_buffer luasandbox) -add_dependencies(lua_hyperloglog luasandbox) -add_dependencies(lua_cuckoo_filter luasandbox) if(WIN32) set(LIB_DIR bin) diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt index 3fa403c..4806638 100644 --- a/src/heka/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -21,9 +21,7 @@ endif() if(WIN32) STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) - set_tests_properties(test_message_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) elseif(APPLE) STRING(REPLACE ";" ":" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) - set_tests_properties(test_message_matcher PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt index a50d423..c247612 100644 --- a/src/util/test/CMakeLists.txt +++ b/src/util/test/CMakeLists.txt @@ -43,4 +43,5 @@ if(WIN32) set_tests_properties(test_running_stats PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) set_tests_properties(test_util PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) set_tests_properties(test_heka_message PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_heka_message_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) endif() diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c index 41ed3fc..f90bbca 100644 --- a/src/util/test/test_protobuf.c +++ b/src/util/test/test_protobuf.c @@ -26,7 +26,7 @@ static char* test_lsb_pb_read_key() int tag, wt; const char input[] = "\x0b"; const char *p = lsb_pb_read_key(input, &tag, &wt); - mu_assert(p = input + 1, "received: %p", p); + mu_assert(p = input + 1, "received: %p", (void *)p); mu_assert(tag == 1, "received: %d", tag); mu_assert(wt == 3, "received: %d", wt); p = lsb_pb_read_key(NULL, &tag, &wt); @@ -80,7 +80,7 @@ static char* test_varint() size_t len = strlen(s); len = len ? len : 1; p = lsb_pb_read_varint(s, s + len, &vi); - mu_assert(p = s + len, "received: %p", p); + mu_assert(p = s + len, "received: %p", (void *)p); mu_assert(v == vi, "expected: %lld received: %lld", v, vi); lsb_output_buffer ob; From de405d7f5935ba5089ee4a68bbe62f58ee7eb99f Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Mon, 9 May 2016 13:36:51 -0700 Subject: [PATCH 143/235] Fix the Windows build --- src/heka/rapidjson_impl.h | 6 ++++++ src/luasandbox.c | 6 ++++++ src/util/heka_message.c | 6 ++++++ src/util/heka_message_matcher_parser.c | 2 ++ src/util/heka_message_matcher_parser.leg | 2 ++ src/util/test/test_util.c | 2 +- src/util/util.c | 2 +- 7 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/heka/rapidjson_impl.h b/src/heka/rapidjson_impl.h index 6e4055c..3390d3e 100644 --- a/src/heka/rapidjson_impl.h +++ b/src/heka/rapidjson_impl.h @@ -18,6 +18,12 @@ extern "C" #include "luasandbox/lua.h" #include "luasandbox/util/heka_message.h" +#ifdef _MSC_VER +#ifndef __func__ +#define __func__ __FUNCTION__ +#endif +#endif + extern const char *mozsvc_heka_json; extern const char *mozsvc_heka_json_table; diff --git a/src/luasandbox.c b/src/luasandbox.c index a09d6b4..75ca4b2 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -23,6 +23,12 @@ #include "luasandbox_impl.h" #include "luasandbox_serialize.h" +#ifdef _MSC_VER +#ifndef __func__ +#define __func__ __FUNCTION__ +#endif +#endif + lsb_err_id LSB_ERR_INIT = "already initialized"; lsb_err_id LSB_ERR_LUA = "lua error"; // use lsb_get_error for details lsb_err_id LSB_ERR_TERMINATED = "sandbox already terminated"; diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 70848f9..72b6333 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -15,6 +15,12 @@ #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" +#ifdef _MSC_VER +#ifndef __func__ +#define __func__ __FUNCTION__ +#endif +#endif + static size_t decode_header(char *buf, size_t len, size_t max_message_size) { if (*buf != 0x08) { diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index 85f0105..0a2105e 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -26,6 +26,8 @@ #ifndef _MSC_VER #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-function" +#else +#pragma warning( disable : 4267 4244 ) #endif typedef struct match_node_array { diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index 2f5eb6a..cab78b8 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -20,6 +20,8 @@ #ifndef _MSC_VER #pragma GCC diagnostic ignored "-Wunused-parameter" #pragma GCC diagnostic ignored "-Wunused-function" +#else +#pragma warning( disable : 4267 4244 ) #endif typedef struct match_node_array { diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index c541645..4155091 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -54,7 +54,7 @@ static char* test_lsb_set_tz() mu_assert(lsb_set_tz(NULL), "set_tz failed"); mu_assert(lsb_set_tz("America/Los_Angeles"), "set_tz failed"); static const char too_long[] = "01234567890123456789012345678901"; -#ifdef WIN32 +#ifdef _WIN32 mu_assert(!lsb_set_tz(too_long), "set_tz succeeded"); #else mu_assert(lsb_set_tz(too_long), "set_tz failed"); diff --git a/src/util/util.c b/src/util/util.c index 28b16b3..c8d8489 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -116,7 +116,7 @@ bool lsb_set_tz(const char *tz) } #if _WIN32 char s[32]; - int n = snprintf(s, sizeof(s), "TZ=%s", tz); + int n = _snprintf(s, sizeof(s), "TZ=%s", tz); if (n < 0 || n >= sizeof(s) || _putenv(s) != 0) { return false; } From eba10b762edb3dd4d594cbfe44592fb90af835e4 Mon Sep 17 00:00:00 2001 From: robbie Date: Sun, 17 Apr 2016 10:30:42 -0700 Subject: [PATCH 144/235] Adding Percona support --- modules/mysql.lua | 2 +- src/test/lua/lpeg_mysql.lua | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/mysql.lua b/modules/mysql.lua index 115b674..afd932e 100644 --- a/modules/mysql.lua +++ b/modules/mysql.lua @@ -43,7 +43,7 @@ local use_db = l.P"use " * line local last_insert = l.P"last_insert_id=" * l.Cg(integer, "Last_insert") * "," local insert = l.P"insert_id=" * l.Cg(integer, "Insert_id") * "," -local timestamp = l.P"timestamp=" * l.Cg(integer / "%0000000000", "Timestamp") +local timestamp = l.P"timestamp=" * l.Cg((l.digit^1 / "%0000000000"), "Timestamp") local set = l.P"SET " * last_insert^0 * insert^0 * timestamp * ";" * sep local admin = l.P"# administrator command: " * line diff --git a/src/test/lua/lpeg_mysql.lua b/src/test/lua/lpeg_mysql.lua index e1f2b86..3ad12ef 100644 --- a/src/test/lua/lpeg_mysql.lua +++ b/src/test/lua/lpeg_mysql.lua @@ -224,14 +224,14 @@ end local function header() local t = mysql.slow_query_grammar:match(slow_query_log[1]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end validate(fields, t) end local function standard() local t = mysql.slow_query_grammar:match(slow_query_log[2]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end validate(fields, t) end @@ -244,7 +244,7 @@ end local function mariadb_standard() local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[1]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end validate(mariadb_fields, t) end @@ -257,21 +257,21 @@ end local function mariadb_verbose() local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[3]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end validate(mariadb_verbose_fields, t) end local function percona_standard() local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[1]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end validate(percona_fields, t) end local function percona_verbose() local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[2]) if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Address: " .. t.Hostname) end + if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end validate(percona_verbose_fields, t) end From ec6f176f49efcad927e227aee54663b3814acf11 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 16 May 2016 13:39:38 -0700 Subject: [PATCH 145/235] Use the close_on_exec flag on all file opens - Fix print() with an empty string --- CMakeLists.txt | 3 +-- src/luasandbox.c | 3 ++- src/luasandbox_impl.h | 2 ++ src/luasandbox_serialize.c | 4 ++-- src/util/util.c | 3 ++- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2228464..971dc33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 19) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) @@ -83,5 +83,4 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake ${CMAKE_CURRENT add_subdirectory(src) add_custom_target(copy_includes COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/include) -add_dependencies(copy_includes ${LUA_PROJECT}) add_dependencies(luasandboxutil copy_includes) diff --git a/src/luasandbox.c b/src/luasandbox.c index 75ca4b2..0d92a73 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -147,6 +147,7 @@ static int output_print(lua_State *lua) if (!lsb) { return luaL_error(lua, "print() invalid lightuserdata"); } + lsb->output.buf[0] = 0; lsb->output.pos = 0; // clear the buffer int n = lua_gettop(lua); @@ -402,7 +403,7 @@ static void set_random_seed() #ifdef _WIN32 // todo use CryptGenRandom to seed srand #else - FILE *fh = fopen("/dev/urandom", "r"); + FILE *fh = fopen("/dev/urandom", "r" CLOSE_ON_EXEC); if (fh) { unsigned seed; unsigned char advance; diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index 91f04b9..144b886 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -15,6 +15,8 @@ #ifdef _WIN32 #define snprintf _snprintf +#elif __linux +#define CLOSE_ON_EXEC "e" #endif struct lsb_lua_sandbox { diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 7bc65fb..a97b8ec 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -395,7 +395,7 @@ lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) lua_pushvalue(lsb->lua, LUA_GLOBALSINDEX); - FILE *fh = fopen(lsb->state_file, "wb"); + FILE *fh = fopen(lsb->state_file, "wb" CLOSE_ON_EXEC); if (fh == NULL) { int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "preserve_global_data could not open: %s", @@ -484,7 +484,7 @@ lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) static int file_exists(const char *fn) { - FILE *fh = fopen(fn, "r"); + FILE *fh = fopen(fn, "r" CLOSE_ON_EXEC); if (fh) { fclose(fh); return 1; diff --git a/src/util/util.c b/src/util/util.c index c8d8489..3c73cee 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -7,6 +7,7 @@ /** General purpose utility functions @file */ #include "luasandbox/util/util.h" +#include "../luasandbox_impl.h" #include #include @@ -52,7 +53,7 @@ char* lsb_read_file(const char *fn) { char *str = NULL; size_t b; - FILE *fh = fopen(fn, "rb"); + FILE *fh = fopen(fn, "rb" CLOSE_ON_EXEC); if (!fh) return str; if (fseek(fh, 0, SEEK_END)) goto cleanup; From 11b56661b2c878f8f9959a7ec8955ca08a3d9d07 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 24 May 2016 10:18:18 -0700 Subject: [PATCH 146/235] Add lsb_heka_cat utility for examining Heka protobuf streams/files --- CMakeLists.txt | 2 +- docs/heka/cli_tools.md | 23 ++ sandboxes/heka/output/debug.lua | 8 +- src/CMakeLists.txt | 3 + src/cli/CMakeLists.txt | 13 + src/cli/lsb_heka_cat.c | 423 ++++++++++++++++++++++++++++++++ 6 files changed, 467 insertions(+), 5 deletions(-) create mode 100644 docs/heka/cli_tools.md create mode 100644 src/cli/CMakeLists.txt create mode 100644 src/cli/lsb_heka_cat.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 971dc33..ed497a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 19) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/docs/heka/cli_tools.md b/docs/heka/cli_tools.md new file mode 100644 index 0000000..0eec7dc --- /dev/null +++ b/docs/heka/cli_tools.md @@ -0,0 +1,23 @@ +## Heka CLI Tools + +### lsb_heka_cat + +A command-line utility for counting, viewing, filtering, and extracting messages +from a Heka protobuf log/stream. + +``` +usage: src/cli/lsb_heka_cat [-t|-c|-h] [-m message_matcher] [-f] [-n #] +description: + -t output the messages in text format (default) + -c only output the message count + -h output the messages as a Heka protobuf stream + -f output appended data as the file grows + -n output the last # of messages (simple header check so not 100% accurate) + -m message_matcher expression (default "TRUE") + FILE name of the file to cat or '-' for stdin +notes: + All output is written to stdout and all log/error messages are written to stderr. + +``` +Set the [message matcher](message_matcher.md) documentation for more details about the message_matcher expression. + diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua index 5cc8dd2..946f41a 100644 --- a/sandboxes/heka/output/debug.lua +++ b/sandboxes/heka/output/debug.lua @@ -30,10 +30,10 @@ function process_message() write(":Hostname: ", msg.Hostname or "", "\n") write(":Fields:\n") for i, v in ipairs(msg.Fields or {}) do - write(" | name:", v.name, - " type:", v.value_type or 0, - " representation:", v.representation or "", - " value:") + write(" | name: ", v.name, + " type: ", v.value_type or 0, + " representation: ", v.representation or "", + " value: ") if v.value_type == 4 then for j, w in ipairs(v.value) do if j ~= 1 then write(",") end diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a078f3d..d0df9d2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,4 +49,7 @@ install(DIRECTORY "${EP_BASE}/include/" DESTINATION include COMPONENT core) add_subdirectory(util) add_subdirectory(heka) +if(NOT WIN32) # todo need to add getopt support for Windows + add_subdirectory(cli) +endif() add_subdirectory(test) diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt new file mode 100644 index 0000000..b3d969c --- /dev/null +++ b/src/cli/CMakeLists.txt @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +add_executable(lsb_heka_cat lsb_heka_cat) +target_link_libraries(lsb_heka_cat luasandboxutil) + +if(LIBM_LIBRARY) + target_link_libraries(lsb_heka_cat ${LIBM_LIBRARY}) +endif() + +install(TARGETS lsb_heka_cat DESTINATION bin) +install(DIRECTORY scripts/ DESTINATION bin) diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c new file mode 100644 index 0000000..903232f --- /dev/null +++ b/src/cli/lsb_heka_cat.c @@ -0,0 +1,423 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lua_sandbox Heka file stream cat @file */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" +#include "luasandbox/util/input_buffer.h" +#include "luasandbox/util/protobuf.h" +#include "luasandbox/util/util.h" + +typedef void (*output_function)(lsb_heka_message *msg); + +static void +log_cb(void *context, const char *component, int level, const char *fmt, ...) +{ + (void)context; + (void)component; + (void)level; + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fwrite("\n", 1, 1, stderr); +} +static lsb_logger logger = { .context = NULL, .cb = log_cb }; + + +// todo this function is duplicated in util/heka_message.c +// we may want to add it to the API if it is generally useful +static const char* +read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) +{ + if (wiretype != LSB_PB_WT_LENGTH) { + return NULL; + } + + long long vi; + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || p + vi > e) { + return NULL; + } + s->s = p; + s->len = (size_t)vi; + return p + vi; +} + + +static void output_cs(const char *key, lsb_const_string *cs, bool eol) +{ + if (cs->s) { + fprintf(stdout, "%s: %.*s", key, (int)cs->len, cs->s); + } else { + fprintf(stdout, "%s: ", key); + } + if (eol) { + fprintf(stdout, "\n"); + } +} + + +static void output_text(lsb_heka_message *msg) +{ + static char tstr[64]; + if (!msg->raw.s) return; + + fprintf(stdout, ":Uuid: %02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx" + "-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx\n", + msg->uuid.s[0], msg->uuid.s[1], msg->uuid.s[2], msg->uuid.s[3], + msg->uuid.s[4], msg->uuid.s[5], msg->uuid.s[6], msg->uuid.s[7], + msg->uuid.s[8], msg->uuid.s[9], msg->uuid.s[10], msg->uuid.s[11], + msg->uuid.s[12], msg->uuid.s[13], msg->uuid.s[14], msg->uuid.s[15]); + fprintf(stdout, ":Timestamp: "); + + time_t t = floor(msg->timestamp / 1e9); + double frac = msg->timestamp - t * 1e9; + struct tm *tms = gmtime(&t); + strftime(tstr, sizeof(tstr) - 1, "%Y-%m-%dT%H:%M:%S", tms); + fprintf(stdout, "%s.%09lldZ\n", tstr, (long long)frac); + output_cs(":Type", &msg->type, true); + output_cs(":Logger", &msg->logger, true); + fprintf(stdout, ":Severity: %d\n", msg->severity); + output_cs(":Payload", &msg->payload, true); + output_cs(":EnvVersion", &msg->env_version, true); + fprintf(stdout, ":Pid: %d\n", msg->pid); + output_cs(":Hostname", &msg->hostname, true); + fprintf(stdout, ":Fields:\n"); + for (int i = 0; i < msg->fields_len; ++i) { + fprintf(stdout, " | name: %.*s type: %d ", (int)msg->fields[i].name.len, + msg->fields[i].name.s, msg->fields[i].value_type); + output_cs("representation", &msg->fields[i].representation, false); + fprintf(stdout, " value: "); + const char *p = msg->fields[i].value.s; + const char *e = msg->fields[i].value.s + msg->fields[i].value.len; + switch (msg->fields[i].value_type) { + case LSB_PB_STRING: + { + lsb_const_string cs; + int tag = 0; + int wiretype = 0; + while (p && p < e) { + p = lsb_pb_read_key(p, &tag, &wiretype); + p = read_string(wiretype, p, e, &cs); + if (p) { + fprintf(stdout, "%.*s", (int)cs.len, cs.s); + } + } + } + break; + case LSB_PB_BYTES: + { + lsb_const_string cs; + int tag = 0; + int wiretype = 0; + while (p && p < e) { + p = lsb_pb_read_key(p, &tag, &wiretype); + p = read_string(wiretype, p, e, &cs); + if (p) { + for (size_t i = 0; i < cs.len; ++i) { + fprintf(stdout, "%02hxx", (unsigned char)cs.s[i]); + } + } + } + } + break; + case LSB_PB_INTEGER: + { + long long ll = 0; + while (p && p < e) { + p = lsb_pb_read_varint(p, e, &ll); + if (p) { + fprintf(stdout, "%lld", ll); + } + } + } + break; + case LSB_PB_DOUBLE: + { + double d; + for (int i = 0; p <= (e - sizeof(double)); p += sizeof(double), ++i) { + memcpy(&d, p, sizeof(double)); + if (i > 0) { + fprintf(stdout, ","); + } + fprintf(stdout, "%.17g", d); + } + } + break; + case LSB_PB_BOOL: + { + long long ll = 0; + while (p && p < e) { + p = lsb_pb_read_varint(p, e, &ll); + if (p) { + fprintf(stdout, "%s", ll == 0 ? "false" : "true"); + } + } + } + break; + } + fprintf(stdout, "\n"); + } + fprintf(stdout, "\n"); + return; +} + + +static void output_heka(lsb_heka_message *msg) +{ + static char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f + int hlen = lsb_pb_output_varint(header + 3, msg->raw.len) + 1; + header[1] = (char)hlen; + header[hlen + 2] = '\x1f'; + if (fwrite(header, hlen + 3, 1, stdout) != 1) { + log_cb(NULL, NULL, 0, "error outputting header"); + exit(1); + } + if (fwrite(msg->raw.s, msg->raw.len, 1, stdout) != 1) { + log_cb(NULL, NULL, 0, "error outputting message"); + exit(1); + } + return; +} + + +static size_t read_file(FILE *fh, lsb_input_buffer *ib) +{ + size_t need; + if (ib->msglen) { + need = ib->msglen + (size_t)ib->buf[ib->scanpos + 1] + LSB_HDR_FRAME_SIZE + - (ib->readpos - ib->scanpos); + } else { + need = ib->scanpos + ib->size - ib->readpos; + } + + if (lsb_expand_input_buffer(ib, need)) { + log_cb(NULL, NULL, 0, "buffer reallocation failed"); + exit(EXIT_FAILURE); + } + size_t nread = fread(ib->buf + ib->readpos, + 1, + ib->size - ib->readpos, + fh); + ib->readpos += nread; + return nread; +} + + +static int find_header(const char *cur, int clen, const char *prev, int num) +{ + static int cnt = 0; + for (int i = clen - 1; i >= 0; --i) { + if (cur[i] == 0x1e) { + unsigned char hlen; + if (i + 1 < clen) { + hlen = (unsigned char)cur[i + 1]; + } else { + hlen = (unsigned char)prev[0]; + } + + char flag; + int hend = i + 2 + hlen; + if (hend < clen) { + flag = cur[hend]; + } else { + flag = prev[hend - clen]; + } + + if (flag == 0x1f) { + if (++cnt == num) { + return i; + } + } + } + } + return -1; +} + + +static void move_to_offset(FILE *fh, int num) +{ + char buf[2][BUFSIZ]; + memset(buf, 0, sizeof(buf)); + char *cur = buf[0]; + char *prev = buf[1]; + char *tmp; + + fseek(fh, 0, SEEK_END); + long len = ftell(fh); + if (len < 0) { + log_cb(NULL, NULL, 0, "ftell failed"); + } + size_t consume = len > BUFSIZ ? BUFSIZ : len; + size_t pos = len - consume; + while (consume > 0) { + if (fseek(fh, pos, SEEK_SET)) { + log_cb(NULL, NULL, 0, "fseek failed (reading)"); + break; + } + if (fread(cur, consume, 1, fh) != 1) { + log_cb(NULL, NULL, 0, "fread failed"); + break; + } + + int loc = find_header(cur, consume, prev, num); + if (loc >= 0) { + if (fseek(fh, -(consume - loc) , SEEK_CUR)) { + log_cb(NULL, NULL, 0, "fseek failed (find position)"); + } + return; + } + + if (pos >= consume) { + pos -= consume; + } else { + consume = pos; + pos = 0; + } + + tmp = cur; + cur = prev; + prev = tmp; + } + fseek(fh, 0, SEEK_SET); +} + + +int main(int argc, char **argv) +{ + bool argerr = false; + bool follow = false; + bool use_stdin = false; + long num = -1; + char *matcher = "TRUE"; + char *filename = NULL; + + output_function ofn = output_text; + + int c; + while ((c = getopt(argc, argv, "tchfn:m:")) != -1) { + switch (c) { + case 't': + ofn = output_text; + break; + case 'c': + ofn = NULL; + break; + case 'h': + ofn = output_heka; + break; + case 'f': + follow = true; + break; + case 'n': + num = strtol(optarg, NULL, 10); + if (num < 0) argerr = true; + break; + case 'm': + matcher = optarg; + break; + default: + argerr = true; + break; + } + } + + if (argc - optind == 1) { + filename = argv[optind]; + use_stdin = strcmp("-", filename) == 0; + } else { + argerr = true; + } + + if (argerr) { + log_cb(NULL, NULL, 0, + "usage: %s [-t|-c|-h] [-m message_matcher] [-f] [-n #] \n" + "description:\n" + " -t output the messages in text format (default)\n" + " -c only output the message count\n" + " -h output the messages as a Heka protobuf stream\n" + " -f output appended data as the file grows\n" + " -n output the last # of messages (simple header check so not " + "100%% accurate)\n" + " -m message_matcher expression (default \"TRUE\")\n" + " FILE name of the file to cat or '-' for stdin\n" + "notes:\n" + " All output is written to stdout and all log/error messages are " + "written to stderr.\n", + argv[0]); + return EXIT_FAILURE; + } + + lsb_message_matcher *mm = lsb_create_message_matcher(matcher); + if (!mm) { + log_cb(NULL, NULL, 0, "invalid message matcher: %s", matcher); + return EXIT_FAILURE; + } + + FILE *fh = stdin; + if (!use_stdin) { + fh = fopen(filename, "r"); + if (!fh) { + log_cb(NULL, NULL, 0, "error opening: %s", filename); + return EXIT_FAILURE; + } + if (num >= 0) { + move_to_offset(fh, num); + } + } + + size_t discarded_bytes; + size_t bytes_read = 0; + size_t pcnt = 0; + size_t mcnt = 0; + + lsb_input_buffer ib; + lsb_init_input_buffer(&ib, 1024 * 1024 * 1024); + lsb_heka_message msg; + lsb_init_heka_message(&msg, 8); + + do { + if (lsb_find_heka_message(&msg, &ib, true, &discarded_bytes, &logger)) { + if (lsb_eval_message_matcher(mm, &msg)) { + if (ofn) { + ofn(&msg); + } + ++mcnt; + } + ++pcnt; + } else { + bytes_read = read_file(fh, &ib); + } + if (bytes_read == 0 && follow && !use_stdin) { + sleep(1); + } + } while (bytes_read > 0 || follow); + + lsb_free_heka_message(&msg); + lsb_free_input_buffer(&ib); + lsb_destroy_message_matcher(mm); + if (!use_stdin) { + fclose(fh); + } + + if (ofn) { + log_cb(NULL, NULL, 0, "Processed: %zu, matched: %zu messages\n", pcnt, + mcnt); + } else { + printf("Processed: %zu, matched: %zu messages\n", pcnt, mcnt); + } +} From b68915e056451bd7fb470e00b1076aa82937cd63 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 24 May 2016 14:22:50 -0700 Subject: [PATCH 147/235] Fix up the Kafka tests - add support for the logger context - update error message checks for the latest librdkafka --- CMakeLists.txt | 2 +- src/heka/test/lua/kafka_consumer.lua | 4 ++-- src/heka/test/lua/kafka_producer.lua | 5 +++-- src/heka/test/test_kafka.c | 9 +++++---- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ed497a2..ae27ee8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 19) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_PATCH 3) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/src/heka/test/lua/kafka_consumer.lua b/src/heka/test/lua/kafka_consumer.lua index f8c7763..1afe996 100644 --- a/src/heka/test/lua/kafka_consumer.lua +++ b/src/heka/test/lua/kafka_consumer.lua @@ -52,10 +52,10 @@ ok, err = pcall(heka_kafka_consumer.new, "test", {"one", item = "test"}, {["grou assert(err == "topics must be an array of strings", err) ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo", ["message.max.bytes"] = true}) -assert(err == 'Failed to set message.max.bytes = true : Configuration property "message.max.bytes" value 0 is outside allowed range 1000..1000000000\n', err) +assert(err:match("^Failed to set message.max.bytes = true"), err) -local consumer = heka_kafka_consumer.new("localhost:9092", {"test"}, {["group.id"] = "integration_testing"}) +local consumer = heka_kafka_consumer.new("localhost:9092", {"test"}, {["group.id"] = "integration_testing"}, {["auto.offset.reset"] = "smallest"}) local consumer1 = heka_kafka_consumer.new("localhost:9092", {"test:1"}, {["group.id"] = "other"}) local pb, topic, partition, key = consumer1:receive() assert(not pb) diff --git a/src/heka/test/lua/kafka_producer.lua b/src/heka/test/lua/kafka_producer.lua index 5943802..d076a2c 100644 --- a/src/heka/test/lua/kafka_producer.lua +++ b/src/heka/test/lua/kafka_producer.lua @@ -3,6 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "heka_kafka_producer" +require "string" local ok, err ok, err = pcall(heka_kafka_producer.new) @@ -15,13 +16,13 @@ ok, err = pcall(heka_kafka_producer.new, "brokerlist", true) assert(err == "bad argument #2 to '?' (table expected, got boolean)", err) ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = "foo"}) -assert(err == 'Failed to set message.max.bytes = foo : Configuration property "message.max.bytes" value 0 is outside allowed range 1000..1000000000\n', err) +assert(err:match("^Failed to set message.max.bytes = foo"), err) ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = 1}) assert(err == 'Failed to set message.max.bytes = 1 : Configuration property "message.max.bytes" value 1 is outside allowed range 1000..1000000000\n', err) ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = true}) -assert(err == 'Failed to set message.max.bytes = true : Configuration property "message.max.bytes" value 0 is outside allowed range 1000..1000000000\n', err) +assert(err:match("^Failed to set message.max.bytes = true"), err) ok, err = pcall(heka_kafka_producer.new, "brokerlist", {[assert] = true}) assert(err == "invalid config key type: function", err) diff --git a/src/heka/test/test_kafka.c b/src/heka/test/test_kafka.c index cb66b12..4f58062 100644 --- a/src/heka/test/test_kafka.c +++ b/src/heka/test/test_kafka.c @@ -15,9 +15,9 @@ char *e = NULL; -void dlog(const char *component, int level, const char *fmt, ...) +void dlog(void *context, const char *component, int level, const char *fmt, ...) { - + (void)context; va_list args; va_start(args, fmt); fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, @@ -26,6 +26,7 @@ void dlog(const char *component, int level, const char *fmt, ...) fwrite("\n", 1, 1, stderr); va_end(args); } +static lsb_logger logger = { .context = NULL, .cb = dlog }; static volatile ptrdiff_t g_sequence = 0; static int ucp(void *parent, void *sequence_id) @@ -43,7 +44,7 @@ static char* test_producer() mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, NULL), "failed"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_output(NULL, "lua/kafka_producer.lua", NULL, NULL, dlog, + hsb = lsb_heka_create_output(NULL, "lua/kafka_producer.lua", NULL, NULL, &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)1, false), "err: %s", @@ -81,7 +82,7 @@ static char* test_consumer() { lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/kafka_consumer.lua", NULL, NULL, - dlog, iim); + &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); mu_assert(0 == lsb_heka_pm_input(hsb, 0, NULL, false), "err: %s", lsb_heka_get_error(hsb)); From c7fb38a659f021f39a76eeabdc87ee1c76d83f89 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 26 May 2016 08:34:55 -0700 Subject: [PATCH 148/235] Remove the installation of the cli/scripts directory --- src/cli/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt index b3d969c..f2556f1 100644 --- a/src/cli/CMakeLists.txt +++ b/src/cli/CMakeLists.txt @@ -10,4 +10,3 @@ if(LIBM_LIBRARY) endif() install(TARGETS lsb_heka_cat DESTINATION bin) -install(DIRECTORY scripts/ DESTINATION bin) From f92d78deb02b831dfa858edd7b38c25d4a1f7657 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 26 May 2016 10:08:46 -0700 Subject: [PATCH 149/235] Add lsb_heka_cat to the installer packages --- src/cli/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt index f2556f1..02850cf 100644 --- a/src/cli/CMakeLists.txt +++ b/src/cli/CMakeLists.txt @@ -9,4 +9,4 @@ if(LIBM_LIBRARY) target_link_libraries(lsb_heka_cat ${LIBM_LIBRARY}) endif() -install(TARGETS lsb_heka_cat DESTINATION bin) +install(TARGETS lsb_heka_cat DESTINATION bin COMPONENT core) From 90540b8d9ab9e67efd5bb5c694bbb405c94f3e8f Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 27 May 2016 07:02:48 -0700 Subject: [PATCH 150/235] Improve the human readable lsb_heka_cat output for binary fields Since all userdata output is treated as binary (e.g. heka_json) it is better to only encode non printable chars and escape slashes for the human readable output. --- CMakeLists.txt | 2 +- src/cli/lsb_heka_cat.c | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ae27ee8..3fe56e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 19) -set(CPACK_PACKAGE_VERSION_PATCH 3) +set(CPACK_PACKAGE_VERSION_PATCH 4) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 903232f..4630f18 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -6,6 +6,7 @@ /** @brief lua_sandbox Heka file stream cat @file */ +#include #include #include #include @@ -129,7 +130,15 @@ static void output_text(lsb_heka_message *msg) p = read_string(wiretype, p, e, &cs); if (p) { for (size_t i = 0; i < cs.len; ++i) { - fprintf(stdout, "%02hxx", (unsigned char)cs.s[i]); + if (isprint(cs.s[i])) { + if (cs.s[i] == '\\') { + fwrite("\\\\", 2, 1, stdout); + } else { + putchar(cs.s[i]); + } + } else { + fprintf(stdout, "\\x%02hhx", (unsigned char)cs.s[i]); + } } } } From 5f3107b6ad0cdfae9974df2d3ed039a9544650c2 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 27 May 2016 18:43:14 -0700 Subject: [PATCH 151/235] Move the inject_payload arg check up so it fails on invalid args The payload check was first, so if there was no data it would just no-op instead of throwing an error. This was an unintentional change in behaviour compared to earlier versions of the API. --- CMakeLists.txt | 2 +- src/heka/sandbox.c | 15 ++++++++------- src/heka/test/lua/aim.lua | 8 ++++++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3fe56e9..0fc4de2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 19) -set(CPACK_PACKAGE_VERSION_PATCH 4) +set(CPACK_PACKAGE_VERSION_PATCH 5) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index f46ed63..c853da6 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -232,13 +232,6 @@ static int inject_payload(lua_State *lua) } int n = lua_gettop(lua); - if (n > 2) { - lsb_output(lsb, 3, n, 1); - lua_pop(lua, n - 2); - } - size_t len = 0; - const char *output = lsb_get_output(lsb, &len); - if (!len) return 0; if (n > 0) { if (lua_type(lua, 1) != LUA_TSTRING) { @@ -254,6 +247,14 @@ static int inject_payload(lua_State *lua) } } + if (n > 2) { + lsb_output(lsb, 3, n, 1); + lua_pop(lua, n - 2); + } + size_t len = 0; + const char *output = lsb_get_output(lsb, &len); + if (!len) return 0; + // build up a heka message table lua_createtable(lua, 0, 2); // message lua_createtable(lua, 0, 2); // Fields diff --git a/src/heka/test/lua/aim.lua b/src/heka/test/lua/aim.lua index 353edd7..07106e5 100644 --- a/src/heka/test/lua/aim.lua +++ b/src/heka/test/lua/aim.lua @@ -47,6 +47,14 @@ ok, err = pcall(inject_payload, "txt", true, "data") assert(not ok) assert("inject_payload() payload_name argument must be a string" == err, string.format("received: %s", err)) +ok, err = pcall(inject_payload, nil) +assert(not ok) +assert("inject_payload() payload_type argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_payload, "", nil) +assert(not ok) +assert("inject_payload() payload_name argument must be a string" == err, string.format("received: %s", err)) + ok, err = pcall(inject_message, {Fields = {foo = {value = {"s", true}}}}) assert(not ok) assert("inject_message() failed: array has mixed types" == err, string.format("received: %s", err)) From 849ba0b52f01c97c6e1582ef51304f1b626a1f94 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 1 Jun 2016 15:19:32 -0700 Subject: [PATCH 152/235] Pull some common defines out of the impl header Prevent the heka_message_matcher from pulling in header files it no longer requires --- src/heka/sandbox.c | 2 +- src/heka/test/test.h.in | 1 - src/luasandbox.c | 1 + src/luasandbox_defines.h | 18 ++++++++++++++++++ src/luasandbox_impl.h | 6 ------ src/luasandbox_serialize.c | 1 + src/test/mu_test.h | 4 +--- src/util/heka_message_matcher.c | 4 ---- src/util/util.c | 2 +- 9 files changed, 23 insertions(+), 16 deletions(-) create mode 100644 src/luasandbox_defines.h diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index c853da6..c8b9a17 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -12,6 +12,7 @@ #include #include +#include "../luasandbox_defines.h" #include "luasandbox.h" #include "luasandbox/util/heka_message_matcher.h" #include "luasandbox/util/protobuf.h" @@ -27,7 +28,6 @@ #ifdef _WIN32 #include -#define snprintf _snprintf #else #include #endif diff --git a/src/heka/test/test.h.in b/src/heka/test/test.h.in index 4ded823..fde124c 100644 --- a/src/heka/test/test.h.in +++ b/src/heka/test/test.h.in @@ -19,7 +19,6 @@ #define TEST_LUA_PATH "${CMAKE_SOURCE_DIR}/modules/?.lua" #ifdef _WIN32 -#define snprintf _snprintf #define TEST_LUA_CPATH "${CMAKE_BINARY_DIR}/ep_base/lib/lua/?.dll" #else #define TEST_LUA_CPATH "${CMAKE_BINARY_DIR}/ep_base/lib/lua/?.so" diff --git a/src/luasandbox.c b/src/luasandbox.c index 0d92a73..5c1d2ff 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -20,6 +20,7 @@ #include "luasandbox/lua.h" #include "luasandbox/lualib.h" #include "luasandbox/util/output_buffer.h" +#include "luasandbox_defines.h" #include "luasandbox_impl.h" #include "luasandbox_serialize.h" diff --git a/src/luasandbox_defines.h b/src/luasandbox_defines.h new file mode 100644 index 0000000..6d491d7 --- /dev/null +++ b/src/luasandbox_defines.h @@ -0,0 +1,18 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Shared compiler defines @file */ + +#ifndef luasandbox_defines_h_ +#define luasandbox_defines_h_ + +#ifdef _WIN32 +#define snprintf _snprintf +#elif __linux +#define CLOSE_ON_EXEC "e" +#endif + +#endif diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index 144b886..0a5f8c5 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -13,12 +13,6 @@ #include "luasandbox/lua.h" #include "luasandbox/util/output_buffer.h" -#ifdef _WIN32 -#define snprintf _snprintf -#elif __linux -#define CLOSE_ON_EXEC "e" -#endif - struct lsb_lua_sandbox { lua_State *lua; void *parent; diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index a97b8ec..63d6d98 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -14,6 +14,7 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lualib.h" +#include "luasandbox_defines.h" #include "luasandbox_impl.h" #ifdef _MSC_VER diff --git a/src/test/mu_test.h b/src/test/mu_test.h index e8d6bc0..fcaa293 100644 --- a/src/test/mu_test.h +++ b/src/test/mu_test.h @@ -11,9 +11,7 @@ #include -#ifdef _WIN32 -#define snprintf _snprintf -#endif +#include "../luasandbox_defines.h" #if defined(_MSC_VER) #define PRIuSIZE "Iu" diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c index 538898c..4b6dcb6 100644 --- a/src/util/heka_message_matcher.c +++ b/src/util/heka_message_matcher.c @@ -12,10 +12,6 @@ #include #include -#include "luasandbox.h" -#include "luasandbox/lauxlib.h" -#include "luasandbox/lua.h" -#include "luasandbox/lualib.h" #include "luasandbox/util/heka_message.h" #include "luasandbox/util/heka_message_matcher.h" #include "luasandbox/util/string.h" diff --git a/src/util/util.c b/src/util/util.c index 3c73cee..845f3ae 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -7,7 +7,7 @@ /** General purpose utility functions @file */ #include "luasandbox/util/util.h" -#include "../luasandbox_impl.h" +#include "../luasandbox_defines.h" #include #include From 9e87dcd4896afc668cbc879374e668f5198e9856 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 3 Jun 2016 12:51:37 -0700 Subject: [PATCH 153/235] BREAKING CHANGES - the modules directory was restructured - Sandbox 'require' calls need to be updated to the new module structure - As the sandbox has grown in scope a top level parsing module e.g. 'mysql' is no longer appropriate so it has been corrected by restructuring the module directory. - Code changes to sync up with the documentation - read_message("Pid") can now return nil if no pid is set - os.getenv calls are allowed in Heka input/output sandboxes - Documentation cleanup, generation and publication to gh-pages http://mozilla-services.github.io/lua_sandbox/ --- .gitignore | 1 + CMakeLists.txt | 4 +- README.md | 10 +- cmake/doxygen.cmake | 51 +-- docs/{heka/cli_tools.md => cli/index.md} | 8 +- docs/heka/analysis.md | 93 +++--- docs/heka/heka_json.md | 30 +- docs/heka/index.md | 40 +-- docs/heka/input.md | 51 +-- docs/heka/kafka_consumer.md | 10 +- docs/heka/kafka_producer.md | 18 +- docs/heka/message.md | 6 +- docs/heka/output.md | 124 ++++---- docs/heka/stream_reader.md | 22 +- docs/index.md | 31 +- docs/sandbox.md | 61 ++-- docs/{heka => util}/message_matcher.md | 38 +-- gen_gh_pages.lua | 297 ++++++++++++++++++ include/luasandbox/heka/sandbox.h | 2 +- include/luasandbox_output.h | 3 +- include/luasandbox_serialize.h | 1 + modules/heka/elasticsearch.lua | 10 +- modules/heka/elasticsearch/moz_telemetry.lua | 133 -------- modules/heka/elasticsearch/payload.lua | 22 +- modules/heka/msg_interpolate.lua | 8 +- modules/{ => heka}/util.lua | 43 ++- modules/{ => lpeg}/cbufd.lua | 61 ++-- modules/{ => lpeg}/common_log_format.lua | 46 ++- modules/{ => lpeg}/date_time.lua | 71 ++++- modules/{ => lpeg}/ip_address.lua | 9 + modules/{ => lpeg}/mysql.lua | 14 +- modules/{ => lpeg}/postfix.lua | 19 +- modules/{ => lpeg}/syslog.lua | 29 +- modules/{ => lpeg}/syslog_message.lua | 46 ++- modules/lsb/util.lua | 77 +++++ modules/sfl4j.lua | 58 ---- sandboxes/heka/analysis/throughput.lua | 9 +- sandboxes/heka/input/heka_kafka.lua | 49 +-- sandboxes/heka/input/heka_stdin.lua | 7 +- sandboxes/heka/input/heka_tcp.lua | 7 +- sandboxes/heka/input/syslog_udp.lua | 7 +- sandboxes/heka/output/debug.lua | 9 +- .../heka/output/elasticsearch_bulk_api.lua | 4 +- sandboxes/heka/output/heka_kafka.lua | 39 +-- sandboxes/heka/output/heka_log_rolling.lua | 9 +- sandboxes/heka/output/heka_s3_partition.lua | 10 +- sandboxes/heka/output/heka_tcp.lua | 7 +- sandboxes/heka/output/heka_tcp_matcher.lua | 11 +- sandboxes/heka/output/inject_payload.lua | 13 +- src/heka/message.c | 7 +- src/heka/sandbox.c | 2 +- src/heka/test/lua/heka_util.lua | 28 ++ src/heka/test/lua/read_message.lua | 2 +- src/heka/test/test_sandbox.c | 23 ++ src/test/lua/lpeg_cbufd.lua | 2 +- src/test/lua/lpeg_clf.lua | 3 +- src/test/lua/lpeg_date_time.lua | 2 +- src/test/lua/lpeg_ip_address.lua | 2 +- src/test/lua/lpeg_mysql.lua | 2 +- src/test/lua/lpeg_postfix.lua | 2 +- src/test/lua/lpeg_sfl4j.lua | 118 ------- src/test/lua/lpeg_syslog.lua | 2 +- src/test/lua/lpeg_syslog_message.lua | 4 +- src/test/lua/serialize.lua | 2 +- src/test/lua/util_test.lua | 42 +-- src/test/test_luasandbox.c | 3 +- src/util/heka_message.c | 3 +- 67 files changed, 1184 insertions(+), 793 deletions(-) rename docs/{heka/cli_tools.md => cli/index.md} (70%) rename docs/{heka => util}/message_matcher.md (70%) create mode 100644 gen_gh_pages.lua delete mode 100644 modules/heka/elasticsearch/moz_telemetry.lua rename modules/{ => heka}/util.lua (60%) rename modules/{ => lpeg}/cbufd.lua (55%) rename modules/{ => lpeg}/common_log_format.lua (94%) rename modules/{ => lpeg}/date_time.lua (84%) rename modules/{ => lpeg}/ip_address.lua (89%) rename modules/{ => lpeg}/mysql.lua (96%) rename modules/{ => lpeg}/postfix.lua (98%) rename modules/{ => lpeg}/syslog.lua (94%) rename modules/{ => lpeg}/syslog_message.lua (98%) create mode 100644 modules/lsb/util.lua delete mode 100644 modules/sfl4j.lua create mode 100644 src/heka/test/lua/heka_util.lua delete mode 100644 src/test/lua/lpeg_sfl4j.lua diff --git a/.gitignore b/.gitignore index 2fdd6b1..201bee5 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ release/ src/test/modules/ Testing *~ +gh-pages/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 0fc4de2..b4d00c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 19) -set(CPACK_PACKAGE_VERSION_PATCH 5) +set(CPACK_PACKAGE_VERSION_MINOR 20) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/README.md b/README.md index a102272..1d8aa8d 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,10 @@ environment including functionality like global data preservation/restoration on shutdown/startup, output collection in textual or binary formats and an array of parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) +These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/trink/hindsight). +The goal was to decouple the Heka/Hindsight functionality from any particular +infrastructure and make it embeddable into any tool or language. + ### Features - small - memory requirements are as little as 8 KiB for a basic sandbox @@ -25,7 +29,7 @@ parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC gramma Containment is defined in terms of restriction to the operating system, file system, libraries, memory use, Lua instruction use, and output size. -[Full Documentation](docs/index.md) +[Full Documentation](http://mozilla-services.github.io/lua_sandbox) ## Installation @@ -36,7 +40,9 @@ parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC gramma #### Optional (used for documentation) * Graphviz (2.28.0) - http://graphviz.org/Download..php -* Doxygen (1.8+)- http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc +* Doxygen (1.8.11+)- http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc +* pandoc (1.17) - http://pandoc.org/ +* lua (5.1) - https://www.lua.org/download.html ### CMake Build Instructions diff --git a/cmake/doxygen.cmake b/cmake/doxygen.cmake index ca21c84..063d70e 100644 --- a/cmake/doxygen.cmake +++ b/cmake/doxygen.cmake @@ -3,36 +3,41 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. find_package(Doxygen QUIET) -if(DOXYGEN_FOUND) +find_program(LUA_EXE lua QUIET) +find_program(PANDOC_EXE pandoc QUIET) +if(DOXYGEN_FOUND AND LUA_EXE AND PANDOC_EXE) set(DOXYCONF_IN ${CMAKE_SOURCE_DIR}/doxygen.in.conf) set(DOXYCONF_OUT ${CMAKE_BINARY_DIR}/doxygen.conf) if(EXISTS ${DOXYCONF_IN}) configure_file(${DOXYCONF_IN} ${DOXYCONF_OUT}) else() file(WRITE ${DOXYCONF_OUT} " -PROJECT_NAME = \"${PROJECT_NAME}\" -PROJECT_BRIEF = \"${CPACK_PACKAGE_DESCRIPTION_SUMMARY}\" -OUTPUT_DIRECTORY = docs -GENERATE_LATEX = NO -GENERATE_TODOLIST = YES -FULL_PATH_NAMES = YES -STRIP_FROPATH = \"${CMAKE_SOURCE_DIR}\" -SOURCE_BROWSER = YES -TAB_SIZE = 4 -EXTRACT_ALL = YES -JAVADOC_AUTOBRIEF = YES -RECURSIVE = YES -INPUT = \"${CMAKE_SOURCE_DIR}\" -EXCLUDE_PATTERNS = \"${CMAKE_SOURCE_DIR}/.*\" \"${CMAKE_SOURCE_DIR}/debug*\" \"${CMAKE_SOURCE_DIR}/release*\" -EXAMPLE_PATH = ${EXAMPLE_PATHS} -IMAGE_PATH = ${IMAGE_PATHS} -BUILTIN_STL_SUPPORT = YES -STRIP_CODE_COMMENTS = NO -SHOW_DIRECTORIES = YES -PROJECT_NUMBER = ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") +PROJECT_NAME = \"${PROJECT_NAME}\" +PROJECT_BRIEF = \"${CPACK_PACKAGE_DESCRIPTION_SUMMARY}\" +OUTPUT_DIRECTORY = \"${CMAKE_SOURCE_DIR}/gh-pages\" +HTML_OUTPUT = doxygen +GENERATE_LATEX = NO +GENERATE_TODOLIST = YES +FULL_PATH_NAMES = YES +STRIP_FROPATH = \"${CMAKE_SOURCE_DIR}\" +SOURCE_BROWSER = YES +TAB_SIZE = 4 +EXTRACT_ALL = YES +JAVADOC_AUTOBRIEF = YES +RECURSIVE = YES +INPUT = \"${CMAKE_SOURCE_DIR}/include\" \"${CMAKE_SOURCE_DIR}/README.md\" +USE_MDFILE_AS_MAINPAGE = \"${CMAKE_SOURCE_DIR}/README.md\" +EXAMPLE_PATH = ${EXAMPLE_PATHS} +IMAGE_PATH = ${IMAGE_PATHS} +BUILTIN_STL_SUPPORT = YES +STRIP_CODE_COMMENTS = NO +SHOW_DIRECTORIES = YES +PROJECT_NUMBER = ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") endif() - add_custom_target(docs ${DOXYGEN_EXECUTABLE} ${DOXYCONF_OUT}) + add_custom_target(docs ${DOXYGEN_EXECUTABLE} ${DOXYCONF_OUT} + COMMAND lua gen_gh_pages.lua "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) else() - message("Doxygen was not found, the documentation pages will not be generated") + message("The optional documentation tools were not found; the doc target has not been created") endif() diff --git a/docs/heka/cli_tools.md b/docs/cli/index.md similarity index 70% rename from docs/heka/cli_tools.md rename to docs/cli/index.md index 0eec7dc..80ea510 100644 --- a/docs/heka/cli_tools.md +++ b/docs/cli/index.md @@ -1,12 +1,12 @@ -## Heka CLI Tools +# Command Line Interface Tools -### lsb_heka_cat +## lsb_heka_cat A command-line utility for counting, viewing, filtering, and extracting messages from a Heka protobuf log/stream. ``` -usage: src/cli/lsb_heka_cat [-t|-c|-h] [-m message_matcher] [-f] [-n #] +usage: lsb_heka_cat [-t|-c|-h] [-m message_matcher] [-f] [-n #] description: -t output the messages in text format (default) -c only output the message count @@ -19,5 +19,5 @@ notes: All output is written to stdout and all log/error messages are written to stderr. ``` -Set the [message matcher](message_matcher.md) documentation for more details about the message_matcher expression. +See the [message matcher](../util/message_matcher.html) documentation for more details about the message_matcher expression. diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index a37d683..76e58c9 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -1,11 +1,30 @@ -## Analysis Sandbox - -### Required Lua Functions (called by the host) - -#### process_message +# Analysis Sandbox Interface + +[Available Analysis Sandboxes](/lua_sandbox/sandboxes/heka/analysis/index.html) + +## Recommendations +Since he sandbox does not run in isolation there are some expectations of how +the host infrastructure behaves. The current recommendation are based on the +Hindsight reference implementation. + +## Disabled Functionality +- [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - collectgarbage, dofile, load, loadfile, loadstring, newproxy +- [coroutine](http://www.lua.org/manual/5.1/manual.html#5.2) + - the entire module is inaccessible +- [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - dump +- [io](http://www.lua.org/manual/5.1/manual.html#5.7) + - the entire module is inaccessible +- [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - getenv, execute, exit, remove, rename, setlocale, tmpname + +## Required Lua Functions (called by the host) + +### process_message Called when the host has a message available for analysis. Usually used in -combination with a [message matcher](message_matcher.md) expression. +combination with a [message matcher](../util/message_matcher.html) expression. Recommenation: specify this as a `message_matcher` configuration option. @@ -14,11 +33,11 @@ Recommenation: specify this as a `message_matcher` configuration option. *Return* * status_code (number) - - success (less than or equal to zero) - - fatal error (greater than zero) + * success (less than or equal to zero) + * fatal error (greater than zero) * status_message (optional: string) logged when the status code is less than zero -#### timer_event +### timer_event Called when the host timer expires or on shutdown. @@ -32,9 +51,9 @@ to keep the timestamp units consistent so it will only have a one second resolut *Return* * none -### Available C Functions (called from the plugin) +## Available C Functions (called from the plugin) -#### read_config +### read_config Provides access to the sandbox configuration variables. @@ -44,26 +63,26 @@ Provides access to the sandbox configuration variables. *Return* * value (string, number, bool, table) -#### read_message +### read_message Provides access to the Heka message data. Note that both fieldIndex and arrayIndex are zero-based (i.e. the first element is 0) as opposed to Lua's standard indexing, which is one-based. *Arguments* * variableName (string) - * framed (returns the Heka message protobuf string including the framing header) - * raw (returns the Heka message protobuf string) - * size (returns the size of the raw Heka message protobuf string) - * Uuid - * Type - * Logger - * Payload - * EnvVersion - * Hostname - * Timestamp - * Severity - * Pid - * Fields[*name*] + * framed (returns the Heka message protobuf string including the framing header) + * raw (returns the Heka message protobuf string) + * size (returns the size of the raw Heka message protobuf string) + * Uuid + * Type + * Logger + * Payload + * EnvVersion + * Hostname + * Timestamp + * Severity + * Pid + * Fields[*name*] * fieldIndex (unsigned) - only used in combination with the Fields variableName use to retrieve a specific instance of a repeated field *name*; zero indexed * arrayIndex (unsigned) - only used in combination with the Fields variableName @@ -72,17 +91,21 @@ Provides access to the Heka message data. Note that both fieldIndex and arrayInd *Return* * value (number, string, bool, nil depending on the type of variable requested) -#### decode_message +### decode_message -Converts a Heka protobuf encoded message string into a Lua table. +Converts a Heka protobuf encoded message string into a Lua table or throws an error. *Arguments* * heka_pb (string) - Heka protobuf binary string *Return* -* msg ( [Heka message table (array fields)](message.md#array-based-message-fields)) +* msg ([Heka message table (array fields)](message.html#array-based-message-fields)) + with the value member always being an array (even if there is only a single item). + This format makes working with the output more consistent. The wide variation + in the inject table formats is to ease the construction of the message especially + when using an LPeg grammar transformation. -#### inject_message +### inject_message Creates a new Heka protocol buffer message using the contents of the specified Lua table (overwriting whatever is in the payload buffer). `Logger` and `Hostname` are set by @@ -90,12 +113,12 @@ the infrastructure using the corresponding configuration setting and cannot be o by the plugin. *Arguments* -* msg ([Heka message table](message.md)) +* msg ([Heka message table](message.html)) *Return* * none (throws an error if the table does not match the Heka message schema) -#### add_to_payload +### add_to_payload Appends the arguments to the payload buffer for incremental construction of the final payload output (`inject_payload` finalizes the buffer and sends the message to the infrastructure). This function @@ -107,7 +130,7 @@ is a rename of the generic sandbox output function to improve the readability of *Return* * none (throws an error if arg is an unsupported type) -#### inject_payload +### inject_payload This is a wrapper function for `inject_message` that is included for backwards compatibility. The function creates a new Heka message using the contents of the payload buffer (pre-populated with `add_to_payload`) and combined @@ -139,12 +162,12 @@ Fields = { *Return* * none (throws an error if arg is an unsupported type) -#### Modes of Operation +### Modes of Operation -##### Lock Step +#### Lock Step * Receives one call to `process_message`, operates on the message, and returns success (0) or failure (-1) -##### Example simple counter plugin +#### Example simple counter plugin ```lua -- cfg message_matcher = "TRUE" diff --git a/docs/heka/heka_json.md b/docs/heka/heka_json.md index 9536e87..30cefc1 100644 --- a/docs/heka/heka_json.md +++ b/docs/heka/heka_json.md @@ -1,4 +1,4 @@ -## Heka JSON Module +# Heka JSON Module Allows for JSON-Schema validation and more efficient manipulation of large JSON structures where only a small amount of the data is consumed by the plugin. @@ -6,13 +6,13 @@ Schema pattern matching is restricted to a subset of regex decribed in the Regular Expression section of the [RapidJSON Schema Documentation](http://rapidjson.org/md_doc_schema.html). -### Availability +## Availability Input/Output plugins only. -### API +## API Functions -#### parse +### parse Creates a Heka JSON Document from a string. @@ -27,7 +27,7 @@ assert(ok, doc) *Return* * doc (userdata) - Heka JSON document or an error is thrown -#### parse_schema +### parse_schema Creates a Heka JSON Schema. @@ -42,7 +42,7 @@ assert(ok, doc) *Return* * schema (userdata) - Heka JSON schema or an error is thrown -#### parse_message +### parse_message Creates a Heka JSON Document from a message variable. @@ -67,9 +67,9 @@ assert(ok, doc) *Return* * doc (userdata) - Heka JSON document or an error is thrown -### Heka JSON Document API Methods +## Heka JSON Document API Methods -#### validate +### validate Checks that the JSON document conforms to the specified schema. @@ -85,7 +85,7 @@ assert(ok, err) * ok (bool) - true if valid * err (string) - error message on failure -#### find +### find Searches for and returns a value in the JSON structure. @@ -102,7 +102,7 @@ assert(v, "not found") *Return* * value (lightuserdata) - handle to be passed to other methods, nil if not found -#### remove +### remove Searches for and NULL's out the resulting value in the JSON structure returning the extracted Heka JSON document. @@ -121,7 +121,7 @@ rv:size() -- number of elements in the extracted array *Return* * doc (userdata) - the object removed from the original JSON or nil -#### value +### value Returns the primitive value of the JSON element. @@ -139,7 +139,7 @@ assert("bar" == str, tostring(str)) *Return* * primitive - string, number, bool, nil or throws an error if not convertable (object, array) -#### type +### type Returns the type of the value in the JSON structure. @@ -155,7 +155,7 @@ assert(t == "object", t) *Return* * type (string, nil) - "string", "number", "boolean", "object", "array" or "null" -#### iter +### iter Retrieves an interator function for an object/array. @@ -173,7 +173,7 @@ end * iter (function, nil) - iterator function returning an index/value for arrays or a key/value for objects. Throws an error on primitive types. -#### size +### size Returns the size of the value. ```lua @@ -189,7 +189,7 @@ local n = doc:size(v) * size (number, nil) - Number of element in an array/object or the length of the string. Throws an error on numeric, boolean and null types. -#### make_field +### make_field Helper function to wrap the lightuserdata so it can be used in an inject_message field. diff --git a/docs/heka/index.md b/docs/heka/index.md index 4e59df4..3ac826d 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -1,34 +1,22 @@ -# Heka Sandbox +# Heka Sandbox Interface ## Overview -This is the 1.0 release of the Heka sandbox API. It is a refined implementation -of its predecessor which was developed in [Heka](https://github.com/mozilla-services/heka). -The goal is to decople it from Go and make it easily embeddable in any language. -Although this could be ported back to the Heka Go implementation a new design -of that infrastructure has been created to replace it called +This document describes the pending 1.0 release of the Heka sandbox API built on +the [Generic Sandbox Interface](../sandbox.html). The 1.0 release is a refined +implementation of its predecessor which was developed in +[Heka](https://github.com/mozilla-services/heka). The goal is to decople it from +Go and make it easily embeddable in any language. We are in the process of +porting it back to the Heka Go implementation but a new design of that +infrastructure has been created to replace it called [Hindsight](https://github.com/trink/hindsight). -## Table of Contents - -* [Message Matcher](message_matcher.md) -* [Input Sandbox](input.md) - * [Heka Stream Reader](stream_reader.md) - * [Heka JSON](heka_json.md) - * [Heka Kafka Consumer](kafka_consumer.md) -* [Analysis Sandbox](analysis.md) -* [Output Sandbox](output.md) - * [Heka JSON](heka_json.md) - * [Heka Kafka Producer](kafka_producer.md) - ## Sandbox API Changes from the Go Heka Sandbox There are a few intentional changes between tho original Heka sandbox and this version. ### Changes -1. `read_message` now has a `framed` parameter to retrive the raw message with stream framing. - #### Input Sandbox 1. `inject_message` accepts a numeric or string checkpoint as the second argument @@ -41,19 +29,23 @@ There are a few intentional changes between tho original Heka sandbox and this v #### Analysis/Output Sandbox +1. `read_message` + * returns `nil` for optional header fields if they don't exist instead of an empty string or zero + * added a `framed` parameter to retrive the raw message with stream framing + * added a `size` parameter to retrive size of the raw message without having to copy it down 1. `timer_event` has a second parameter `shutdown` that is set to true when the sandbox is exiting. ### Additions #### Input Sandbox -1. A [Heka Stream Reader](heka_stream_reader.md) Lua module was added. -1. A [Heka JSON](heka_json.md) Lua module was added. +1. A [Heka Stream Reader](stream_reader.html) Lua module was added. +1. A [Heka JSON](heka_json.html) Lua module was added. #### Output Sandbox -1. [update_checkpoint](output.md#update_checkpoint) was added for batch and asynchronous processing. -1. A [Heka JSON](heka_json.md) Lua module was added. +1. [update_checkpoint](output.html#update_checkpoint) was added for batch and asynchronous processing. +1. A [Heka JSON](heka_json.html) Lua module was added. ### Removals diff --git a/docs/heka/input.md b/docs/heka/input.md index 272540d..6752444 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -1,13 +1,23 @@ -## Input Sandbox +# Input Sandbox Interface -### Recommendations +[Available Input Sandboxes](/lua_sandbox/sandboxes/heka/input/index.html) + +## Recommendations Since he sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the Hindsight reference implementation. -### Required Lua Functions (called by the host) +## Disabled Functionality +- [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - dofile, load, loadfile, loadstring, newproxy +- [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - dump +- [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - exit, setlocale + +## Required Lua Functions (called by the host) -#### process_message +### process_message Entry point for message creation. @@ -21,9 +31,9 @@ Entry point for message creation. - fatal error (greater than zero) * status_message (optional: string) logged when the status code is less than zero -### Available C Functions (called from the plugin) +## Available C Functions (called from the plugin) -#### read_config +### read_config Provides access to the sandbox configuration variables. @@ -33,35 +43,30 @@ Provides access to the sandbox configuration variables. *Return* * value (string, number, bool, table) -#### decode_message +### decode_message Converts a Heka protobuf encoded message string into a Lua table. +See [decode_message](analysis.html#decode_message) for details. -*Arguments* -* heka_pb (string) - Heka protobuf binary string - -*Return* -* msg ([Heka message table (array fields)](message.md#array-based-message-fields)) - -#### inject_message +### inject_message Sends a Heka protocol buffer message into the host. *Arguments* -* msg ([Heka message table](message.md), [Heka stream reader](stream_reader.md) or Heka protobuf string) +* msg ([Heka message table](message.html), [Heka stream reader](stream_reader.html) or Heka protobuf string) * checkpoint (optional: number, string) - checkpoint to be returned in the `process_message` call *Return* * none - throws an error on invalid input -### Modes of Operation +## Modes of Operation -#### Run Once +### Run Once * Set the `ticker_interval` to zero and return from `process_message` when you are done. * The `instruction_limit` configuration can be set if desired. -##### Example startup ping +#### Example startup ping ```lua -- cfg -- send a simple 'hello' messages every time the host is started @@ -81,14 +86,14 @@ end ``` -#### Polling +### Polling * Set the `ticker_interval` greater than zero and non fatally (<=0) return from `process_message`, when the ticker interval expires `process_message` will be called again. * The `instruction_limit` configuration can be set if desired. -##### Example startup ping +#### Example startup ping ```lua -- cfg ticker_interval = 60 @@ -120,12 +125,12 @@ end ``` -#### Continuous +### Continuous * Don't return from `process_message`. * The `instruction_limit` configuration **MUST** be set to zero. -##### Example of a Heka protobuf stdin reader +#### Example of a Heka protobuf stdin reader ```lua -- This Source Code Form is subject to the terms of the Mozilla Public @@ -157,4 +162,4 @@ function process_message() until read == 0 return 0 end -``` +``' diff --git a/docs/heka/kafka_consumer.md b/docs/heka/kafka_consumer.md index a11584e..5fe75fb 100644 --- a/docs/heka/kafka_consumer.md +++ b/docs/heka/kafka_consumer.md @@ -1,10 +1,10 @@ -## Heka Kafka Consumer +# Heka Kafka Consumer Kafka Lua module for input sandbox plugins. -### API +## Functions -#### new +### new Creates a Heka Kafka consumer. @@ -28,9 +28,9 @@ local consumer = heka_kafka_consumer.new(brokerlist, topics, consumer_conf, to *Return* * consumer (userdata) - Kafka consumer or an error is thrown -### API Methods +## Methods -#### receive +### receive Receives a message from the specified Kafka topic(s). diff --git a/docs/heka/kafka_producer.md b/docs/heka/kafka_producer.md index 623b5ae..036c57c 100644 --- a/docs/heka/kafka_producer.md +++ b/docs/heka/kafka_producer.md @@ -1,10 +1,10 @@ -## Heka Kafka Producer +# Heka Kafka Producer Kafka Lua module for output sandboxes. -### API +## Functions -#### new +### new Creates a Heka Kafka producer. @@ -28,9 +28,9 @@ local producer = heka_kafka_producer.new(brokerlist, producer_conf) *Return* * producer (userdata) - Kafka producer or an error is thrown -### API Methods +## Methods -#### create_topic +### create_topic Creates a topic to be used by a producer, no-op if the topic already exists. @@ -46,7 +46,7 @@ producer:create_topic(topic) -- creates the topic if it does not exist * none -#### has_topic +### has_topic Tests if a producer is managing a topic. @@ -62,7 +62,7 @@ local b = producer:has_topic(topic) * bool - True if the producer is managing a topic with the specificed name -#### destroy_topic +### destroy_topic Removes a topic from the producer. @@ -78,7 +78,7 @@ producer:destroy_topic(topic) * none - no-op on non-existent topic -#### send +### send Sends a message using the specified topic. @@ -101,7 +101,7 @@ local ret = producer:send(topic, -1, sequence_id, payload) - ESRCH (2) requested partition is unknown in the Kafka cluster - ENOENT (3) topic is unknown in the Kafka cluster -#### poll +### poll Polls the provided Kafka producer for events and invokes callback. This should be called after every send. diff --git a/docs/heka/message.md b/docs/heka/message.md index e6d651b..d8532fd 100644 --- a/docs/heka/message.md +++ b/docs/heka/message.md @@ -1,6 +1,6 @@ -## Heka Message Table +# Heka Message Table -### Hash Based Message Fields +## Hash Based Message Fields ```lua { @@ -20,7 +20,7 @@ Fields = { } ``` -### Array Based Message Fields +## Array Based Message Fields ```lua { -- Message headers are the same as above diff --git a/docs/heka/output.md b/docs/heka/output.md index c67041f..8dedc88 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -1,11 +1,26 @@ -## Output Sandbox +# Output Sandbox Interface -### Required Lua Functions (called by the host) +[Available Output Sandboxes](/lua_sandbox/sandboxes/heka/output/index.html) -#### process_message +## Recommendations +Since he sandbox does not run in isolation there are some expectations of how +the host infrastructure behaves. The current recommendation are based on the +Hindsight reference implementation. + +## Disabled Functionality +- [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - dofile, load, loadfile, loadstring, newproxy +- [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - dump +- [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - exit, setlocale + +## Required Lua Functions (called by the host) + +### process_message Called when the host has a message available for analysis. Usually used in -combination with a [message matcher](message_matcher.md) expression. +combination with a [message matcher](../util/message_matcher.html) expression. Recommenation: specify this as a `message_matcher` configuration option. @@ -15,16 +30,16 @@ Recommenation: specify this as a `message_matcher` configuration option. *Return* * status_code (number) - see the [Modes of Operation](#modes-of-operation) for a detail explanation of the return codes - - fatal error (greater than zero) - - success (0) - - non fatal failure (-1) - - skip (-2) - - retry (-3) - - batching (-4) - - async output (-5) + * fatal error (greater than zero) + * success (0) + * non fatal failure (-1) + * skip (-2) + * retry (-3) + * batching (-4) + * async output (-5) * status_message (optional: string) logged when the status code is -1 -#### timer_event +### timer_event Called when the host timer expires or on shutdown. @@ -38,9 +53,9 @@ to keep the timestamp units consistent so it will only have a one second resolut *Return* * none -### Available C Functions (called from the plugin) +## Available C Functions (called from the plugin) -#### read_config +### read_config Provides access to the sandbox configuration variables. @@ -50,21 +65,16 @@ Provides access to the sandbox configuration variables. *Return* * value (string, number, bool, table) -#### read_message +### read_message -Provides access to the Heka message data. See [read_message](analysis.md#read_message) for details. +Provides access to the Heka message data. See [read_message](analysis.html#read_message) for details. -#### decode_message +### decode_message Converts a Heka protobuf encoded message string into a Lua table. +See [decode_message](analysis.html#decode_message) for details. -*Arguments* -* heka_pb (string) - Heka protobuf binary string - -*Return* -* msg ([Heka message table (array fields)](heka_message_table.md#array-based-message-fields)) or an error is thrown - -#### encode_message +### encode_message Returns a Heka protocol buffer message using the contents of the specified Lua table. Note: this operation uses the internal output buffer so it is goverened by the @@ -72,31 +82,31 @@ Note: this operation uses the internal output buffer so it is goverened by the be added by infrastructure using the corresponding configuration setting. *Arguments* -* msg ([Heka message table](heka_message_table.md) +* msg ([Heka message table](message.html)) * framed (bool default: false) A value of true includes the framing header *Return* * heka_pb (string) - Heka protobuf binary string, framed as specified or an error is thrown -#### create_message_matcher +### create_message_matcher Returns a Heka protocol buffer message matcher; used to dynamic filter messages sent to the output plugin. *Arguments* -* message_matcher [message matcher](message_matcher.md) +* message_matcher [message matcher](../util/message_matcher.html) *Return* * message_matcher (userdata) - or an error is thrown The message matcher object has one method `eval` that returns true if the current message matches, false if it does not. -##### Example +#### Example -See: [heka_tcp_matcher.lua](../../sandboxes/heka/output/heka_tcp_matcher.lua) +See: [heka_tcp_matcher.lua](https://github.com/mozilla-services/lua_sandbox/blob/master/sandboxes/heka/output/heka_tcp_matcher.lua) -#### update_checkpoint +### update_checkpoint -##### Batch Mode +#### Batch Mode Advances the output checkpoint when in batching mode. The standard use case is to call it from `timer_event` after successfully flushing a batch on timeout/shutdown. @@ -104,7 +114,7 @@ timeout/shutdown. *Arguments* * none -##### Asynchronous Mode +#### Asynchronous Mode Advances the output checkpoint and optionally reports the number of failures that occured. *Arguments* @@ -114,22 +124,22 @@ Advances the output checkpoint and optionally reports the number of failures tha *Return* * none (throws an error on invalid arg types) -#### Modes of Operation +### Modes of Operation -##### Lock Step +#### Lock Step * `process_message` operates on the message and returns one of the following values: - * success (0) - the message was successfully processed and the output checkpoint is advanced - * failure (-1) - the message was not successfully processed - * the failure count is incremented - * any optional error message is written to the log - * the message is skipped - * the checkpoint is advanced - * skip (-2) - the message was intentionally not processed and the checkpoint is advanced - * retry (-3) - the message was not successfully processed and the host will call `process_message` + * success (0) - the message was successfully processed and the output checkpoint is advanced + * failure (-1) - the message was not successfully processed + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * the checkpoint is advanced + * skip (-2) - the message was intentionally not processed and the checkpoint is advanced + * retry (-3) - the message was not successfully processed and the host will call `process_message` again, with the same message, after a one second delay -##### Example Payload Output +#### Example Payload Output ```lua -- cfg @@ -178,20 +188,20 @@ function timer_event(ns) end ``` -##### Batching +#### Batching * `process_message` batches the message/transformation in memory or on disk and returns one of the following values: - * batching (-4) - the message was successfully added to the batch - * failure (-1) - the message cannot be batch - * the failure count is incremented - * any optional error message is written to the log - * the message is skipped - * skip (-2) - the message was intentionally not added to the batch - * retry (-3) - the message was not successfully added to the batch and the host will call `process_message` - again, with the same message, after a one second delay - * success (0) - the batch has been successfully committed and the output checkpoint is advanced to the most recent message - -##### Example Postgres Output + * batching (-4) - the message was successfully added to the batch + * failure (-1) - the message cannot be batch + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * skip (-2) - the message was intentionally not added to the batch + * retry (-3) - the message was not successfully added to the batch and the host will call `process_message` + again, with the same message, after a one second delay + * success (0) - the batch has been successfully committed and the output checkpoint is advanced to the most recent message + +#### Example Postgres Output ```lua -- cfg @@ -357,7 +367,7 @@ function timer_event(ns, shutdown) end ``` -##### Asynchronous +#### Asynchronous * `async_buffer_size` **RECOMMENDED** that this configuration variable be set and consumed by the host * `process_message` is called with a sequence_id parameter and asynchronously sends the message/transformation @@ -373,7 +383,7 @@ to the destination and returns one of the following values: * When an asynchronously sent message is acknowledged [update_checkpoint](#update_checkpoint) **MUST** be called to advance the checkpoint to that specific message -##### Example Kafka Output +#### Example Kafka Output ```lua -- cfg diff --git a/docs/heka/stream_reader.md b/docs/heka/stream_reader.md index 0edfdfb..71d8e5c 100644 --- a/docs/heka/stream_reader.md +++ b/docs/heka/stream_reader.md @@ -1,11 +1,11 @@ -## Heka Stream Reader Module +# Heka Stream Reader Module Enables parsing of a framed Heka protobuf stream in a Lua sandbox. See: -[Example of a Heka protobuf reader](input.md#example-of-a-heka-protobuf-stdin-reader) +[Example of a Heka protobuf reader](input.html#example-of-a-heka-protobuf-stdin-reader) -### API +## Functions -#### new +### new Creates a Heka stream reader. @@ -20,9 +20,9 @@ local hsr = heka_stream_reader.new("stdin") *Return* * hsr (userdata) - Heka stream reader or an error is thrown -### API Methods +## Methods -#### find_message +### find_message Locates a Heka message within the stream. @@ -41,10 +41,10 @@ local found, consumed, need = hsr:find_message(buf) * need/read (number) - number of bytes needed to complete the message or fill the underlying buffer or in the case of a file object the number of bytes added to the buffer -#### decode_message +### decode_message -Converts a Heka protobuf encoded message string into a stream reader representation. Note: this operation -clears the internal stream reader buffer. +Converts a Heka protobuf encoded message string into a stream reader representation. +Note: this operation clears the internal stream reader buffer. *Arguments* * heka_pb (string) - Heka protobuf binary string @@ -52,7 +52,7 @@ clears the internal stream reader buffer. *Return* * none - throws an error on failure -#### read_message +### read_message Provides access to the Heka message data within the reader object. @@ -60,4 +60,4 @@ Provides access to the Heka message data within the reader object. local ts = hsr:read_message("Timestamp") ``` -See [read_message](analysis.md#read_message) for details. +See [read_message](analysis.html#read_message) for details. diff --git a/docs/index.md b/docs/index.md index 3448f4f..f0340dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,16 +1,27 @@ -# Library Components +# Overview + +Sandboxes provide a dynamic and isolated execution environment +for data parsing, transformation, and analysis. They allow access to data +without jeopardizing the integrity or performance of the processing +infrastructure. This broadens the audience that the data can be +exposed to and facilitates new uses of the data (i.e. debugging, monitoring, +dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, +etc.) + +The Lua sandbox is a library allowing customized control over the Lua execution +environment including functionality like global data preservation/restoration on +shutdown/startup, output collection in textual or binary formats and an array of +parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) -## Overview These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/trink/hindsight). The goal was to decouple the Heka/Hindsight functionality from any particular infrastructure and make it embeddable into any tool or language. -## Table of Contents - -### Sandbox User APIs -* [Sandbox](sandbox.md) -* [Heka Sandbox](heka/index.md) - -### Sandbox Developer APIs -* todo - Links to the Doxygen output +## Features +- small - memory requirements are as little as 8 KiB for a basic sandbox +- fast - microsecond execution times +- stateful - ability to resume where it left off after a restart/reboot +- isolated - failures are contained and malfunctioning sandboxes are terminated. + Containment is defined in terms of restriction to the operating system, + file system, libraries, memory use, Lua instruction use, and output size. diff --git a/docs/sandbox.md b/docs/sandbox.md index b873814..0724fdb 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -1,7 +1,6 @@ -Sandbox API ------------ +# Generic Sandbox Interface -### Configuration +## Configuration * **output_limit** - the largest output string an input or analysis plugin can inject into the host (bytes, default 64KiB) * **memory_limit** - the maximum amount of memory a plugin can use before being terminated (bytes, default 8MiB) @@ -24,25 +23,32 @@ remove_entries = { * **log_level** - Integer specifying the syslog severity level, when set to debug (7) the print function will be wired to the specified logger * *user defined* any other variable (string, bool, number, table) is passed through as-is and available via [read_config](#read_config) -### Lua functions exposed to C by the core sandbox +## Lua functions exposed to C by the core sandbox There are no functions exposed by default, see lsb_pcall_setup() and lsb_pcall_teardown() when defining an API. -### Functions exposed to Lua by the core sandbox +## Functions exposed to Lua by the core sandbox -#### require +### require By default only the base library is loaded additional libraries must be loaded with require(). *Arguments* - libraryName (string) + +*Return* +- a table - For non user provided libraries the table is also globally registered + with the library name. User provided libraries may implement there own semantics + i.e. the grammar libraries return a table but do not globally register the table name + (see the individual module documentation for the correct usage). + +*Notes* + +The following modules have been modified, as described, for use in the sandbox. - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) - The require() function has been modified to not expose any of the package table to the sandbox. - - Disabled functions (default): collectgarbage, coroutine, dofile, load, loadfile, loadstring, newproxy, print. - - [bloom_filter](https://github.com/mozilla-services/lua_bloom_filter/blob/master/README.md) Test whether an element is a member of a set - - [circular_buffer](https://github.com/mozilla-services/lua_circular_buffer/blob/master/README.md) In memory time series data base and analysis - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) JSON parser with the following modifications: - Loads the cjson module in a global cjson table - The encode buffer is limited to the sandbox output_limit. @@ -51,37 +57,10 @@ By default only the base library is loaded additional libraries must be loaded w If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - The new() function has been disabled so only a single cjson parser can be created. - The encode_keep_buffer() function has been disabled (the buffer is always reused). - - [cuckoo_filter](https://github.com/mozilla-services/lua_cuckoo_filter/blob/master/README.md) Bloom filter alternative supporting deletions - - [hyperloglog](https://github.com/mozilla-services/lua_hyperloglog/blob/master/README.md) Efficiently count the number of elements in a set - - [lpeg](http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html) Lua Parsing Expression Grammar Library - - [re](http://www.inf.puc-rio.br/~roberto/lpeg/re.html) Regex syntax for LPEG - - [math](http://www.lua.org/manual/5.1/manual.html#5.6) - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. - - Disabled functions (default): getenv, execute, exit, remove, rename, setlocale, tmpname. - - [sax](https://github.com/trink/symtseries/blob/master/README.md) Symbolic time series data analysis based on - [Symbolic Aggregate approXimation](http://www.cs.ucr.edu/~eamonn/SAX.pdf). - - [string](http://www.lua.org/manual/5.1/manual.html#5.4) - - [struct](http://www.inf.puc-rio.br/~roberto/struct/) Converts data to/from C structs - - [table](http://www.lua.org/manual/5.1/manual.html#5.5) - - Grammar Libraries - - cbufd - Parses the circular buffer library delta output (use for data aggregation) - - common_log_format - Nginx and Apache meta grammar generators (creates a grammar based on the log_format configuration) - - data_time - RFC3339, RFC3164, strftime, common log format, MySQL and Postgres timestamps - - ip_address - IPv4 and IPv6 address - - mysql - MySQL and MariaDB slow query and short slow query parsers - - postfix - Postfix messages - - syslog - Rsyslog meta grammar generator (creates a grammar based on the template configuration) - - syslog_message - Syslog messages - - _user provided_ (lua, so/dll) - -*Return* -- a table - For non user provided libraries the table is also globally registered - with the library name. User provided libraries may implement there own semantics - i.e. the grammar libraries return a table but do not globally register the table name - (see the individual module documentation for the correct usage). -#### read_config +### read_config Provides access to the sandbox configuration variables. @@ -91,7 +70,7 @@ Provides access to the sandbox configuration variables. *Return* * value (string, number, bool, table) -#### output +### output Receives any number of arguments and appends data to the output buffer, which cannot exceed the output_limit configuration parameter. See lsb_get_output() to connect the output to the host application. @@ -103,7 +82,7 @@ connect the output to the host application. *Return* - none -#### print +### print Receives any number of arguments and sends a debug message to the host's logger function as a tab delimited message. The function clears and then uses the output buffer so if pending output has been queued and not flushed it will be @@ -125,7 +104,7 @@ The best place to start is with some examples: ### Unit Test API -[Unit Test Source Code](../src/test/test_luasandbox.c) +[Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_luasandbox.c) #### Lua Functions Exposed to C @@ -142,5 +121,5 @@ The best place to start is with some examples: ### Heka Sandbox -[Heka Sandbox](heka/index.md) +[Heka Sandbox Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/include/luasandbox/heka/sandbox.h) diff --git a/docs/heka/message_matcher.md b/docs/util/message_matcher.md similarity index 70% rename from docs/heka/message_matcher.md rename to docs/util/message_matcher.md index c73211e..2dc0fe2 100644 --- a/docs/heka/message_matcher.md +++ b/docs/util/message_matcher.md @@ -1,9 +1,9 @@ -### Message Matcher Syntax +# Message Matcher Syntax The message matcher allows sandboxes to select which messages they want to consume -(see [Heka Message Structure](message.md)) +(see [Heka Message Structure](../heka/message.html)) -#### Examples +## Examples * Type == "test" && Severity == 6 * (Severity == 7 || Payload == "Test Payload") && Type == "test" @@ -13,38 +13,40 @@ The message matcher allows sandboxes to select which messages they want to consu * TRUE * Fields[created] =~ "^2015" * Fields[widget] != NIL +* Timestamp >= "2016-05-24T00:00:00Z" +* Timestamp >= 1464048000000000000 -#### Relational Operators +## Relational Operators * == equals * != not equals -* > greater than -* >= greater than equals +* > greater than +* >= greater than equals * < less than * <= less than equals * =~ Lua pattern match * !~ Lua negated pattern match -#### Logical Operators +## Logical Operators * Parentheses are used for grouping expressions * && and (higher precedence) * || or -#### Boolean +## Boolean * TRUE * FALSE -#### Constants +## Constants * NIL used to test the existence (!=) or non-existence (==) of a field variable -#### Message Variables +## Message Variables All message variables must be on the left hand side of the relational comparison -##### String +### String * Uuid - 16 byte raw binary type 4 UUID (useful for partitioning data) * Type @@ -53,30 +55,30 @@ All message variables must be on the left hand side of the relational comparison * EnvVersion * Hostname -##### Numeric +### Numeric -* Timestamp +* Timestamp - in addition to nanoseconds since the UNIX epoch an RFC3339 string is also accepted e.g., "2016-05-24T21:51:00Z" * Severity * Pid -##### Fields +### Fields * Fields[_field_name_] - shorthand for Field[_field_name_][0][0] * Fields[_field_name_][_field_index_] - shorthand for Field[_field_name_][_field_index_][0] * Fields[_field_name_][_field_index_][_array_index_] the indices are restricted to 0-255 * If a field type is mis-match for the relational comparison, false will be returned e.g., Fields[foo] == 6 where "foo" is a string -#### Quoted String +## Quoted String * Single or double quoted strings are allowed * The maximum string length is 255 bytes -#### Lua Pattern Matching Expression +## Lua Pattern Matching Expression * Patterns are quoted string values * See [Lua Patterns](http://www.lua.org/manual/5.1/manual.html#5.4.1) * Capture groups are ignored -#### Additional Restrictions +## Additional Restrictions -* Message matchers are restricted to 128 tests +* Message matchers are restricted to 128 relational comparisons diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua new file mode 100644 index 0000000..75447fa --- /dev/null +++ b/gen_gh_pages.lua @@ -0,0 +1,297 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "os" +require "io" +require "string" +require "table" + +local function get_path(s) + return s:match("(.+)/[^/]-$") +end + + +local function get_filename(s) + return s:match("/([^/]-)$") +end + + +local function strip_ext(s) + return s:sub(1, #s - 4) +end + + +local function sort_entries(t) + local a = {} + for n in pairs(t) do table.insert(a, n) end + table.sort(a) + local i = 0 -- iterator variable + local iter = function () -- iterator function + i = i + 1 + if a[i] == nil then + return nil + else + return a[i], t[a[i]] + end + end + return iter +end + + +local function create_index(path, dir) + local fh = assert(io.open("gh-pages/" .. path .. "/index.md", "w")) + fh:write(string.format("# %s\n", path)) + + for k,v in sort_entries(dir.entries) do + if k:match(".lua$") then + fh:write(string.format("* [%s](%s.html) - %s\n", k, strip_ext(k), v.title)) + else + fh:write(string.format("* [%s](%s/index.html)\n", k, k)) + end + end + + fh:close() +end + + +local function output_tree(fh, list, key, dir) + list[#list + 1] = key + local path = table.concat(list, "/") + create_index(path, dir) + if #list == 1 then fh:write("
    \n") end + fh:write(string.format('
  • %s
    • \n', path, key)) + + fh:write("
      \n") + for k, v in sort_entries(dir.entries) do + if k:match(".lua$") then + fh:write(string.format('
    • %s
      • \n', strip_ext(v.line), strip_ext(k))) + else + output_tree(fh, list, k, v) + end + end + fh:write("
    \n") + + if #list == 1 then fh:write("
\n") end + table.remove(list) +end + + +local function output_css() + local fh = assert(io.open("gh-pages/docs.css", "w")) + fh:write([[ + html { + height: 100%; + } + + body { + font-family:verdana, arial, sans-serif; + font-size:small; + width: 90%; + background:white; + margin-left: auto; + margin-right: auto; + height: 100%; + } + + h1 { + border-bottom:1px black solid; + } + + h2 { + border-bottom:1px gray solid; + } + + h3 { + border-bottom:1px lightgray solid; + } + + h4 { + border-bottom:1px black dotted; + } + + h5 { + border-bottom:1px gray dotted; + } + + h6 { + border-bottom:1px lightgray dotted; + } + + #title { + width:100%; + font-size:large; + font-weight: bold; + font-style: normal; + font-variant: normal; + text-transform: uppercase; + letter-spacing: .1em; + } + + .menu { + display:table-cell; + font-size: small; + font-weight: normal; + font-style: normal; + font-variant: small-caps; + color: #000000; + height: 100%; + padding-right: 10px; + white-space: nowrap; + } + + .menu ul{ + list-style-type: none; + margin-left: 5px; + margin-right: 0px; + padding-left: 10px; + padding-right: 0px; + } + + .main-content { + border-left:1px lightgray dotted; + padding-left:10px; + display:table-cell; + width:100%; + } + + code, pre.code, pre.sourceCode + { + background-color: whitesmoke; + } + ]]) + fh:close() +end + + +local function output_menu(before, after, paths, version) + local fh = assert(io.open(before, "w")) + fh:write(string.format('
Lua Sandbox Library (%s)
\n', version)) + fh:write([[
+ + + + + + + ]]) + + output_tree(fh, {}, "modules", paths.entries.modules) + output_tree(fh, {}, "sandboxes", paths.entries.sandboxes) + + fh:write([[ +
+
+]]) + fh:close() + + fh = assert(io.open(after, "w")) + fh:write("
\n") + fh:close() +end + + +local function handle_path(paths, in_path, out_path) + local list = {} + local d = paths + for dir in string.gmatch(out_path, "[^/]+") do + list[#list + 1] = dir + if dir ~= "gh-pages" then + local full_path = table.concat(list, "/") + local cd = d.entries[dir] + if not cd then + os.execute(string.format("mkdir -p %s", full_path)) + local nd = {path = in_path, entries = {}} + d.entries[dir] = nd + d = nd + else + d = cd + end + end + end + return d +end + + +local function extract_lua_docs(paths) + local fh = assert(io.popen("find sandboxes modules -name \\*.lua")) + for line in fh:lines() do + local sfh = assert(io.open(line)) + local lua = sfh:read("*a") + sfh:close() + + local doc = lua:match("%-%-%[%[%s*(.-)%-%-%]%]") + local title = lua:match("#%s(.-)\n") + if not title then error("doc error, no title: " .. line) end + + local outfn = string.gsub("gh-pages/" .. line, "lua$", "md") + local p = handle_path(paths, get_path(line), get_path(outfn)) + local ofh = assert(io.open(outfn, "w")) + p.entries[get_filename(line)] = {line = line, title = title} + ofh:write(doc) + ofh:write(string.format("\n\nsource code: [%s](https://github.com/mozilla-services/lua_sandbox/blob/master/%s)\n", get_filename(line), line)) + ofh:close() + end + fh:close() +end + + +local function md_to_html(paths, version) + local before = "/tmp/before.html" + local after = "/tmp/after.html" + output_menu(before, after, paths, version) + + local fh = assert(io.popen("find gh-pages -name \\*.md")) + for line in fh:lines() do + local css_path = "/lua_sandbox/docs.css" + local cmd = string.format("pandoc --from markdown_github-hard_line_breaks --to html --standalone -B %s -A %s -c %s -o %s.html %s", before, after, css_path, line:sub(1, #line -3), line) + local rv = os.execute(cmd) + if rv ~= 0 then error(cmd) end + os.remove(line) + end + fh:close() + + os.remove(before) + os.remove(after) +end + + +local args = {...} +local function main() + local rv = os.execute("rsync -rav docs/ gh-pages/") + if rv ~= 0 then error"rsync" end + output_css() + local paths = {entries = {}} + extract_lua_docs(paths) + md_to_html(paths, args[1]) +end + +main() diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index d0b88a3..876d068 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -216,7 +216,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_heka_update_checkpoint ucp); /** - * Host access to the output sandobx process_message API + * Host access to the output sandbox process_message API * * @param hsb Heka output sandbox * @param msg Heka message to process diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index 52e4029..859c2b5 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -4,7 +4,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/** @brief Lua sandbox output buffer functions @file */ +/** @brief Lua sandbox output generation/retrieval functions @file */ #ifndef luasandbox_output_h_ #define luasandbox_output_h_ @@ -12,7 +12,6 @@ #include #include "luasandbox.h" -#include "luasandbox/util/output_buffer.h" #ifdef __cplusplus extern "C" { diff --git a/include/luasandbox_serialize.h b/include/luasandbox_serialize.h index bb9d95d..d7d8209 100644 --- a/include/luasandbox_serialize.h +++ b/include/luasandbox_serialize.h @@ -11,6 +11,7 @@ #include +#include "luasandbox/util/output_buffer.h" #include "luasandbox_output.h" #ifdef __cplusplus diff --git a/modules/heka/elasticsearch.lua b/modules/heka/elasticsearch.lua index 5fe13b3..62af0c5 100644 --- a/modules/heka/elasticsearch.lua +++ b/modules/heka/elasticsearch.lua @@ -3,9 +3,9 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -## Elasticsearch Utility Functions +# Elasticsearch Utility Functions -### Module Configuration Table (common options) +## Module Configuration Table (common options) ```lua -- Boolean flag, if true then any time interpolation (often used to generate the -- ElasticSeach index) will use the timestamp from the processed message rather @@ -26,9 +26,9 @@ id = nil -- optional, default shown ``` -### API +## Functions -#### bulkapi_index_json(index, type_name, id, ns) +### bulkapi_index_json Returns a simple JSON 'index' structure satisfying the [ElasticSearch BulkAPI](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html) @@ -46,7 +46,7 @@ Returns a simple JSON 'index' structure satisfying the [ElasticSearch BulkAPI](h * JSON - String suitable for use as ElasticSearch BulkAPI index directive. *See* -[Field Interpolation](msg_interpolate.lua#L11) +[Field Interpolation](msg_interpolate.html) --]] local cjson = require "cjson" diff --git a/modules/heka/elasticsearch/moz_telemetry.lua b/modules/heka/elasticsearch/moz_telemetry.lua deleted file mode 100644 index bee2f4c..0000000 --- a/modules/heka/elasticsearch/moz_telemetry.lua +++ /dev/null @@ -1,133 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -## Elasticsearch Encoder for Unified Telemetry Messages - -### Module Configuration Table - -[Common Options](../elasticsearch.lua#8) -```lua --- Array of Heka message field names that should be passed to Elasticsearch. -fields = {"Payload", "Fields[docType]"} -- required -``` -### Sample Output -```json -{"index":{"_index":"mylogger-2014.06.05","_type":"mytype-host.domain.com"}} -{"Payload":"data","docType":"main"} -``` ---]] - --- Imports -local cjson = require "cjson" -local string = require "string" -local os = require "os" -local math = require "math" -local mi = require "heka.msg_interpolate" -local es = require "heka.elasticsearch" -local hj = require "heka_json" -local ipairs = ipairs -local pcall = pcall -local read_message = read_message -local cfg = es.load_encoder_cfg() - -if not cfg.fields or type(cfg.fields) ~= "table" or #cfg.fields == 0 then - error("fields must be specified") -end - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local info_fields = { - "sessionId" - , "subsessionId" - , "previousSessionId" - , "previousSubsessionId" - , "subsessionCounter" - , "profileSubsessionCounter" - , "sessionStartDate" - , "subsessionStartDate" - , "subsessionLength" - , "sessionLength" -} - -local environment_fields = { - "telemetryEnabled" -} - -local static_fields = {} -local dynamic_fields = {} - -local function key(str) - return str:match("^Fields%[(.+)%]$") or error("invalid field name: " .. str) -end - -for i, field in ipairs(cfg.fields) do - local fp = mi.header_fields[field] - if fp then - static_fields[#static_fields+1] = {field, fp} - else - dynamic_fields[#dynamic_fields+1] = {field, key(field)} - end -end - -function encode() - local ns - if cfg.es_index_from_timestamp then ns = read_message("Timestamp") end - - local idx_json = es.bulkapi_index_json(cfg.index, cfg.type_name, cfg.id, ns) - - local tbl = {} - for i, field in ipairs(static_fields) do - tbl[field[1]] = field[2]() - end - - for i, field in ipairs(dynamic_fields) do - local full_name = field[1] - local short_name = field[2] - local z = 0 - local v = read_message(full_name, nil, z) - while v do - if z == 0 then - tbl[short_name] = v - elseif z == 1 then - tbl[short_name] = {tbl[short_name], v} - elseif z > 1 then - tbl[short_name][z+1] = v - end - z = z + 1 - v = read_message(full_name, nil, z) - end - end - - local ok, doc = pcall(hj.parse_message, "Fields[payload.info]") - if ok then - for i, k in ipairs(info_fields) do - tbl[k] = doc:value(doc:find(k)) - end - end - - ok, doc = pcall(hj.parse_message, "Fields[environment.settings]") - if ok then - for i, k in ipairs(environment_fields) do - tbl[k] = doc:value(doc:find(k)) - end - end - - ok, doc = pcall(hj.parse_message, "Payload") - if ok then - tbl.architecture = doc:value(doc:find("application", "architecture")) - end - - if tbl.creationTimestamp then - if ns then - tbl.Latency = math.floor((ns - tbl.creationTimestamp) / 1e9) - end - tbl.creationTimestamp = os.date("!%Y-%m-%dT%H:%M:%SZ", tbl.creationTimestamp / 1e9) - end - - return string.format("%s\n%s\n", idx_json, cjson.encode(tbl)) -end - -return M diff --git a/modules/heka/elasticsearch/payload.lua b/modules/heka/elasticsearch/payload.lua index 70fc58c..24740cd 100644 --- a/modules/heka/elasticsearch/payload.lua +++ b/modules/heka/elasticsearch/payload.lua @@ -3,15 +3,29 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -## Elasticsearch Encoder for Heka Payload-only Messages +# Elasticsearch Encoder for Heka Payload-only Messages + The message payload must be pre-formatted JSON in an ElasticSearch compatible format. -### Module Configuration Table +## Module Configuration Table + +[Common Options](../elasticsearch.html) + +## Functions + +### encode + +Creates the ElasticSearch bulk API index JSON and combines it with the +pre-formatted JSON from the message payload (a new line is added if necessary). + +*Arguments* +- none -[Common Options](../elasticsearch.lua#8) +*Return* +- JSON (string) -### Sample Output +## Sample Output ```json {"index":{"_index":"mylogger-2014.06.05","_type":"mytype-host.domain.com"}} {"json":"data","extracted":"from","message":"payload"} diff --git a/modules/heka/msg_interpolate.lua b/modules/heka/msg_interpolate.lua index 46e6d67..6631228 100644 --- a/modules/heka/msg_interpolate.lua +++ b/modules/heka/msg_interpolate.lua @@ -3,11 +3,11 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -## Simple Templating to Transform Message Fields into Strings +# Simple Templating to Transform Message Fields into Strings -### API +## Functions -#### interpolate(value, secs) +### interpolate Interpolates values from the currently processed message into the provided string value. A `%{}` enclosed field name will be replaced by the field value @@ -15,7 +15,7 @@ from the current message. All message header fields are supported ("Uuid", "Timestamp", "Type", "Logger", "Severity", "Payload", "EnvVersion", "Pid", "Hostname"). Any other values will be checked against the defined dynamic message fields. If no field matches, then a -[C strftime](http://man7.org/linux/man-pages/man3/strftime.3.html) (*nix) +[C strftime](http://man7.org/linux/man-pages/man3/strftime.3.html) (\*nix) or [C89 strftime](http://msdn.microsoft.com/en-us/library/fe06s4ak.aspx) (Windows) time substitution will be attempted. The time used for time substitution will be diff --git a/modules/util.lua b/modules/heka/util.lua similarity index 60% rename from modules/util.lua rename to modules/heka/util.lua index 7b154c2..8d209c5 100644 --- a/modules/util.lua +++ b/modules/heka/util.lua @@ -2,6 +2,27 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +--[[ +# Heka Sandbox Utility Module + +## Functions + +### table_to_fields + +Flattens a Lua table so it can be encoded as a protobuf fields object. + +*Arguments* +- hash (table) - table to flatten (not modified) +- fields (table) - table to receive the flattened output +- parent (string) - key prefix +- separator (string) - key separator (default = ".") i.e. 'foo.bar' +- max_depth (number) - maximum nesting before converting the remainder of the + structure to a JSON string + +*Return* +- none - in-place modification of `fields` +--]] + -- Imports local pairs = pairs local type = type @@ -11,7 +32,6 @@ local cjson = require "cjson" local M = {} setfenv(1, M) -- Remove external access to contain everything in the module --- Flattens a Lua table so it can be encoded as a protobuf fields object. function table_to_fields(t, fields, parent, char, max_depth) if type(char) ~= "string" then char = "." @@ -41,25 +61,4 @@ function table_to_fields(t, fields, parent, char, max_depth) end end --- Effectively removes all array values up to the provided index from an array --- by copying end values to the array head and setting now unused entries at --- the end of the array to `nil`. -function behead_array(idx, array) - if idx <= 1 then return end - local array_len = #array - local start_nil_idx = 1 -- If idx > #array we zero it out completely. - if idx <= array_len then - -- Copy values to lower indexes. - local difference = idx - 1 - for i = idx, array_len do - array[i-difference] = array[i] - end - start_nil_idx = array_len - difference + 1 - end - -- Empty out the end of the array. - for i = start_nil_idx, array_len do - array[i] = nil - end -end - return M diff --git a/modules/cbufd.lua b/modules/lpeg/cbufd.lua similarity index 55% rename from modules/cbufd.lua rename to modules/lpeg/cbufd.lua index 1424f63..0117ab9 100644 --- a/modules/cbufd.lua +++ b/modules/lpeg/cbufd.lua @@ -2,6 +2,47 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +--[[ +# Circular Buffer Delta Module + +## Variables + +* `grammar` - LPEG grammar to parse cbufd output + +## Usage +```lua +local cbufd = require "cbufd".grammar + +function process_message() + local payload = read_message("Payload") + local t = cbufd:match(payload) + if not t then + return -1, "invalid cbufd string" + end + -- use t + return 0 +end +``` + +## Sample Input/Output + +### Circular Buffer Delta Input +``` +{"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]} +1379660520 12075 159901 +1379660280 11837 154880 +``` + +### Lua Table Output +```lua +{ +header='{"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]}', +{12075, 159901, time = 1379660520000000000}, +{11837, 154880, time = 1379660280000000000} +} +``` +--]] + -- Imports local l = require "lpeg" l.locale(l) @@ -10,26 +51,6 @@ local tonumber = tonumber local M = {} setfenv(1, M) -- Remove external access to contain everything in the module ---[[ cbufd grammar -sample input: -{"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]} -1379660520 12075 159901 -1379660280 11837 154880 - -output table: -1 - 1=12075 (number) - 2=159901 (number) - time=1379660520000000000 (number) - -2 - 1=11837 (number) - 2=154880 (number) - time=1379660280000000000 (number) - -header={"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]} ---]] - local function not_a_number() return 0/0 end diff --git a/modules/common_log_format.lua b/modules/lpeg/common_log_format.lua similarity index 94% rename from modules/common_log_format.lua rename to modules/lpeg/common_log_format.lua index 5d40952..3da41ee 100644 --- a/modules/common_log_format.lua +++ b/modules/lpeg/common_log_format.lua @@ -2,12 +2,54 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +--[[ +# Common Log Format Module + +## Variables + +* `nginx_error_grammar` - LPEG grammar to parse an Nginx error log line + +## Functions + +### build_nginx_grammar + +Constructs an LPEG grammar based on the Nginx log_format configuration string. + +*Arguments* +- log_format (string) - Nginx log_format configuration string + +*Return* +- grammar (LPEG user data object) or an error is thrown + +### build_apache_grammar + +Constructs an LPEG grammar based on the Apache log_format configuration string. + +*Arguments* +- log_format (string) - Apache log_format configuration string + +*Return* +- grammar (LPEG user data object) or an error is thrown + +### normalize_user_agent + +Extracts the browser, version and os information from a user agent string. + +*Arguments* +- ua (string) - user agent string + +*Return* +- browser (string) +- version (number) +- os (string) +--]] + -- Imports local l = require "lpeg" l.locale(l) local string = require "string" -local dt = require "date_time" -local ip = require "ip_address" +local dt = require "lpeg.date_time" +local ip = require "lpeg.ip_address" local tonumber = tonumber local ipairs = ipairs local pairs = pairs diff --git a/modules/date_time.lua b/modules/lpeg/date_time.lua similarity index 84% rename from modules/date_time.lua rename to modules/lpeg/date_time.lua index 3973d0a..c80cf40 100644 --- a/modules/date_time.lua +++ b/modules/lpeg/date_time.lua @@ -1,3 +1,70 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +# Date Time Module + +## Variables +### LPEG Grammars +* `date_fullyear` - 4 digit year +* `date_month` - 2 digit month +* `date_mabbr` - month abbreviation to a numeric string +* `date_mfull` - month name to a numeric string +* `date_mday` - 2 digit month day (zero padded) +* `date_mday_sp` - 2 digit month day (space padded) +* `time_hour` - 2 digit hour +* `time_minute` - 2 digit minute +* `time_second` - 2 digit second (including leap second) +* `time_secfrac` - fraction seconds +* `timezone` - timezone abbreviations (US and UTC others create an offset of 0) +* `timezone_offset` - numeric time zone offset +* `clf_timestamp` - common log format timestamp +* `mysql_timestamp` - MySQL timestamp +* `pgsql_timestamp` - Postgres timestamp +* `rfc3164_timestamp` +* `rfc3339_time_numoffset` +* `rfc3339_time_offset` +* `rfc3339_partial_time` +* `rfc3339_full_date` +* `rfc3339_full_time` +* `rfc3339` + + +## Functions + +### build_strftime_grammar + +Constructs an LPEG grammar based on the strftime format. + +*Arguments* +- strftime (string) - strftime format specifier + +*Return* +- grammar (LPEG user data object) or an error is thrown + +### time_to_ns + +Converts time table to a time_ns + +*Arguments* +- t (table) - table returned by the various date/time grammars + +*Return* +- time_ns (number) - number of nanoseconds since the Unix epoch + +### seconds_to_ns + +Converts time_t to a time_ns + +*Arguments* +- time_t + +*Return* +- time_ns (number) - number of nanoseconds since the Unix epoch + +--]] + -- Imports local l = require "lpeg" l.locale(l) @@ -6,14 +73,14 @@ local string = require "string" local tonumber = tonumber local ipairs = ipairs local error = error +local type = type local M = {} setfenv(1, M) -- Remove external access to contain everything in the module --[[ Utility Functions --]] --- Converts a time table into the number of nanoseconds since the UNIX epoch function time_to_ns(t) - if not t then return 0 end + if type(t) ~= "table" then return 0 end if t.time_t then return t.time_t * 1e9 end local offset = 0 diff --git a/modules/ip_address.lua b/modules/lpeg/ip_address.lua similarity index 89% rename from modules/ip_address.lua rename to modules/lpeg/ip_address.lua index c4c9503..67b3571 100644 --- a/modules/ip_address.lua +++ b/modules/lpeg/ip_address.lua @@ -2,6 +2,15 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +--[[ +# IP Address Module + +## Variables +### LPEG Grammars +* `v4` - IPv4 address matcher +* `v6` - IPv6 address matcher +--]] + -- Imports local l = require "lpeg" l.locale(l) diff --git a/modules/mysql.lua b/modules/lpeg/mysql.lua similarity index 96% rename from modules/mysql.lua rename to modules/lpeg/mysql.lua index afd932e..3ff2c10 100644 --- a/modules/mysql.lua +++ b/modules/lpeg/mysql.lua @@ -2,11 +2,23 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +--[[ +# MySQL Module + +## Variables +### LPEG Grammars +* `slow_query_grammar` +* `mariadb_slow_query_grammar` +* `percona_slow_query_grammar` +* `short_slow_query_grammar` +* `mariadb_short_slow_query_grammar` +--]] + -- Imports local l = require "lpeg" l.locale(l) local tonumber = tonumber -local ip = require "ip_address" +local ip = require "lpeg.ip_address" local M = {} setfenv(1, M) -- Remove external access to contain everything in the module diff --git a/modules/postfix.lua b/modules/lpeg/postfix.lua similarity index 98% rename from modules/postfix.lua rename to modules/lpeg/postfix.lua index cf8960a..23dce4a 100644 --- a/modules/postfix.lua +++ b/modules/lpeg/postfix.lua @@ -4,8 +4,25 @@ -- Copyright 2015 Mathieu Parent +--[[ +# Postfix Module + +## Function +### function postfix_match + +Parses a variety of postfix log message types. + +*Arguments* +- programname (string) - name of the logging program e.g., 'smtpd' +- message (string) - message string to match +- extract_keyvalue_data (bool) - true if key/value pairs should be split out into table entries + +*Return* +- matched results (table) or nil if no match +--]] + local string = require 'string' -local ip = require 'ip_address' +local ip = require 'lpeg.ip_address' local l = require 'lpeg' l.locale(l) local ipairs = ipairs diff --git a/modules/syslog.lua b/modules/lpeg/syslog.lua similarity index 94% rename from modules/syslog.lua rename to modules/lpeg/syslog.lua index 4f4ced1..df6afd1 100644 --- a/modules/syslog.lua +++ b/modules/lpeg/syslog.lua @@ -2,13 +2,34 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. +--[[ +# Syslog Module + +## Variables + +* `severity` - LPEG grammar to parse a syslog severity string and return the numeric value + +## Functions + +### build_rsyslog_grammar + +Constructs an LPEG grammar based on the rsyslog template configuration string. + +*Arguments* +- template (string) - http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html + +*Return* +- grammar (LPEG user data object) or an error is thrown +--]] + + -- Imports local l = require "lpeg" l.locale(l) local math = require "math" local string = require "string" -local dt = require "date_time" -local ip = require "ip_address" +local dt = require "lpeg.date_time" +local ip = require "lpeg.ip_address" local tonumber = tonumber local error = error local type = type @@ -60,7 +81,7 @@ local function convert_pri(pri) end local pri = digit^-3 / convert_pri --- https://github.com/rsyslog/rsyslog/blob/35ef2408dfec0e8abdebd33c74578f9bb3299f20/runtime/msg.c#L298 +-- https://github.com/rsyslog/rsyslog/blob/35ef2408dfec0e8abdebd33c74578f9bb3299f20/runtime/msg.c#L309 local syslog_severity_text = ( (l.P"debug" + "DEBUG") / "7" + (l.P"info" + "INFO") / "6" @@ -227,8 +248,6 @@ end -- -- Public Interface -- - --- http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html function build_rsyslog_grammar(template) local ws = l.space / space_grammar local options = l.P":" * l.Cg((1 - l.P"%")^0, "options") -- todo support multiple options diff --git a/modules/syslog_message.lua b/modules/lpeg/syslog_message.lua similarity index 98% rename from modules/syslog_message.lua rename to modules/lpeg/syslog_message.lua index 8b30038..6b61559 100644 --- a/modules/syslog_message.lua +++ b/modules/lpeg/syslog_message.lua @@ -4,8 +4,34 @@ -- Copyright 2015 Mathieu Parent +--[[ +# Syslog Message Module + + +## Functions + +### get_prog_grammar + +Retrieves the parser for a particular program. + +*Arguments* +- prog (string) - program name e.g. "CRON", "dhclient", "dhcpd"... + +*Return* +- grammar (LPEG user data object) or nil if the `programname` isn't found + +### get_wildcard_grammar + +*Arguments* +- prog (string) - program name, currently only accepts "PAM" + +*Return* +- grammar (LPEG user data object) or nil if the `programname` isn't found +--]] + + local string = require "string" -local ip = require "ip_address" +local ip = require "lpeg.ip_address" local l = require "lpeg" l.locale(l) local tonumber = tonumber @@ -14,7 +40,6 @@ local type = type local M = {} setfenv(1, M) -- Remove external access to contain everything in the module --- Exported variables: local prog_grammar = {} local wildcard_grammar = {} @@ -602,25 +627,14 @@ wildcard_grammar["PAM"] = l.Ct( * l.P"unknown" )) + function get_prog_grammar(prog) return prog_grammar[prog] end --- If prog is a table, return a table of wildcard grammars --- If prog is a string, return this particular wildcard grammar --- If prog is nil, return all wildcard grammars + function get_wildcard_grammar(prog) - if type(prog) == "table" then - local ret = {} - for i,v in ipairs(prog) do - table.insert(ret, wildcard_grammar[prog]) - end - return ret - elseif type(prog) == "string" then - return wildcard_grammar[prog] - elseif prog == nil then - return wildcard_grammar - end + return wildcard_grammar[prog] end return M diff --git a/modules/lsb/util.lua b/modules/lsb/util.lua new file mode 100644 index 0000000..953018f --- /dev/null +++ b/modules/lsb/util.lua @@ -0,0 +1,77 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +# Lua Sandbox Utility Module + +## Functions + +### behead_array + +Effectively removes all array values up to the provided index from an array +by copying end values to the array head and setting now unused entries at +the end of the array to `nil`. + +*Arguments* +- index (number) - remove the values in the array up to the index value +- array (table) - array to behead + +*Return* +- none - in-place operation + +### pairs_by_key + +Sorts the keys into an array, and then iterates on the array. + +*Arguments* +- hash (table) - hash table to iterate in sorted key order +- sort (function) - function to specify an alternate sort order + +*Return* +- function - iterator that traverses the table keys in sort function order +--]] + +-- Imports +local pairs = pairs +local table = require "table" + +local M = {} +setfenv(1, M) -- Remove external access to contain everything in the module + +function behead_array(idx, array) + if idx <= 1 then return end + local array_len = #array + local start_nil_idx = 1 -- If idx > #array we zero it out completely. + if idx <= array_len then + -- Copy values to lower indexes. + local difference = idx - 1 + for i = idx, array_len do + array[i-difference] = array[i] + end + start_nil_idx = array_len - difference + 1 + end + -- Empty out the end of the array. + for i = start_nil_idx, array_len do + array[i] = nil + end +end + +-- http://www.lua.org/pil/19.3.html +function pairs_by_key(t, f) + local a = {} + for n in pairs(t) do table.insert(a, n) end + table.sort(a, f) + local i = 0 -- iterator variable + local iter = function () -- iterator function + i = i + 1 + if a[i] == nil then + return nil + else + return a[i], t[a[i]] + end + end + return iter +end + +return M diff --git a/modules/sfl4j.lua b/modules/sfl4j.lua deleted file mode 100644 index 94bce7f..0000000 --- a/modules/sfl4j.lua +++ /dev/null @@ -1,58 +0,0 @@ --- imports -local l = require "lpeg" -local dt = require "date_time" -local tonumber = tonumber - -l.locale(l) - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local space = l.space^1 -local sep = l.P"\n" -local class = (l.P(1) - ":")^1 -- com.domain.client.jobs.OutgoingQueue -local msg = (l.P(1) - sep)^0 -local line = (l.P(1) - sep)^0 * sep - --- slf4j uses the following levels/severity --- "FATAL" = 6 --- "ERROR" = 5 --- "WARN" = 4 --- "INFO" = 3 --- "DEBUG" = 2 --- "TRACE" = 1 - --- Map sfl4j log levels to syslog severity -local sfl4j_levels = l.Cg(( - (l.P"TRACE" + "trace") / "7" -+ (l.P"DEBUG" + "debug") / "7" -+ (l.P"INFO" + "info") / "6" -+ (l.P"WARN" + "warn") / "4" -+ (l.P"ERROR" + "error") / "3" -+ (l.P"FATAL" + "fatal") / "0") -/ tonumber, "severity") - -local function convert_to_frac(sec) - return tonumber("0." .. sec) -end - --- Example: 2014-11-21 16:35:59,501 -local time_secfrac = l.Cg(l.digit^3 / convert_to_frac, "sec_frac") -local partial_time = dt.time_hour * ":" * dt.time_minute * ":" * dt.time_second * "," * time_secfrac^-1 -local sfl4j_datetime = l.Ct(dt.rfc3339_full_date * space * partial_time) / dt.time_to_ns - --- Example: ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name -local logline = sfl4j_levels -- ERROR - * space * "[" - * l.Cg(sfl4j_datetime, "timestamp") -- 2014-11-21 16:35:59,501 - * "]" * space - * l.Cg(class, "class") -- com.domain.client.jobs.OutgoingQueue - * ":" * space - * l.Cg(msg, "message") -- Error handling output... - --- A representation of a full log event -local logevent = logline * sep * l.Cg(line^0, "stacktrace") - -logevent_grammar = l.Ct(logevent) - -return M diff --git a/sandboxes/heka/analysis/throughput.lua b/sandboxes/heka/analysis/throughput.lua index 5bbb2c2..6f05478 100644 --- a/sandboxes/heka/analysis/throughput.lua +++ b/sandboxes/heka/analysis/throughput.lua @@ -3,9 +3,12 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Simple message throughput counter/visualization +# Throughput Analysis --- .cfg +Simple message throughput counter/visualization. + +## Sample Configuration +```lua filename = "throughput.lua" message_matcher = "TRUE" ticker_interval = 60 @@ -21,7 +24,7 @@ preservation_version = 0 -- preservation_version (integer) - This must be incremented if the rows or -- sec_per_row value is change when using preservation (the old data is cleared) -- preservation_version = 0 - +``` --]] _PRESERVATION_VERSION = read_config("preservation_version") or 0 diff --git a/sandboxes/heka/input/heka_kafka.lua b/sandboxes/heka/input/heka_kafka.lua index bdca290..305d26c 100644 --- a/sandboxes/heka/input/heka_kafka.lua +++ b/sandboxes/heka/input/heka_kafka.lua @@ -5,31 +5,32 @@ require "heka_kafka_consumer" --[[ +# Heka Kafka Consumer Input + +## Sample Configuration +```lua +filename = "heka_kafka.lua" +output_limit = 8 * 1024 * 1024 +brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 + +-- in balanced consumer group mode a consumer can only subscribe on topics, not topics:partitions. +-- The partition syntax is only used for manual assignments (without balanced consumer groups). +topics = {"test"} +ticker_interval = 60 + +-- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties +consumer_conf = { + ["group.id"] = "test_group", -- must always be provided (a single consumer is considered a group of one + -- in that case make this a unique identifier) + ["message.max.bytes"] = output_limit, +} -*Example Configuration* - - filename = "heka_kafka.lua" - output_limit = 8 * 1024 * 1024 - brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 - - -- in balanced consumer group mode a consumer can only subscribe on topics, not topics:partitions. - -- The partition syntax is only used for manual assignments (without balanced consumer groups). - topics = {"test"} - ticker_interval = 60 - - -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties - consumer_conf = { - ["group.id"] = "test_group", -- must always be provided (a single consumer is considered a group of one - -- in that case make this a unique identifier) - ["message.max.bytes"] = output_limit, - } - - -- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties - topic_conf = { - -- ["auto.commit.enable"] = true, -- cannot be overridden - -- ["offset.store.method"] = "broker, -- cannot be overridden - } - +-- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties +topic_conf = { + -- ["auto.commit.enable"] = true, -- cannot be overridden + -- ["offset.store.method"] = "broker, -- cannot be overridden +} +``` --]] local brokerlist = read_config("brokerlist") or error("brokerlist must be set") local topics = read_config("topics") or error("topics must be set") diff --git a/sandboxes/heka/input/heka_stdin.lua b/sandboxes/heka/input/heka_stdin.lua index 2231636..d788092 100644 --- a/sandboxes/heka/input/heka_stdin.lua +++ b/sandboxes/heka/input/heka_stdin.lua @@ -3,11 +3,12 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Reads a Heka protobuf stream from the stdin file handle +# Heka Protobuf Stream Input (stdin) --- .cfg +## Sample Configuration +```lua filename = "heka_stdin.lua" - +``` --]] local stdin = require "io".stdin diff --git a/sandboxes/heka/input/heka_tcp.lua b/sandboxes/heka/input/heka_tcp.lua index 9675c4a..bf326ef 100644 --- a/sandboxes/heka/input/heka_tcp.lua +++ b/sandboxes/heka/input/heka_tcp.lua @@ -3,9 +3,10 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Heka compatible TCP input +# Heka Compatible TCP Input --- .cfg +## Sample Configuration +```lua filename = "heka_tcp.lua" instruction_limit = 0 @@ -21,7 +22,7 @@ ssl_params = { verify = {"peer", "fail_if_no_peer_cert"}, options = {"all", "no_sslv3"} } - +``` --]] require "coroutine" diff --git a/sandboxes/heka/input/syslog_udp.lua b/sandboxes/heka/input/syslog_udp.lua index 1e5b17e..1a35399 100644 --- a/sandboxes/heka/input/syslog_udp.lua +++ b/sandboxes/heka/input/syslog_udp.lua @@ -3,9 +3,10 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Syslog Collector +# Syslog UDP Input --- .cfg +## Sample Configuration +```lua filename = "syslog_udp.lua" instruction_limit = 0 @@ -18,7 +19,7 @@ instruction_limit = 0 -- template (string) - The 'template' configuration string from rsyslog.conf -- see http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html -- template = "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" -- RSYSLOG_TraditionalForwardFormat - +``` --]] local syslog = require "syslog" diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua index 946f41a..37652c4 100644 --- a/sandboxes/heka/output/debug.lua +++ b/sandboxes/heka/output/debug.lua @@ -3,12 +3,15 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Outputs a more user friendly version (RST format) of the full Heka message to stdout. +# RST Heka Message Output --- .cfg +Writes a user friendly version (RST format) of the full Heka message to stdout + +## Sample Configuration +```lua filename = "debug.lua" message_matcher = "TRUE" - +``` --]] local write = require "io".write diff --git a/sandboxes/heka/output/elasticsearch_bulk_api.lua b/sandboxes/heka/output/elasticsearch_bulk_api.lua index bfd4279..8b21cb3 100644 --- a/sandboxes/heka/output/elasticsearch_bulk_api.lua +++ b/sandboxes/heka/output/elasticsearch_bulk_api.lua @@ -3,9 +3,9 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -## Elasticsearch Bulk API Output +# Elasticsearch Bulk API Output -### Sample Configuration +## Sample Configuration ```lua filename = "elasticsearch_bulk_api.lua" message_matcher = "Type == 'nginx'" diff --git a/sandboxes/heka/output/heka_kafka.lua b/sandboxes/heka/output/heka_kafka.lua index 6cb697d..661dbd9 100644 --- a/sandboxes/heka/output/heka_kafka.lua +++ b/sandboxes/heka/output/heka_kafka.lua @@ -5,25 +5,26 @@ require "heka_kafka_producer" --[[ - -*Example Configuration* - - filename = "heka_kafka.lua" - message_matcher = "TRUE" - output_limit = 8 * 1024 * 1024 - brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 - ticker_interval = 60 - async_buffer_size = 20000 - - topic_constant = "test" - producer_conf = { - ["queue.buffering.max.messages"] = async_buffer_size, - ["batch.num.messages"] = 200, - ["message.max.bytes"] = output_limit, - ["queue.buffering.max.ms"] = 10, - ["topic.metadata.refresh.interval.ms"] = -1, - } - +# Heka Kafka Producer Output + +## Sample Configuration +```lua +filename = "heka_kafka.lua" +message_matcher = "TRUE" +output_limit = 8 * 1024 * 1024 +brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 +ticker_interval = 60 +async_buffer_size = 20000 + +topic_constant = "test" +producer_conf = { + ["queue.buffering.max.messages"] = async_buffer_size, + ["batch.num.messages"] = 200, + ["message.max.bytes"] = output_limit, + ["queue.buffering.max.ms"] = 10, + ["topic.metadata.refresh.interval.ms"] = -1, +} +``` --]] local brokerlist = read_config("brokerlist") or error("brokerlist must be set") local topic_constant = read_config("topic_constant") diff --git a/sandboxes/heka/output/heka_log_rolling.lua b/sandboxes/heka/output/heka_log_rolling.lua index f46cf2c..943e8e3 100644 --- a/sandboxes/heka/output/heka_log_rolling.lua +++ b/sandboxes/heka/output/heka_log_rolling.lua @@ -6,11 +6,12 @@ require "io" require "string" --[[ -Outputs a Heka protobuf stream rolling the log file every time it reaches the -roll_size. +# Heka Protobuf Stream Output (rolled by size) --- .cfg +Outputs a Heka protobuf stream rolling the log file every time it reaches the `roll_size`. +## Sample Configuration +```lua filename = "heka_log_rolling.lua" message_matcher = "TRUE" ticker_interval = 0 @@ -19,7 +20,7 @@ preserve_data = true --location where the payload is written output_dir = "/tmp" roll_size = 1024 * 1024 * 1024 - +``` --]] file_num = 0 diff --git a/sandboxes/heka/output/heka_s3_partition.lua b/sandboxes/heka/output/heka_s3_partition.lua index d4b15cc..3236b36 100644 --- a/sandboxes/heka/output/heka_s3_partition.lua +++ b/sandboxes/heka/output/heka_s3_partition.lua @@ -22,15 +22,15 @@ local max_file_age = read_config("max_file_age") or 60 * 60 local flush_on_shutdown = read_config("flush_on_shutdown") --[[ -## Heka Protobuf Message S3 Output Partitioner +# Heka Protobuf Stream S3 Output Partitioner Batches message data into Heka protobuf stream files based on the specified path dimensions and copies them to S3 when they reach the maximum size or maximum age. -### Configuration +## Configuration -#### Dimension Specification File +### Dimension Specification File This file contains a JSON specification for how the data should be partitioned into files and the location each file will reside at in S3. The JSON is shared with other tools so it is not directly included in the configuration. @@ -40,7 +40,7 @@ TODO: This specifification should be expanded (when necessary) to include - field/array index support - patterns instead of exact matches -##### Sample Specification File +#### Sample Specification File ```json { "version": 1, @@ -63,7 +63,7 @@ TODO: This specifification should be expanded (when necessary) to include } ``` -#### Sample Configuration +### Sample Configuration ```lua filename = "heka_s3_partition.lua" diff --git a/sandboxes/heka/output/heka_tcp.lua b/sandboxes/heka/output/heka_tcp.lua index 698451e..342eca6 100644 --- a/sandboxes/heka/output/heka_tcp.lua +++ b/sandboxes/heka/output/heka_tcp.lua @@ -3,9 +3,10 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Heka compatible TCP output +# Heka Compatible TCP Output --- .cfg +## Sample Configuration +```lua filename = "heka_tcp.lua" message_matcher = "TRUE" @@ -22,7 +23,7 @@ ssl_params = { verify = "peer", options = {"all", "no_sslv3"} } - +``` --]] local socket = require "socket" diff --git a/sandboxes/heka/output/heka_tcp_matcher.lua b/sandboxes/heka/output/heka_tcp_matcher.lua index 5023b47..58a4791 100644 --- a/sandboxes/heka/output/heka_tcp_matcher.lua +++ b/sandboxes/heka/output/heka_tcp_matcher.lua @@ -3,10 +3,13 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. --[[ -Heka compatible TCP output with dynamic matching. Used for generating a debug -stream of messages for as long as the connection is maintained. +# Heka Compatible TCP Output with Dynamic Matching --- .cfg +Used for generating a debug stream of messages for as long as the connection is +maintained. + +## Sample Configuration +```lua filename = "heka_tcp_matcher.lua" instruction_limit = 0 messsage_matcher = "TRUE" @@ -24,7 +27,7 @@ ssl_params = { verify = {"peer", "fail_if_no_peer_cert"}, options = {"all", "no_sslv3"} } - +``` --]] require "string" diff --git a/sandboxes/heka/output/inject_payload.lua b/sandboxes/heka/output/inject_payload.lua index 796b1e3..9514950 100644 --- a/sandboxes/heka/output/inject_payload.lua +++ b/sandboxes/heka/output/inject_payload.lua @@ -6,11 +6,14 @@ require "io" require "string" --[[ -Outputs inject_payload messages to the configured directory. -Filename: output_dir/logger.payload_name.payload_type -Contents: message.Payload +# Message Payload Output --- .cfg +Outputs the message payload to the configured directory. The output filename is +`output_dir`/`Logger.Fields[payload_name]`.`Fields[payload_type]` +and the contents are the message `Payload`. + +## Sample Configuration +```lua filename = "inject_payload.lua" message_matcher = "Type == 'inject_payload'" ticker_interval = 0 @@ -18,7 +21,7 @@ ticker_interval = 0 -- location where the payload is written (e.g. make them accessible from a web -- server for external consumption) output_dir = "/var/www/hindsight/payload" - +``` --]] local output_dir = read_config("output_dir") or "/tmp" diff --git a/src/heka/message.c b/src/heka/message.c index 80671fa..cb0e3e0 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -8,6 +8,7 @@ #include "message_impl.h" +#include #include #include #include @@ -967,7 +968,11 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) lua_pushnil(lua); } } else if (strcmp(field, LSB_PID) == 0) { - lua_pushinteger(lua, m->pid); + if (m->pid == INT_MIN) { + lua_pushnil(lua); + } else { + lua_pushinteger(lua, m->pid); + } } else if (strcmp(field, LSB_HOSTNAME) == 0) { if (m->hostname.s) { lua_pushlstring(lua, m->hostname.s, m->hostname.len); diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index c8b9a17..baa2561 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -318,7 +318,7 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) { static const char *io[] = { NULL, "", "dofile", "load", "loadfile", "loadstring", "newproxy", - NULL, "os", "getenv", "exit", "setlocale", + NULL, "os", "exit", "setlocale", NULL, "string", "dump" }; diff --git a/src/heka/test/lua/heka_util.lua b/src/heka/test/lua/heka_util.lua new file mode 100644 index 0000000..6eda8a1 --- /dev/null +++ b/src/heka/test/lua/heka_util.lua @@ -0,0 +1,28 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local string = require "string" +local util = require "heka.util" + +local t = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} +local fa = {} +local fb = {} + +local function table_to_fields() + util.table_to_fields(t, fa, nil) + assert(fa.toplevel == 0, fa.toplevel) + assert(fa["struct.item0"] == 0, fa["struct.item0"]) + assert(fa["struct.item1"] == 1, fa["struct.item1"]) + assert(fa["struct.item2.nested"] == "n1", fa["struct.item2.nested"]) + util.table_to_fields(t, fb, nil, "_", 2) + assert(fb.toplevel == 0, fb.toplevel) + assert(fb["struct_item0"] == 0, fb["struct_item0"]) + assert(fb["struct_item1"] == 1, fb["struct_item1"]) + assert(fb["struct_item2"] == '{"nested":"n1"}', fb["struct_item2"]) +end + +function process_message () + table_to_fields() + return 0 +end diff --git a/src/heka/test/lua/read_message.lua b/src/heka/test/lua/read_message.lua index 4831ae8..9a6f501 100644 --- a/src/heka/test/lua/read_message.lua +++ b/src/heka/test/lua/read_message.lua @@ -13,7 +13,7 @@ local tests = { {"EnvVersion", "env_version"}, {"Hostname", "hostname"}, {"Severity", 9}, - {"Pid", 0}, + {"Pid", nil}, {"raw", 208}, {"size", 208}, {"framed", 214}, diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 2dfc4b0..4044b3f 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -614,6 +614,28 @@ static char* test_heka_json() } +static char* test_heka_util() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/heka_util.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n", + &logger, aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + int rv = lsb_heka_pm_analysis(hsb, &m, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + + static char* benchmark_decode_message() { int iter = 100000; @@ -658,6 +680,7 @@ static char* all_tests() mu_run_test(test_decode_message); mu_run_test(test_read_message); mu_run_test(test_heka_json); + mu_run_test(test_heka_util); mu_run_test(benchmark_decode_message); return NULL; diff --git a/src/test/lua/lpeg_cbufd.lua b/src/test/lua/lpeg_cbufd.lua index ea9972c..7b0670e 100644 --- a/src/test/lua/lpeg_cbufd.lua +++ b/src/test/lua/lpeg_cbufd.lua @@ -4,7 +4,7 @@ require "lpeg" -local cbufd = require("cbufd") +local cbufd = require "lpeg.cbufd" function process(tc) if tc == 0 then diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index 49c7be4..440b4d8 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -2,7 +2,8 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -local clf = require "common_log_format" +local ok, clf = pcall(require,"lpeg.common_log_format") +if not ok then print(clf) end require "string" local combined_log = '127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"' diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua index ee26f07..57174cf 100644 --- a/src/test/lua/lpeg_date_time.lua +++ b/src/test/lua/lpeg_date_time.lua @@ -4,7 +4,7 @@ require "string" -local dt = require("date_time") +local dt = require "lpeg.date_time" local function test_valid(grammar, tests, results) for i,v in ipairs(tests) do diff --git a/src/test/lua/lpeg_ip_address.lua b/src/test/lua/lpeg_ip_address.lua index 020b435..3445bd9 100644 --- a/src/test/lua/lpeg_ip_address.lua +++ b/src/test/lua/lpeg_ip_address.lua @@ -5,7 +5,7 @@ require "lpeg" -local ip = require("ip_address") +local ip = require "lpeg.ip_address" local function ipv6() local tests = {"2001:0db8:85a3:0000:0000:8a2e:0370:7334", diff --git a/src/test/lua/lpeg_mysql.lua b/src/test/lua/lpeg_mysql.lua index 3ad12ef..0b41ebb 100644 --- a/src/test/lua/lpeg_mysql.lua +++ b/src/test/lua/lpeg_mysql.lua @@ -3,7 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "string" -local mysql = require("mysql") +local mysql = require "lpeg.mysql" local slow_query_log = { [[ diff --git a/src/test/lua/lpeg_postfix.lua b/src/test/lua/lpeg_postfix.lua index 64b526b..0d3a561 100644 --- a/src/test/lua/lpeg_postfix.lua +++ b/src/test/lua/lpeg_postfix.lua @@ -4,7 +4,7 @@ require 'string' require 'table' -local pf = require 'postfix' +local pf = require 'lpeg.postfix' -- From https://github.com/whyscream/postfix-grok-patterns -- 2015-05-26 8fdd562d159845b97bf8d9c273e5357f3a143a94 diff --git a/src/test/lua/lpeg_sfl4j.lua b/src/test/lua/lpeg_sfl4j.lua deleted file mode 100644 index 2fb720d..0000000 --- a/src/test/lua/lpeg_sfl4j.lua +++ /dev/null @@ -1,118 +0,0 @@ -local sfl4j = require("sfl4j") -local dt = require("date_time") - --- Parse a basic message, these are typically not errors, but whatever. -local function single_line_logevent() - local single_line_log = 'ERROR [2014-11-21 16:35:59,501] com.domain.client.jobs.OutgoingQueue: Error handling output file with job job-name\n' - local single_line_test_fields = { - severity = 3, - timestamp = 1.416587759501e+18, - class = "com.domain.client.jobs.OutgoingQueue", - message = "Error handling output file with job job-name" - } - - local fields = sfl4j.logevent_grammar:match(single_line_log) - if not fields then error("match didn't work " .. single_line_log) end - - assert(fields.severity == single_line_test_fields.severity, fields.severity) - assert(fields.timestamp == single_line_test_fields.timestamp, fields.timestamp) - assert(fields.class == single_line_test_fields.class, fields.class) - assert(fields.message == single_line_test_fields.message, fields.message) -end - --- Use this to easily debug multi-line messages -local function two_line_logevent() - local two_line_log = [[ -ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in user loop -! java.net.SocketTimeoutException: Read timed out - ]] - local two_line_test_fields = { - severity = 3, - timestamp = 1.416501651577e+18, - class = "com.domain.substitute.UsersLoop", - message = "Error caught in user loop", - stacktrace = "! java.net.SocketTimeoutException: Read timed out\n" - } - - local fields = sfl4j.logevent_grammar:match(two_line_log) - if not fields then error("match didn't work " .. two_line_log) end - - assert(fields.severity == two_line_test_fields.severity, fields.severity) - assert(fields.timestamp == two_line_test_fields.timestamp, fields.timestamp) - assert(fields.class == two_line_test_fields.class, fields.class) - assert(fields.message == two_line_test_fields.message, fields.message) - assert(fields.Stacktrace == two_line_test_fields.Stacktrace, fields.Stacktrace) -end - - -local multi_line_log = [[ -ERROR [2014-11-20 16:40:51,577] com.domain.substitute.UsersLoop: Error caught in user loop -! java.net.SocketTimeoutException: Read timed out -! at java.net.SocketInputStream.socketRead0(Native Method) -! at java.net.SocketInputStream.read(SocketInputStream.java:152) -! at java.net.SocketInputStream.read(SocketInputStream.java:122) -! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442) -! at sun.security.ssl.InputRecord.read(InputRecord.java:480) -! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927) -! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884) -! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102) -! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136) -! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152) -! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270) -! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140) -! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57) -! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260) -! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161) -! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) -! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) -! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) -! at java.lang.reflect.Method.invoke(Method.java:606) -! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138) -! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source) -! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271) -! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123) -! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253) -! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194) -! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85) -! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108) -! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186) -! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) -! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) -! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) -! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) -! at com.yammer.metrics.scala.Timer.time(Timer.scala:17) -! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471) -! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304) -! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178) -! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) -! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) -! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) -! at java.lang.Thread.run(Thread.java:744) -]] - -local multi_line_test_fields = { - severity = 3, - timestamp = 1.416501651577e+18, - class = "com.domain.substitute.UsersLoop", - message = "Error caught in user loop", - stacktrace = "! java.net.SocketTimeoutException: Read timed out\n! at java.net.SocketInputStream.socketRead0(Native Method)\n! at java.net.SocketInputStream.read(SocketInputStream.java:152)\n! at java.net.SocketInputStream.read(SocketInputStream.java:122)\n! at sun.security.ssl.InputRecord.readFully(InputRecord.java:442)\n! at sun.security.ssl.InputRecord.read(InputRecord.java:480)\n! at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:927)\n! at sun.security.ssl.SSLSocketImpl.readDataRecord(SSLSocketImpl.java:884)\n! at sun.security.ssl.AppInputStream.read(AppInputStream.java:102)\n! at org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:136)\n! at org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:152)\n! at org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:270)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)\n! at org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)\n! at org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:260)\n! at org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:161)\n! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n! at java.lang.reflect.Method.invoke(Method.java:606)\n! at org.apache.http.impl.conn.CPoolProxy.invoke(CPoolProxy.java:138)\n! at com.sun.proxy.$Proxy25.receiveResponseHeader(Unknown Source)\n! at org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:271)\n! at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:123)\n! at org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:253)\n! at org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:194)\n! at org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:85)\n! at org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:108)\n! at org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:186)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)\n! at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at com.yammer.metrics.scala.Timer.time(Timer.scala:17)\n! at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)\n! at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:304)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:178)\n! at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)\n! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)\n! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n! at java.lang.Thread.run(Thread.java:744)\n" -} - -local function multi_line_logevent() - local fields = sfl4j.logevent_grammar:match(multi_line_log) - if not fields then error("match didn't work " .. multi_line_log) end - - assert(fields.severity == multi_line_test_fields.severity, fields.severity) - assert(fields.timestamp == multi_line_test_fields.timestamp, fields.timestamp) - assert(fields.class == multi_line_test_fields.class, fields.class) - assert(fields.message == multi_line_test_fields.message, fields.message) - assert(fields.Stacktrace == multi_line_test_fields.Stacktrace, fields.Stacktrace) -end - -function process() - single_line_logevent() - two_line_logevent() - multi_line_logevent() - - return 0 -end diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua index 93d34d3..0fcc219 100644 --- a/src/test/lua/lpeg_syslog.lua +++ b/src/test/lua/lpeg_syslog.lua @@ -2,7 +2,7 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -local syslog = require("syslog") +local syslog = require "lpeg.syslog" local function traditional_file_format() local grammar = syslog.build_rsyslog_grammar('%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n') diff --git a/src/test/lua/lpeg_syslog_message.lua b/src/test/lua/lpeg_syslog_message.lua index 2e57326..75c6bde 100644 --- a/src/test/lua/lpeg_syslog_message.lua +++ b/src/test/lua/lpeg_syslog_message.lua @@ -2,7 +2,7 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -local syslog_message = require("syslog_message") +local syslog_message = require "lpeg.syslog_message" local function CRON() local grammar = syslog_message.get_prog_grammar('CRON') @@ -42,7 +42,7 @@ local function dhclient() assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) assert(fields.dhcp_server_port == 67, fields.dhcp_server_port) assert(fields.dhcp_client_interval_seconds == 18000, fields.dhcp_client_interval_seconds) - + log = 'DHCPREQUEST on eth0 to 10.20.30.42 port 67' fields = grammar:match(log) diff --git a/src/test/lua/serialize.lua b/src/test/lua/serialize.lua index a2b5e14..92921c0 100644 --- a/src/test/lua/serialize.lua +++ b/src/test/lua/serialize.lua @@ -3,7 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "circular_buffer" -util = require "util" +util = require "lsb.util" count = 0 rate = 0.12345678 diff --git a/src/test/lua/util_test.lua b/src/test/lua/util_test.lua index be0757a..aa6ba25 100644 --- a/src/test/lua/util_test.lua +++ b/src/test/lua/util_test.lua @@ -2,25 +2,8 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -local string = require("string") -local util = require("util") - -local t = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} -local fa = {} -local fb = {} - -local function table_to_fields() - util.table_to_fields(t, fa, nil) - assert(fa.toplevel == 0, fa.toplevel) - assert(fa["struct.item0"] == 0, fa["struct.item0"]) - assert(fa["struct.item1"] == 1, fa["struct.item1"]) - assert(fa["struct.item2.nested"] == "n1", fa["struct.item2.nested"]) - util.table_to_fields(t, fb, nil, "_", 2) - assert(fb.toplevel == 0, fb.toplevel) - assert(fb["struct_item0"] == 0, fb["struct_item0"]) - assert(fb["struct_item1"] == 1, fb["struct_item1"]) - assert(fb["struct_item2"] == '{"nested":"n1"}', fb["struct_item2"]) -end +local string = require "string" +local util = require "lsb.util" local function alpha() return {"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", @@ -50,8 +33,27 @@ local function behead_array() assert(#a == 0, string.format("#a should be 0, is %d", #a)) end + +local function pairs_by_key() + local hash = { a = true , b = true, c = true } + local abc = {"a", "b", "c"} + + local cnt = 1 + for k, v in util.pairs_by_key(hash) do + assert(abc[cnt] == k) + cnt = cnt + 1 + end + + cnt = 3 + for k, v in util.pairs_by_key(hash, function(a, b) return a > b end) do + assert(abc[cnt] == k) + cnt = cnt - 1 + end +end + + function process () - table_to_fields() behead_array() + pairs_by_key() return 0 end diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 1e0c7aa..c0018e1 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -314,7 +314,7 @@ static char* test_init_error() // disabled external modules ret = lsb_init(sb, NULL); mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); - const char *expected = "lua/lpeg_date_time.lua:7: module 'date_time' not " + const char *expected = "lua/lpeg_date_time.lua:7: module 'lpeg.date_time' not " "found:"; mu_assert(strcmp(lsb_get_error(sb), expected) == 0, "lsb_get_error() received: %s", lsb_get_error(sb)); @@ -836,7 +836,6 @@ static char* test_lpeg() , "lua/lpeg_postfix.lua" , "lua/lpeg_syslog.lua" , "lua/lpeg_syslog_message.lua" - , "lua/lpeg_sfl4j.lua" , NULL }; diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 72b6333..56d4890 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -8,6 +8,7 @@ #include "luasandbox/util/heka_message.h" +#include #include #include #include @@ -454,7 +455,7 @@ void lsb_clear_heka_message(lsb_heka_message *m) if (m->fields) memset(m->fields, 0, m->fields_size * sizeof(lsb_heka_field)); m->timestamp = 0; m->severity = 7; - m->pid = 0; + m->pid = INT_MIN; m->fields_len = 0; } From 8495da915e4604208a150a979e14c0339d3f47ac Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 6 Jun 2016 09:00:14 -0700 Subject: [PATCH 154/235] Misc small fixes from the doc updates (typos, formatting, naming) --- CMakeLists.txt | 2 +- docs/heka/analysis.md | 2 +- docs/heka/input.md | 4 ++-- docs/heka/output.md | 2 +- sandboxes/heka/input/syslog_udp.lua | 2 +- src/cli/lsb_heka_cat.c | 7 ++++++- src/heka/message.c | 7 +------ 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b4d00c3..5739ef4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 20) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") set(CPACK_DEB_COMPONENT_INSTALL ON) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index 76e58c9..fd54ac5 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -3,7 +3,7 @@ [Available Analysis Sandboxes](/lua_sandbox/sandboxes/heka/analysis/index.html) ## Recommendations -Since he sandbox does not run in isolation there are some expectations of how +Since the sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the Hindsight reference implementation. diff --git a/docs/heka/input.md b/docs/heka/input.md index 6752444..ea05051 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -3,7 +3,7 @@ [Available Input Sandboxes](/lua_sandbox/sandboxes/heka/input/index.html) ## Recommendations -Since he sandbox does not run in isolation there are some expectations of how +Since the sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the Hindsight reference implementation. @@ -162,4 +162,4 @@ function process_message() until read == 0 return 0 end -``' +``` diff --git a/docs/heka/output.md b/docs/heka/output.md index 8dedc88..6702eab 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -3,7 +3,7 @@ [Available Output Sandboxes](/lua_sandbox/sandboxes/heka/output/index.html) ## Recommendations -Since he sandbox does not run in isolation there are some expectations of how +Since the sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the Hindsight reference implementation. diff --git a/sandboxes/heka/input/syslog_udp.lua b/sandboxes/heka/input/syslog_udp.lua index 1a35399..bd4f139 100644 --- a/sandboxes/heka/input/syslog_udp.lua +++ b/sandboxes/heka/input/syslog_udp.lua @@ -22,7 +22,7 @@ instruction_limit = 0 ``` --]] -local syslog = require "syslog" +local syslog = require "lpeg.syslog" local socket = require "socket" local address = read_config("address") or "127.0.0.1" diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 4630f18..0f7d37d 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -7,6 +7,7 @@ /** @brief lua_sandbox Heka file stream cat @file */ #include +#include #include #include #include @@ -95,7 +96,11 @@ static void output_text(lsb_heka_message *msg) fprintf(stdout, ":Severity: %d\n", msg->severity); output_cs(":Payload", &msg->payload, true); output_cs(":EnvVersion", &msg->env_version, true); - fprintf(stdout, ":Pid: %d\n", msg->pid); + if (msg->pid == INT_MIN) { + fprintf(stdout, ":Pid: \n"); + } else { + fprintf(stdout, ":Pid: %d\n", msg->pid); + } output_cs(":Hostname", &msg->hostname, true); fprintf(stdout, ":Fields:\n"); for (int i = 0; i < msg->fields_len; ++i) { diff --git a/src/heka/message.c b/src/heka/message.c index cb0e3e0..74bcb1b 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -24,12 +24,7 @@ #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" -/** - * Adds missing headers specified in the configuration to the message output. - * - * @param lua Pointer the Lua state. - * @param idx Lua stack index of the message table. - */ + static void set_missing_headers(lua_State *lua, int idx, lsb_heka_sandbox *hsb) { lua_getfield(lua, idx, LSB_LOGGER); From f1ee9eb19f4d237b78b585a8b1c6d056e3b3c9fb Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 7 Jun 2016 13:21:21 -0700 Subject: [PATCH 155/235] Fix the RPM packaging Misc. cleanup - remove the patch requirement - remove the mingw support - remove the old LuaJIT support --- CMakeLists.txt | 34 ++++---- cmake/externals.cmake | 45 +--------- cmake/luajit-2_0_2.patch | 140 -------------------------------- cmake/luasandboxConfig.cmake.in | 16 ++-- cmake/mozsvc.cmake | 2 - src/CMakeLists.txt | 42 +++------- src/cli/CMakeLists.txt | 2 +- src/heka/CMakeLists.txt | 2 +- src/heka/test/CMakeLists.txt | 2 +- src/test/CMakeLists.txt | 2 +- src/util/CMakeLists.txt | 3 +- 11 files changed, 49 insertions(+), 241 deletions(-) delete mode 100644 cmake/luajit-2_0_2.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index 5739ef4..c68c3b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,19 +7,32 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 20) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") +include(GNUInstallDirs) +if(WIN32) + set(INSTALL_CMAKE_DIR cmake) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +else() + set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/${CMAKE_PROJECT_NAME}/cmake") +endif() + set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_COMPONENT_INSTALL ON) +if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") +else() + set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0") +endif() set(CPACK_COMPONENTS_ALL core) option(LUA_JIT "Enable LuaJIT" off) if(LUA_JIT) - add_definitions(-DLUA_JIT) + add_definitions(-DLUA_JIT) endif() find_library(LIBM_LIBRARY m) @@ -56,24 +69,15 @@ if(HAVE_CLOCK_GETTIME) add_definitions(-DHAVE_CLOCK_GETTIME) endif() -set(LIB_INSTALL_DIR lib/) -set(INCLUDE_INSTALL_DIR include/) - include_directories("${EP_BASE}/include") -install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${LIB_INSTALL_DIR}${PROJECT_NAME}/modules COMPONENT core) -install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION share/doc/${PROJECT_NAME} COMPONENT core) -install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandboxes/" DESTINATION share/${PROJECT_NAME}/sandboxes COMPONENT core) - -if(WIN32 AND NOT CYGWIN) - set(INSTALL_CMAKE_DIR cmake) -else() - set(INSTALL_CMAKE_DIR ${LIB_INSTALL_DIR}${CMAKE_PROJECT_NAME}/cmake) -endif() +install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/modules COMPONENT core) +install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT core) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandboxes/" DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/sandboxes COMPONENT core) include(CMakePackageConfigHelpers) configure_package_config_file(cmake/luasandboxConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake INSTALL_DESTINATION ${INSTALL_CMAKE_DIR} - PATH_VARS INCLUDE_INSTALL_DIR LIB_INSTALL_DIR) + PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR) write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfigVersion.cmake VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} COMPATIBILITY AnyNewerVersion ) # todo change to SameMajorVersion after 1.0 diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 026c134..2b9c9be 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -6,10 +6,6 @@ include(ExternalProject) string(REPLACE ")$" "|INSTALL_ARGS)$" _ep_keywords_ExternalProject_Add ${_ep_keywords_ExternalProject_Add}) get_filename_component(GIT_PATH ${GIT_EXECUTABLE} PATH) -find_program(PATCH_EXECUTABLE patch HINTS "${GIT_PATH}" "${GIT_PATH}/../bin") -if (NOT PATCH_EXECUTABLE) - message(FATAL_ERROR "patch not found") -endif() set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) @@ -23,43 +19,6 @@ endif() if (LUA_JIT) message(FATAL_ERROR, "LuaJIT support has not been added back in yet, issue #66") - set(LUA_PROJECT "luajit-2_0_2") - if(MSVC) - externalproject_add( - ${LUA_PROJECT} - BUILD_IN_SOURCE 1 - URL http://luajit.org/download/LuaJIT-2.0.2.tar.gz - URL_MD5 112dfb82548b03377fbefbba2e0e3a5b - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/luajit-2_0_2.patch - CONFIGURE_COMMAND "" - BUILD_COMMAND ${CMAKE_COMMAND} -E chdir src msvcbuild.bat - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy src/lua.dll ${EP_BASE}/lib/lua.dll - COMMAND ${CMAKE_COMMAND} -E copy src/lua.lib ${EP_BASE}/lib/lua.lib - COMMAND ${CMAKE_COMMAND} -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" - COMMAND ${CMAKE_COMMAND} -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" - COMMAND ${CMAKE_COMMAND} -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" - COMMAND ${CMAKE_COMMAND} -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" - COMMAND ${CMAKE_COMMAND} -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" - ) - elseif(UNIX) - externalproject_add( - ${LUA_PROJECT} - BUILD_IN_SOURCE 1 - URL http://luajit.org/download/LuaJIT-2.0.2.tar.gz - URL_MD5 112dfb82548b03377fbefbba2e0e3a5b - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/luajit-2_0_2.patch - CONFIGURE_COMMAND "" - BUILD_COMMAND make - INSTALL_COMMAND ${CMAKE_COMMAND} -E copy src/libluajit.a ${EP_BASE}/lib/liblua.a - COMMAND ${CMAKE_COMMAND} -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" - COMMAND ${CMAKE_COMMAND} -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" - COMMAND ${CMAKE_COMMAND} -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" - COMMAND ${CMAKE_COMMAND} -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" - COMMAND ${CMAKE_COMMAND} -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" - ) - else() - message(FATAL_ERROR "Cannot use LuaJIT with ${CMAKE_GENERATOR}") - endif() else() set(LUA_PROJECT "lua-5_1_5") externalproject_add( @@ -69,6 +28,10 @@ else() CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) + add_library(luasb SHARED IMPORTED) + set_target_properties(luasb PROPERTIES IMPORTED_LOCATION "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}") + #install(TARGETS luasb DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core OPTIONAL) + #http://public.kitware.com/Bug/view.php?id=14311&nbn=6 endif() externalproject_add( diff --git a/cmake/luajit-2_0_2.patch b/cmake/luajit-2_0_2.patch deleted file mode 100644 index 2c3ad0d..0000000 --- a/cmake/luajit-2_0_2.patch +++ /dev/null @@ -1,140 +0,0 @@ -diff -Naur LuaJIT-2.0.2/src/lj_api.c luajit-2_0_2/src/lj_api.c ---- LuaJIT-2.0.2/src/lj_api.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_api.c 2013-12-03 21:17:05.042524855 +0000 -@@ -1178,6 +1178,18 @@ - res = (int)(g->gc.stepmul); - g->gc.stepmul = (MSize)data; - break; -+ case LUA_GCSETMEMLIMIT: -+ res = data; -+ lj_gc_fullgc(L); -+ g->gc.maxmem = g->gc.total; -+ g->gc.memlimit = (MSize)data; -+ break; -+ case LUA_GCMAXCOUNT: -+ res = (int)(g->gc.maxmem >> 10); -+ break; -+ case LUA_GCMAXCOUNTB: -+ res = (int)(g->gc.maxmem & 0x3ff); -+ break; - default: - res = -1; /* Invalid option. */ - } -diff -Naur LuaJIT-2.0.2/src/lj_dispatch.c luajit-2_0_2/src/lj_dispatch.c ---- LuaJIT-2.0.2/src/lj_dispatch.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_dispatch.c 2013-12-03 21:17:05.042524855 +0000 -@@ -339,6 +339,11 @@ - return (int)G(L)->hookcstart; - } - -+LUA_API int lua_gethookcountremaining(lua_State *L) -+{ -+ return (int)G(L)->hookcount; -+} -+ - /* Call a hook. */ - static void callhook(lua_State *L, int event, BCLine line) - { -diff -Naur LuaJIT-2.0.2/src/lj_gc.c luajit-2_0_2/src/lj_gc.c ---- LuaJIT-2.0.2/src/lj_gc.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_gc.c 2013-12-03 21:17:05.042524855 +0000 -@@ -800,24 +800,37 @@ - { - global_State *g = G(L); - lua_assert((osz == 0) == (p == NULL)); -- p = g->allocf(g->allocd, p, osz, nsz); -+ if (g->gc.memlimit == 0 || (g->gc.total - osz) + nsz < g->gc.memlimit) { -+ p = g->allocf(g->allocd, p, osz, nsz); -+ } else { -+ p = NULL; -+ } - if (p == NULL && nsz > 0) - lj_err_mem(L); - lua_assert((nsz == 0) == (p == NULL)); - lua_assert(checkptr32(p)); - g->gc.total = (g->gc.total - osz) + nsz; -- return p; -+ if (g->gc.total > g->gc.maxmem) { -+ g->gc.maxmem = g->gc.total; -+ } -+ return p; - } - - /* Allocate new GC object and link it to the root set. */ - void * LJ_FASTCALL lj_mem_newgco(lua_State *L, MSize size) - { - global_State *g = G(L); -- GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size); -+ GCobj *o = NULL; -+ if (g->gc.memlimit == 0 || g->gc.total + size < g->gc.memlimit) { -+ o = (GCobj *)g->allocf(g->allocd, NULL, 0, size); -+ } - if (o == NULL) - lj_err_mem(L); - lua_assert(checkptr32(o)); - g->gc.total += size; -+ if (g->gc.total > g->gc.maxmem) { -+ g->gc.maxmem = g->gc.total; -+ } - setgcrefr(o->gch.nextgc, g->gc.root); - setgcref(g->gc.root, o); - newwhite(g, o); -diff -Naur LuaJIT-2.0.2/src/lj_obj.h luajit-2_0_2/src/lj_obj.h ---- LuaJIT-2.0.2/src/lj_obj.h 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_obj.h 2013-12-03 21:17:05.042524855 +0000 -@@ -490,6 +490,8 @@ - - typedef struct GCState { - MSize total; /* Memory currently allocated. */ -+ MSize maxmem; /* Maximum Memory that was allocated */ -+ MSize memlimit; /* Maximum Memory limit 0 = unlimited */ - MSize threshold; /* Memory threshold. */ - uint8_t currentwhite; /* Current white color. */ - uint8_t state; /* GC state. */ -diff -Naur LuaJIT-2.0.2/src/lj_state.c luajit-2_0_2/src/lj_state.c ---- LuaJIT-2.0.2/src/lj_state.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_state.c 2013-12-03 21:17:05.042524855 +0000 -@@ -208,6 +208,8 @@ - setgcref(g->gc.root, obj2gco(L)); - setmref(g->gc.sweep, &g->gc.root); - g->gc.total = sizeof(GG_State); -+ g->gc.maxmem = g->gc.total; -+ g->gc.memlimit = 0; - g->gc.pause = LUAI_GCPAUSE; - g->gc.stepmul = LUAI_GCMUL; - lj_dispatch_init((GG_State *)L); -diff -Naur LuaJIT-2.0.2/src/lua.h luajit-2_0_2/src/lua.h ---- LuaJIT-2.0.2/src/lua.h 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lua.h 2013-12-03 21:17:05.042524855 +0000 -@@ -226,6 +226,9 @@ - #define LUA_GCSTEP 5 - #define LUA_GCSETPAUSE 6 - #define LUA_GCSETSTEPMUL 7 -+#define LUA_GCSETMEMLIMIT 8 -+#define LUA_GCMAXCOUNT 9 -+#define LUA_GCMAXCOUNTB 10 - - LUA_API int (lua_gc) (lua_State *L, int what, int data); - -@@ -340,6 +343,7 @@ - LUA_API lua_Hook lua_gethook (lua_State *L); - LUA_API int lua_gethookmask (lua_State *L); - LUA_API int lua_gethookcount (lua_State *L); -+LUA_API int lua_gethookcountremaining (lua_State *L); - - /* From Lua 5.2. */ - LUA_API void *lua_upvalueid (lua_State *L, int idx, int n); -diff -Naur LuaJIT-2.0.2/src/msvcbuild.bat luajit-2_0_2/src/msvcbuild.bat ---- LuaJIT-2.0.2/src/msvcbuild.bat 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/msvcbuild.bat 2013-12-03 21:24:09.434507458 +0000 -@@ -20,8 +20,8 @@ - @set LJLIB=lib /nologo - @set DASMDIR=..\dynasm - @set DASM=%DASMDIR%\dynasm.lua --@set LJDLLNAME=lua51.dll --@set LJLIBNAME=lua51.lib -+@set LJDLLNAME=lua.dll -+@set LJLIBNAME=lua.lib - @set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c - - %LJCOMPILE% host\minilua.c diff --git a/cmake/luasandboxConfig.cmake.in b/cmake/luasandboxConfig.cmake.in index fcde622..1b068e0 100644 --- a/cmake/luasandboxConfig.cmake.in +++ b/cmake/luasandboxConfig.cmake.in @@ -2,13 +2,13 @@ set(LUASANDBOX_VERSION x.y.z) @PACKAGE_INIT@ -set_and_check(LUASANDBOX_INCLUDE_DIR "@PACKAGE_INCLUDE_INSTALL_DIR@") -set_and_check(LUASANDBOX_LIB_DIR "@PACKAGE_LIB_INSTALL_DIR@") - -FIND_LIBRARY(LUASANDBOX_LUASB_LIBRARY NAMES luasb PATHS ${PACKAGE_LIB_INSTALL_DIR}) -FIND_LIBRARY(LUASANDBOX_UTIL_LIBRARY NAMES luasandboxutil PATHS ${PACKAGE_LIB_INSTALL_DIR}) -FIND_LIBRARY(LUASANDBOX_LIBRARY NAMES luasandbox PATHS ${PACKAGE_LIB_INSTALL_DIR}) -FIND_LIBRARY(LUASANDBOX_HEKA_LIBRARY NAMES luasandboxheka PATHS ${PACKAGE_LIB_INSTALL_DIR}) -FIND_PATH(LUASANDBOX_MODULES NAMES modules PATHS ${PACKAGE_LIB_INSTALL_DIR}luasandbox) +set_and_check(LUASANDBOX_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@") +set_and_check(LUASANDBOX_LIB_DIR "@PACKAGE_CMAKE_INSTALL_LIBDIR@") + +FIND_LIBRARY(LUASANDBOX_LUASB_LIBRARY NAMES luasb PATHS ${LUASANDBOX_LIB_DIR}) +FIND_LIBRARY(LUASANDBOX_UTIL_LIBRARY NAMES luasandboxutil PATHS ${LUASANDBOX_LIB_DIR}) +FIND_LIBRARY(LUASANDBOX_LIBRARY NAMES luasandbox PATHS ${LUASANDBOX_LIB_DIR}) +FIND_LIBRARY(LUASANDBOX_HEKA_LIBRARY NAMES luasandboxheka PATHS ${LUASANDBOX_LIB_DIR}) +FIND_PATH(LUASANDBOX_MODULES NAMES modules PATHS ${LUASANDBOX_LIB_DIR}/luasandbox) set(LUASANDBOX_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY} ${LUASANDBOX_LUASB_LIBRARY}) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 7da0435..eba9c51 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -27,8 +27,6 @@ else() set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra") if (NOT WIN32) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fvisibility=hidden") - else() - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D__USE_MINGW_ANSI_STDIO") endif() set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") set(CMAKE_C_FLAGS_DEBUG "-g") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d0df9d2..51a8e95 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -11,41 +11,25 @@ luasandbox_serialize.c set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) -add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) - -add_dependencies(luasandbox ${LUA_PROJECT}) - -if(WIN32) - set(LIB_DIR bin) -else() - set(LIB_DIR lib) -endif() - if(MSVC) add_definitions(-D_CRT_SECURE_NO_WARNINGS) - set(LUA_SANDBOX_LIBS "${EP_BASE}/lib/luasb.lib") - install(DIRECTORY "${EP_BASE}/${LIB_DIR}/" DESTINATION ${LIB_DIR} COMPONENT core PATTERN "*.dll") -elseif(MINGW) - add_definitions(-D_MINGW) - set(LUA_SANDBOX_LIBS "${EP_BASE}/bin/libluasb.dll") - set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) - install(DIRECTORY "${EP_BASE}/${LIB_DIR}/" DESTINATION ${LIB_DIR} COMPONENT core PATTERN "*.dll") -else() - if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - set(LINK_DL "-ldl") - endif() - set(LUA_SANDBOX_LIBS "${EP_BASE}/${LIB_DIR}/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}" ${LINK_DL} ${LIBM_LIBRARY}) + install(DIRECTORY "${EP_BASE}/lib/" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core PATTERN "*.dll") endif() -set_target_properties(luasandbox PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) +add_dependencies(luasandbox ${LUA_PROJECT}) -target_link_libraries(luasandbox luasandboxutil ${LUA_SANDBOX_LIBS}) +set_target_properties(luasandbox PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_link_libraries(luasandbox luasandboxutil luasb ${CMAKE_DL_LIBS}) +if(LIBM_LIBRARY) + target_link_libraries(luasandbox ${LIBM_LIBRARY}) +endif() -install(TARGETS luasandbox DESTINATION ${LIB_DIR} COMPONENT core) -install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib COMPONENT core PATTERN "lua" EXCLUDE) -install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION lib/${PROJECT_NAME}/modules COMPONENT core) -install(DIRECTORY "${EP_BASE}/io/lib/lua/" DESTINATION lib/${PROJECT_NAME}/io_modules COMPONENT core) -install(DIRECTORY "${EP_BASE}/include/" DESTINATION include COMPONENT core) +install(TARGETS luasandbox DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) +install(DIRECTORY "${EP_BASE}/lib/" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core PATTERN "lua" EXCLUDE) +install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/modules COMPONENT core) +install(DIRECTORY "${EP_BASE}/io/lib/lua/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/io_modules COMPONENT core) +install(DIRECTORY "${EP_BASE}/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core) add_subdirectory(util) add_subdirectory(heka) diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt index 02850cf..dbf37b3 100644 --- a/src/cli/CMakeLists.txt +++ b/src/cli/CMakeLists.txt @@ -9,4 +9,4 @@ if(LIBM_LIBRARY) target_link_libraries(lsb_heka_cat ${LIBM_LIBRARY}) endif() -install(TARGETS lsb_heka_cat DESTINATION bin COMPONENT core) +install(TARGETS lsb_heka_cat DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT core) diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index 2fed407..ed55c4c 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -29,5 +29,5 @@ if(LIBRTKAFKA_LIBRARY) target_link_libraries(luasandboxheka ${LIBRTKAFKA_LIBRARY}) endif() -install(TARGETS luasandboxheka DESTINATION ${LIB_DIR} COMPONENT core) +install(TARGETS luasandboxheka DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) add_subdirectory(test) diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt index 4806638..3a2cd5c 100644 --- a/src/heka/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -4,7 +4,7 @@ configure_file(test.h.in test.h ESCAPE_QUOTES) include_directories(${CMAKE_CURRENT_BINARY_DIR}) -set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/${LIB_DIR};${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/heka") +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/heka") add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index a6d1d36..d251dc3 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -4,7 +4,7 @@ add_executable(test_luasandbox test_luasandbox.c) target_link_libraries(test_luasandbox luasandbox luasandboxutil) -set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/${LIB_DIR};${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util") +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util") add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index ef6f7f9..26b63d5 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -27,6 +27,5 @@ if(ZLIB_FOUND) target_link_libraries(luasandboxutil ${ZLIB_LIBRARIES}) endif() -install(TARGETS luasandboxutil DESTINATION ${LIB_DIR} COMPONENT core) +install(TARGETS luasandboxutil DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) add_subdirectory(test) - From be7636e056e9785895b96a7a2bf51b6bfa09712e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 9 Jun 2016 10:43:15 -0700 Subject: [PATCH 156/235] Pull in the zlib functionality needed by the data-pipeline https://bugzilla.mozilla.org/show_bug.cgi?id=1278365 --- CMakeLists.txt | 4 +- docs/lsb_compression.md | 24 ++++++++ docs/lsb_hash.md | 23 ++++++++ gen_gh_pages.lua | 5 ++ include/luasandbox/util/util.h | 15 +++-- src/CMakeLists.txt | 1 + src/luasandbox.c | 2 + src/luasandbox_impl.h | 21 +++++++ src/luasandbox_modules.c | 94 ++++++++++++++++++++++++++++++++ src/test/lua/lsb_compression.lua | 29 ++++++++++ src/test/lua/lsb_hash.lua | 25 +++++++++ src/test/test_luasandbox.c | 27 +++++++++ src/util/test/test_util.c | 65 ++++++++++++++++++++++ src/util/util.c | 8 +++ 14 files changed, 337 insertions(+), 6 deletions(-) create mode 100644 docs/lsb_compression.md create mode 100644 docs/lsb_hash.md create mode 100644 src/luasandbox_modules.c create mode 100644 src/test/lua/lsb_compression.lua create mode 100644 src/test/lua/lsb_hash.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index c68c3b3..583c761 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 20) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_MINOR 21) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/docs/lsb_compression.md b/docs/lsb_compression.md new file mode 100644 index 0000000..7522efe --- /dev/null +++ b/docs/lsb_compression.md @@ -0,0 +1,24 @@ +# Lua Sandbox Compression Module + +## Overview +Built-in Lua Sandbox module for various types of compresssion. + + +### Example Usage +```lua +require "lsb_compression" +local inflated = lsb_compression.ungzip("\031\139\008\000\000\000\000\000\000\003\075\203\207\079\074\044\226\002\000\071\151\044\178\007\000\000\000") +-- inflated == "foobar\n" +``` + +### Functions + +#### ungzip +Inflates a gzipped string (only availble if libz is installed on the system). + +*Arguments* +- bytes (string) - gzip compressed string to inflate +- max_size (number, nil) - maximum size the of the resulting string (default 0 (unlimited)) + +*Return* +- inflated (string) - nil if bytes are not a gzip string or the output would exceed `max_size` diff --git a/docs/lsb_hash.md b/docs/lsb_hash.md new file mode 100644 index 0000000..3b0bb4f --- /dev/null +++ b/docs/lsb_hash.md @@ -0,0 +1,23 @@ +# Lua Sandbox Hash Module + +## Overview +Built-in Lua Sandbox module for various types of hashing. + +### Example Usage +```lua +require "lsb_hash" +local crc = lsb_hash.crc32("foobar") +-- crc == 2666930069 +``` + +### Functions + +#### crc32 + +CRC-32 checksum (only availble if libz is installed on the system) . + +*Arguments* +- value (string) - string to checksum + +*Return* +- checksum (number) - unsigned int diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index 75447fa..bfd70ab 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -189,6 +189,11 @@ local function output_menu(before, after, paths, version)
  • Source Documentation
      • +
    • Built-in Modules
      • +
    • C Modules
      • bloom_filter
        • diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 36fff96..4966828 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -11,6 +11,7 @@ #include #include +#include #include "../error.h" @@ -72,10 +73,18 @@ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); */ LSB_UTIL_EXPORT bool lsb_set_tz(const char *tz); -#ifdef HAVE_ZLIB +/** + * ZLIB CRC-32 checksum + * + * @param buf - array of bytes + * @param buf_len - length of the array + * + * @return LSB_UTIL_EXPORT unsigned checksum + */ +LSB_UTIL_EXPORT uint32_t lsb_crc32(const char* buf, size_t buf_len); /** - * Ungzip the provided string into a new string + * ZLIB Inflate the provided string into a new string * * @param s String to ungzip * @param s_len Length of the string to ungzip @@ -87,8 +96,6 @@ LSB_UTIL_EXPORT bool lsb_set_tz(const char *tz); LSB_UTIL_EXPORT char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len); -#endif - #ifdef __cplusplus } #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 51a8e95..fa9bf4e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,7 @@ set(LUA_SANDBOX_SRC luasandbox.c +luasandbox_modules.c luasandbox_output.c luasandbox_serialize.c ) diff --git a/src/luasandbox.c b/src/luasandbox.c index 5c1d2ff..4b90f4c 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -44,6 +44,8 @@ static const luaL_Reg preload_module_list[] = { { LUA_OSLIBNAME, luaopen_os }, { LUA_STRLIBNAME, luaopen_string }, { LUA_MATHLIBNAME, luaopen_math }, + { LSB_HASH_MODULE, luaopen_lsb_hash }, + { LSB_COMPRESSION_MODULE, luaopen_lsb_compression }, { NULL, NULL } }; diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index 0a5f8c5..8a17474 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -13,6 +13,9 @@ #include "luasandbox/lua.h" #include "luasandbox/util/output_buffer.h" +#define LSB_HASH_MODULE "lsb_hash" +#define LSB_COMPRESSION_MODULE "lsb_compression" + struct lsb_lua_sandbox { lua_State *lua; void *parent; @@ -43,4 +46,22 @@ lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb); */ lsb_err_value restore_global_data(lsb_lua_sandbox *lsb); +/** + * Built in Lua hash module + * + * @param lua Lua state + * + * @return int number of return values + */ +int luaopen_lsb_hash(lua_State *lua); + +/** + * Built in Lua compression module + * + * @param lua Lua state + * + * @return int number of return values + */ +int luaopen_lsb_compression(lua_State *lua); + #endif diff --git a/src/luasandbox_modules.c b/src/luasandbox_modules.c new file mode 100644 index 0000000..6ac37e7 --- /dev/null +++ b/src/luasandbox_modules.c @@ -0,0 +1,94 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Additional built in Lua sandbox modules @file */ + +#include + +#include "lauxlib.h" +#include "lua.h" +#include "luasandbox/util/util.h" +#include "luasandbox_impl.h" + +#ifdef HAVE_ZLIB +#include + + +static int zlib_crc32(lua_State *lua) +{ + size_t len; + const char *buf; + + if (lua_type(lua, 1) == LUA_TSTRING) { + buf = lua_tolstring(lua, 1, &len); + } else { + return luaL_argerror(lua, 1, "must be a string"); + } + lua_pushnumber(lua, (lua_Number)lsb_crc32(buf, len)); + return 1; +} + + +static int zlib_ungzip(lua_State *lua) +{ + size_t len; + const char *buf; + + if (lua_type(lua, 1) == LUA_TSTRING) { + buf = lua_tolstring(lua, 1, &len); + } else { + return luaL_argerror(lua, 1, "must be a string"); + } + + lua_Number mss = lua_tonumber(lua, 2); // unlimited if max is not provided + int t = lua_type(lua, 2); + if (mss < 0 || (t != LUA_TNUMBER && t != LUA_TNONE && t != LUA_TNIL)) { + return luaL_argerror(lua, 2, "must be a positive number or nil"); + } + + char *gbuf = lsb_ungzip(buf, len, (size_t)mss, &len); + if (gbuf) { + lua_pushlstring(lua, gbuf, len); + free(gbuf); + } else { + lua_pushnil(lua); + } + return 1; +} +#endif + + +static const struct luaL_reg hashlib_f[] = +{ +#ifdef HAVE_ZLIB + { "crc32", zlib_crc32 }, +#endif + { NULL, NULL } +}; + + +int luaopen_lsb_hash(lua_State *lua) +{ + luaL_register(lua, LSB_HASH_MODULE, hashlib_f); + return 1; +} + + +static const struct luaL_reg compressionlib_f[] = +{ +#ifdef HAVE_ZLIB + { "ungzip", zlib_ungzip }, +#endif + { NULL, NULL } +}; + + +int luaopen_lsb_compression(lua_State *lua) +{ + luaL_register(lua, LSB_COMPRESSION_MODULE, compressionlib_f); + return 1; +} + diff --git a/src/test/lua/lsb_compression.lua b/src/test/lua/lsb_compression.lua new file mode 100644 index 0000000..6e0796b --- /dev/null +++ b/src/test/lua/lsb_compression.lua @@ -0,0 +1,29 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local comp = require "lsb_compression" + +local tests = { + {comp.ungzip, nil, nil, false, "bad argument #1 to '?' (must be a string)"}, + {comp.ungzip, "string", true, false, "bad argument #2 to '?' (must be a positive number or nil)"}, + {comp.ungzip, "string", -100, false, "bad argument #2 to '?' (must be a positive number or nil)"}, + {comp.ungzip, "\031\139\008\000\000\000\000\000\000\003\075\203\207\079\074\044\226\002\000\071\151\044\178\007\000\000\000", nil, true, "foobar\n"}, + {comp.ungzip, "\031\139\008\000\000\000\000\000\000\003\075\203\207\079\074\044\226\002\000\071\151\044\178\007\000\000\000", 4, true, nil}, + {comp.ungzip, "not gzip", nil, true, nil}, +} + +function process () + for i,v in ipairs(tests) do + if v[1] then + if v[4] then + local result = v[1](v[2], v[3]) + assert(result == v[5], result) + else + local ok, err = pcall(v[1], v[2], v[3]) + assert(not ok and err == v[5], err) + end + end + end + return 0 +end diff --git a/src/test/lua/lsb_hash.lua b/src/test/lua/lsb_hash.lua new file mode 100644 index 0000000..002aa19 --- /dev/null +++ b/src/test/lua/lsb_hash.lua @@ -0,0 +1,25 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local hash = require "lsb_hash" + +local tests = { + {hash.crc32 , nil , "bad argument #1 to '?' (must be a string)"}, + {hash.crc32 , "foobar", 2666930069}, +} + +function process () + for i,v in ipairs(tests) do + if v[1] then + if type(v[2]) == "string" then + local result = v[1](v[2]) + assert(result == v[3], result) + else + local ok, err = pcall(v[1], v[2]) + assert(not ok and err == v[3], err) + end + end + end + return 0 +end diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index c0018e1..97d22db 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -1394,6 +1394,32 @@ static char* test_print_logger() } +static char* test_builtin_modules() +{ + const char *tests[] = { + "lua/lsb_hash.lua", + "lua/lsb_compression.lua", + NULL + }; + + for (int i = 0; tests[i]; ++i) { + lsb_lua_sandbox *sb = lsb_create(NULL, tests[i], test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + int result = process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + } + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -1693,6 +1719,7 @@ static char* all_tests() mu_run_test(test_print); mu_run_test(test_print_disabled); mu_run_test(test_print_logger); + mu_run_test(test_builtin_modules); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index 4155091..7fa3e32 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -6,6 +6,7 @@ /** @brief lsb_util unit tests @file */ +#include #include #include #include @@ -63,6 +64,68 @@ static char* test_lsb_set_tz() } +static char* test_lsb_crc32() +{ + uint32_t checksum = lsb_crc32(NULL, 0); + mu_assert(checksum == 0U, "received: %" PRIu32, checksum); + + checksum = lsb_crc32("", 0); + mu_assert(checksum == 0U, "received: %" PRIu32, checksum); + + checksum = lsb_crc32("foobar", 6); + mu_assert(checksum == 2666930069U, "received: %" PRIu32, checksum); + return NULL; +} + + +static char* test_lsb_ungzip() +{ + size_t alen = 200; + char gz[] = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x4b\x4c\x1c\x1e\x00\x00\x58\xf0\x9a\x59\xc8\x00\x00\x00"; + + size_t len = 0; + char *ugz = lsb_ungzip(gz, sizeof(gz)-1, 0, &len); + mu_assert(ugz, "lsb_ungzip failed"); + mu_assert(len == alen, "received: %" PRIuSIZE, len); + for (size_t i = 0; i < alen; ++i) { + mu_assert(ugz[i] == 'a', "pos: %" PRIuSIZE " char: %c", i, ugz[i]); + } + free(ugz); + + ugz = lsb_ungzip(gz, sizeof(gz)-1, alen, &len); + mu_assert(ugz, "lsb_ungzip failed"); + mu_assert(len == alen, "received: %" PRIuSIZE, len); + for (size_t i = 0; i < alen; ++i) { + mu_assert(ugz[i] == 'a', "pos: %" PRIuSIZE " char: %c", i, ugz[i]); + } + free(ugz); + + ugz = lsb_ungzip(gz, sizeof(gz)-1, 199, &len); + if (ugz) { + free(ugz); + mu_assert(false, "output larger than max lsb_ungzip succeeded"); + } + + ugz = lsb_ungzip(NULL, 0, alen, &len); + if (ugz) { + free(ugz); + mu_assert(false, "NULL lsb_ungzip succeeded"); + } + + ugz = lsb_ungzip(gz, sizeof(gz)-1, sizeof(gz)-2, &len); + if (ugz) { + free(ugz); + mu_assert(false, "max smaller than input lsb_ungzip succeeded"); + } + + ugz = lsb_ungzip(gz, sizeof(gz)-1, (sizeof(gz)-1) * 2 - 1, &len); + if (ugz) { + free(ugz); + mu_assert(false, "max smaller than 2x input lsb_ungzip succeeded"); + } + return NULL; +} + static char* benchmark_lsb_get_time() { @@ -98,6 +161,8 @@ static char* all_tests() mu_run_test(test_lsb_lp2); mu_run_test(test_lsb_read_file); mu_run_test(test_lsb_set_tz); + mu_run_test(test_lsb_crc32); + mu_run_test(test_lsb_ungzip); mu_run_test(benchmark_lsb_get_time); return NULL; diff --git a/src/util/util.c b/src/util/util.c index 845f3ae..7be3047 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -131,6 +131,14 @@ bool lsb_set_tz(const char *tz) #ifdef HAVE_ZLIB +uint32_t lsb_crc32(const char *buf, size_t buf_len) +{ + uint32_t crc = (uint32_t)crc32(0L, Z_NULL, 0); + crc = (uint32_t)crc32(crc, (const Bytef *)buf, buf_len); + return crc; +} + + char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len) { if (!s || (max_len && s_len > max_len)) { From 4a0f1ca87b4451dfab0d1ec9d64201ab80b1a99b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 9 Jun 2016 13:16:34 -0700 Subject: [PATCH 157/235] Remove the lsb_compression module from the Heka analysis sandbox - it can be used to OOM the system - it forces decompression to happen once in the input only --- CMakeLists.txt | 2 +- docs/heka/analysis.md | 2 ++ gen_gh_pages.lua | 4 ++-- src/heka/sandbox.c | 2 ++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 583c761..1a3d389 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 21) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index fd54ac5..b0dcba1 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -18,6 +18,8 @@ Hindsight reference implementation. - the entire module is inaccessible - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - getenv, execute, exit, remove, rename, setlocale, tmpname +- [lsb_compression](../lsb_compression.html) + - the entire module is inaccessible ## Required Lua Functions (called by the host) diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index bfd70ab..be2033e 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -191,8 +191,8 @@ local function output_menu(before, after, paths, version)
        • Built-in Modules
        • C Modules
            • diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index baa2561..9111d18 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -348,6 +348,8 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) lua_setfield(lua, -2, "io"); lua_pushboolean(lua, true); lua_setfield(lua, -2, "coroutine"); + lua_pushboolean(lua, true); + lua_setfield(lua, -2, "lsb_compression"); lua_setfield(lua, 1, "disable_modules"); break; } From 71c213b077419282d4a49e525beffde921731e6d Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 13 Jun 2016 14:41:23 -0700 Subject: [PATCH 158/235] Conditionally build Lua modules used in the Mozilla data pipeline - The following modules: geoip, openssl, postgres and luasec are not Mozilla specific and are generally useful so they will built with the sandbox when the supporting libraries are available. - Update lpeg to 1.0 - Update luasec to 0.6 from 0.4 --- CMakeLists.txt | 23 +++++++-- cmake/CMakeLists.txt.lpeg | 36 +++++++++++++ cmake/CMakeLists.txt.lua_geoip | 54 ++++++++++++++++++++ cmake/CMakeLists.txt.lua_openssl | 64 +++++++++++++++++++++++ cmake/CMakeLists.txt.lua_postgres | 32 ++++++++++++ cmake/CMakeLists.txt.lua_sec | 43 ++++++++++++++++ cmake/CMakeLists.txt.lua_snappy | 35 +++++++++++++ cmake/externals.cmake | 84 +++++++++++++++++++++++++++---- cmake/fixup_lua_sec.cmake | 1 - gen_gh_pages.lua | 41 +++++++++------ 10 files changed, 383 insertions(+), 30 deletions(-) create mode 100644 cmake/CMakeLists.txt.lpeg create mode 100644 cmake/CMakeLists.txt.lua_geoip create mode 100644 cmake/CMakeLists.txt.lua_openssl create mode 100644 cmake/CMakeLists.txt.lua_postgres create mode 100644 cmake/CMakeLists.txt.lua_sec create mode 100644 cmake/CMakeLists.txt.lua_snappy delete mode 100644 cmake/fixup_lua_sec.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1a3d389..a698ae6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 21) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) @@ -35,10 +35,11 @@ if(LUA_JIT) add_definitions(-DLUA_JIT) endif() +option(LUA_SNAPPY "Include the Lua Snappy module (only used to support a deprecated Mozilla telemetry data format)" off) + find_library(LIBM_LIBRARY m) find_package(ZLIB) - if (ZLIB_FOUND) add_definitions(-DHAVE_ZLIB) endif() @@ -47,12 +48,26 @@ find_library(LIBRTKAFKA_LIBRARY rdkafka) if (LIBRTKAFKA_LIBRARY) add_definitions(-DHAVE_KAFKA) else() - message(STATUS "The Kafka module will not be built") + message(STATUS "librdkafka was not found, dependent modules will not be built") +endif() + +find_package(PostgreSQL) +if (NOT PostgreSQL_FOUND) + message(STATUS "PostgreSQL was not found, dependent modules will not be built") +endif() + +if (APPLE) + set(OPENSSL_ROOT_DIR /opt/local) endif() find_package(OpenSSL) if (NOT OPENSSL_FOUND) - message(STATUS "OpenSSL was not found, the LuaSec module will not be built") + message(STATUS "OpenSSL was not found, dependent modules will not be built") +endif() + +find_library(GEOIP_LIBRARY GeoIP) +if (NOT GEOIP_LIBRARY) + message(STATUS "GeoIP was not found, dependent modules will not be built") endif() find_package(Git REQUIRED) diff --git a/cmake/CMakeLists.txt.lpeg b/cmake/CMakeLists.txt.lpeg new file mode 100644 index 0000000..5c39d13 --- /dev/null +++ b/cmake/CMakeLists.txt.lpeg @@ -0,0 +1,36 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(lpeg C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox) +find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) + +set(INSTALL_PATH lib/lua) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +file(WRITE lpeg.def "EXPORTS\nluaopen_lpeg\n") +set(LPEG_SRC +lpcap.c +lpcode.c +lpprint.c +lptree.c +lpvm.c +lpeg.def +) + +add_library(lpeg SHARED ${LPEG_SRC}) +target_link_libraries(lpeg ${LUA_LIBRARY}) +install(TARGETS lpeg DESTINATION ${INSTALL_PATH}) +install(FILES re.lua DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lua_geoip b/cmake/CMakeLists.txt.lua_geoip new file mode 100644 index 0000000..1a3f089 --- /dev/null +++ b/cmake/CMakeLists.txt.lua_geoip @@ -0,0 +1,54 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(luageoip C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC -fms-extensions") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox) +find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) + +set(INSTALL_PATH lib/lua) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +file(WRITE geoip.def "EXPORTS\nluaopen_geoip") +set(GEOIP_SRC +src/lua-geoip.c +src/database.c +geoip.def +) + +file(WRITE city.def "EXPORTS\nluaopen_city\n") +set(CITY_SRC +src/city.c +src/database.c +city.def +) + +file(WRITE country.def "EXPORTS\nluaopen_country\n") +set(COUNTRY_SRC +src/country.c +src/database.c +country.def +) + +add_library(geoip SHARED ${GEOIP_SRC}) +target_link_libraries(geoip ${LUA_LIBRARY} ${GEOIP_LIBRARY}) +install(TARGETS geoip DESTINATION ${INSTALL_PATH}) + +add_library(city SHARED ${CITY_SRC}) +target_link_libraries(city ${LUA_LIBRARY} ${GEOIP_LIBRARY}) +install(TARGETS city DESTINATION ${INSTALL_PATH}/geoip) + +add_library(country SHARED ${COUNTRY_SRC}) +target_link_libraries(country ${LUA_LIBRARY} ${GEOIP_LIBRARY}) +install(TARGETS country DESTINATION ${INSTALL_PATH}/geoip) diff --git a/cmake/CMakeLists.txt.lua_openssl b/cmake/CMakeLists.txt.lua_openssl new file mode 100644 index 0000000..eae31a6 --- /dev/null +++ b/cmake/CMakeLists.txt.lua_openssl @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(luaopenssl C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-fPIC -DPTHREADS") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/deps) +find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) + +set(INSTALL_PATH lib/lua) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +file(WRITE luaopenssl.def "EXPORTS\nluaopen_openssl\n") +set(LUAOPENSSL_SRC +src/asn1.c +src/auxiliar.c +src/bio.c +src/callback.c +src/cipher.c +src/cms.c +src/compat.c +src/crl.c +src/csr.c +src/dh.c +src/digest.c +src/dsa.c +src/ec.c +src/engine.c +src/hmac.c +src/lbn.c +src/lhash.c +src/misc.c +src/ocsp.c +src/openssl.c +src/ots.c +src/pkcs12.c +src/pkcs7.c +src/pkey.c +src/rsa.c +src/ssl.c +src/th-lock.c +src/util.c +src/x509.c +src/xalgor.c +src/xattrs.c +src/xexts.c +src/xname.c +src/xstore.c +luaopenssl.def +) + +add_library(openssl SHARED ${LUAOPENSSL_SRC}) +target_link_libraries(openssl ${LUA_LIBRARY} ${OPENSSL_LIBRARIES}) +install(TARGETS openssl DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lua_postgres b/cmake/CMakeLists.txt.lua_postgres new file mode 100644 index 0000000..5e6bb13 --- /dev/null +++ b/cmake/CMakeLists.txt.lua_postgres @@ -0,0 +1,32 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(luapostgres C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox ${PostgreSQL_INCLUDE_DIRS}) +find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) + +set(INSTALL_PATH lib/lua) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +file(WRITE postgres.def "EXPORTS\nluaopen_postgres") +set(POSTGRES_SRC +src/ls_postgres.c +src/luasql.c +postgres.def +) + +add_library(postgres SHARED ${POSTGRES_SRC}) +target_link_libraries(postgres ${LUA_LIBRARY} ${PostgreSQL_LIBRARIES}) +install(TARGETS postgres DESTINATION ${INSTALL_PATH}/luasql) diff --git a/cmake/CMakeLists.txt.lua_sec b/cmake/CMakeLists.txt.lua_sec new file mode 100644 index 0000000..dd2c12c --- /dev/null +++ b/cmake/CMakeLists.txt.lua_sec @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(luasec C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-std=gnu99 -Wall -pedantic -fPIC -D_POSIX_C_SOURCE=199309L") + set(CMAKE_C_FLAGS_RELEASE "-O2") +endif() + +include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/src) +find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) + +set(INSTALL_PATH lib/lua) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +set(LUASOCKET_SRC +src/luasocket/buffer.c +src/luasocket/io.c +src/luasocket/timeout.c +src/luasocket/usocket.c +) +add_library(luasocket STATIC ${LUASOCKET_SRC}) +target_compile_definitions(luasocket PRIVATE LUASOCKET_DEBUG) + +file(WRITE luasec.def "EXPORTS\nluaopen_ssl_core\nlua_open_ssl_x509\nluaopen_ssl_context\n") +set(LUASEC_SRC +src/context.c +src/ssl.c +src/x509.c +luasec.def +) + +add_library(ssl SHARED ${LUASEC_SRC}) +target_link_libraries(ssl luasocket ${LUA_LIBRARY} ${OPENSSL_LIBRARIES}) +install(TARGETS ssl DESTINATION ${INSTALL_PATH}) +install(FILES src/ssl.lua DESTINATION ${INSTALL_PATH}) +install(FILES src/https.lua DESTINATION ${INSTALL_PATH}/ssl) diff --git a/cmake/CMakeLists.txt.lua_snappy b/cmake/CMakeLists.txt.lua_snappy new file mode 100644 index 0000000..2534503 --- /dev/null +++ b/cmake/CMakeLists.txt.lua_snappy @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(luasnappy C CXX) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-fPIC -Wall") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox) +find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) + +set(INSTALL_PATH lib/lua) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +file(WRITE snappy.def "EXPORTS\nluaopen_snappy") +set(SNAPPY_SRC +lua-snappy.cc +snappy/snappy.cc +snappy/snappy-c.cc +snappy/snappy-sinksource.cc +snappy/snappy-stubs-internal.cc +snappy.def +) + +add_library(snappy SHARED ${SNAPPY_SRC}) +target_link_libraries(snappy ${LUA_LIBRARY}) +install(TARGETS snappy DESTINATION ${INSTALL_PATH}) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 2b9c9be..089dec2 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -9,7 +9,13 @@ get_filename_component(GIT_PATH ${GIT_EXECUTABLE} PATH) set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) -set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include -DUSE_RPATH=false --no-warn-unused-cli) +set(SANDBOX_CMAKE_ARGS +-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} +-DCMAKE_INSTALL_PREFIX=${EP_BASE} +-DEP_BASE=${EP_BASE} +-DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include +-DUSE_RPATH=false +--no-warn-unused-cli) set(LUA_INCLUDE_DIR "${EP_BASE}/include/${PROJECT_NAME}") set(INST_ARGS "install/strip") @@ -36,9 +42,9 @@ endif() externalproject_add( lua_lpeg - GIT_REPOSITORY https://github.com/LuaDist/lpeg.git - GIT_TAG baf0dc90b9278360be719dbfb8e56d34ce3c61bd - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + URL http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.0.tar.gz + URL_MD5 0aec64ccd13996202ad0c099e2877ece + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lpeg /CMakeLists.txt CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) @@ -74,20 +80,76 @@ externalproject_add( add_dependencies(lua_socket ${LUA_PROJECT}) if (OPENSSL_FOUND) - if (APPLE) - set(SANDBOX_CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_PROJECT_luasec_INCLUDE=${CMAKE_SOURCE_DIR}/cmake/fixup_lua_sec.cmake) - endif() externalproject_add( lua_sec - GIT_REPOSITORY https://github.com/LuaDist/luasec.git - GIT_TAG 329a8789f142bbbfef4fd7ed506285a25c3c0e0e - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + GIT_REPOSITORY https://github.com/brunoos/luasec.git + GIT_TAG 20443861ebc3f6498ee7d9c70fbdaa059bec15e1 + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_sec /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} + -DOPENSSL_LIBRARIES=${OPENSSL_LIBRARIES} INSTALL_ARGS ${INST_ARGS} ) add_dependencies(lua_sec ${LUA_PROJECT}) + + externalproject_add( + lua_openssl + GIT_REPOSITORY https://github.com/zhaozg/lua-openssl.git + GIT_TAG 2e8930c5e13b52705ba9c390110e84b1f4748ba9 + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_openssl /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} + -DOPENSSL_LIBRARIES=${OPENSSL_LIBRARIES} + INSTALL_ARGS ${INST_ARGS} + ) + add_dependencies(lua_openssl ${LUA_PROJECT}) +endif() + +if (PostgreSQL_FOUND) + externalproject_add( + lua_postgres + GIT_REPOSITORY https://github.com/LuaDist/luasql-postgresql.git + GIT_TAG 29a3aa1964aeac93323ec5d1446ac7d32ec700df + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_postgres /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + -DPostgreSQL_INCLUDE_DIRS=${PostgreSQL_INCLUDE_DIRS} + -DPostgreSQL_LIBRARIES=${PostgreSQL_LIBRARIES} + INSTALL_ARGS ${INST_ARGS} + ) + add_dependencies(lua_postgres ${LUA_PROJECT}) endif() +if (GEOIP_LIBRARY) + externalproject_add( + lua_geoip + GIT_REPOSITORY https://github.com/agladysh/lua-geoip.git + GIT_TAG a07d261d8a2c7ff854fe6cd72cb8c2e16ec638ff + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_geoip /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + -DGEOIP_LIBRARY=${GEOIP_LIBRARY} + INSTALL_ARGS ${INST_ARGS} + ) + add_dependencies(lua_geoip ${LUA_PROJECT}) +endif() + +if (LUA_SNAPPY) + externalproject_add( + lua_snappy + GIT_REPOSITORY https://github.com/forhappy/lua-snappy.git + GIT_TAG 6b4f3f6736857d5c72aa6ed0ec566af6e222278d + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_snappy /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + INSTALL_ARGS ${INST_ARGS} + ) + add_dependencies(lua_snappy ${LUA_PROJECT}) +endif() + + # sandbox enhanced modules externalproject_add( diff --git a/cmake/fixup_lua_sec.cmake b/cmake/fixup_lua_sec.cmake deleted file mode 100644 index 95a506b..0000000 --- a/cmake/fixup_lua_sec.cmake +++ /dev/null @@ -1 +0,0 @@ -include_directories("/opt/local/include") diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index be2033e..0e5afd7 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -189,22 +189,35 @@ local function output_menu(before, after, paths, version)
          • Source Documentation
          ]]) From 6b771f8f2e02b16c9209741f5a7feda5fee2691e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 14 Jun 2016 11:24:16 -0700 Subject: [PATCH 159/235] Add a boolean 'restricted_headers' configuration option Indicates if certain headers (Logger and Hostname) are user modifiable from a sandbox. #139 --- CMakeLists.txt | 4 ++-- docs/heka/analysis.md | 11 +++++++---- docs/heka/input.md | 7 ++++++- docs/heka/message.md | 4 ++-- docs/heka/output.md | 11 ++++++++--- src/heka/message.c | 13 ++++++++++--- src/heka/sandbox.c | 16 +++++++++------- src/heka/sandbox_impl.h | 9 +++++---- src/heka/test/lua/aim.lua | 1 + src/heka/test/test_sandbox.c | 28 +++++++++++++++++----------- 10 files changed, 67 insertions(+), 37 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a698ae6..0469d68 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 21) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_MINOR 22) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index b0dcba1..4eb2136 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -109,10 +109,13 @@ Converts a Heka protobuf encoded message string into a Lua table or throws an er ### inject_message -Creates a new Heka protocol buffer message using the contents of the specified Lua table -(overwriting whatever is in the payload buffer). `Logger` and `Hostname` are set by -the infrastructure using the corresponding configuration setting and cannot be overridden -by the plugin. +Creates a new Heka protocol buffer message using the contents of the specified +Lua table (overwriting whatever is in the payload buffer). `Logger` and +`Hostname` are restricted header values. An override configuration option is +provided `restricted_headers`; when true (default) the headers are always set +to the configuration values; when false the headers are set to the values +provide in the message table, if no value is provided it defaults to the +appropriate configuration value. *Arguments* * msg ([Heka message table](message.html)) diff --git a/docs/heka/input.md b/docs/heka/input.md index ea05051..7de7df1 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -50,7 +50,12 @@ See [decode_message](analysis.html#decode_message) for details. ### inject_message -Sends a Heka protocol buffer message into the host. +Sends a Heka protocol buffer message into the host. For the Heka message table +argument `Logger` and `Hostname` are restricted header values. An override +configuration option is provided `restricted_headers`; when true the headers are +always set to the configuration values; when false (default) the headers are set +to the values provide in the message table, if no value is provided it defaults +to the appropriate configuration value. *Arguments* * msg ([Heka message table](message.html), [Heka stream reader](stream_reader.html) or Heka protobuf string) diff --git a/docs/heka/message.md b/docs/heka/message.md index d8532fd..463e36f 100644 --- a/docs/heka/message.md +++ b/docs/heka/message.md @@ -5,8 +5,8 @@ ```lua { Uuid = "data", -- auto generated if not a 16 byte raw binary UUID or a 36 character human readable UUID -Logger = "nginx", -- ignored by analysis sandboxes (the `Logger` configuration is used instead) -Hostname = "example.com", -- ignored by analysis sandboxes (the `Hostname` configuration is used instead) +Logger = "nginx", -- defaults to the Logger configuration value but can be overridden with the `restricted_headers` configuration +Hostname = "example.com", -- defaults to the Hostname configuration value but can be overridden with the `restricted_headers` configuration Timestamp = 1e9, -- auto generated if not a number Type = "TEST", Payload = "Test Payload", diff --git a/docs/heka/output.md b/docs/heka/output.md index 6702eab..0afa04c 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -76,10 +76,15 @@ See [decode_message](analysis.html#decode_message) for details. ### encode_message -Returns a Heka protocol buffer message using the contents of the specified Lua table. +Returns a Heka protocol buffer message using the contents of the specified Lua +table. `Logger` and `Hostname` are restricted header values. An override +configuration option is provided `restricted_headers`; when true the headers are +always set to the configuration values; when false (default) the headers are set +to the values provide in the message table, if no value is provided it defaults +to the appropriate configuration value. + Note: this operation uses the internal output buffer so it is goverened by the -`output_limit` configuration setting. If the `Logger` and `Hostname` are not set they will -be added by infrastructure using the corresponding configuration setting. +`output_limit` configuration setting. *Arguments* * msg ([Heka message table](message.html)) diff --git a/src/heka/message.c b/src/heka/message.c index 74bcb1b..2962462 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -835,9 +835,6 @@ int heka_encode_message(lua_State *lua) return luaL_error(lua, "encode_message() invalid upvalueindex"); } - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); - set_missing_headers(lua, 1, hsb); - lsb->output.pos = 0; lsb_err_value ret = heka_encode_message_table(lsb, 1); if (ret) { @@ -898,6 +895,16 @@ lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) } lua_pop(lsb->lua, 1); // remove timestamp + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + if (hsb->restricted_headers) { + lua_pushstring(lsb->lua, hsb->name); + lua_setfield(lsb->lua, idx, LSB_LOGGER); + lua_pushstring(lsb->lua, hsb->hostname); + lua_setfield(lsb->lua, idx, LSB_HOSTNAME); + } else { + set_missing_headers(lsb->lua, idx, hsb); + } + ret = lsb_pb_write_key(ob, LSB_PB_TIMESTAMP, LSB_PB_WT_VARINT); if (!ret) ret = lsb_pb_write_varint(ob, ts); if (!ret) ret = encode_string(lsb, ob, LSB_PB_TYPE, LSB_TYPE, idx); diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 9111d18..934325a 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -200,18 +200,13 @@ static int inject_message_analysis(lua_State *lua) return luaL_error(lua, "%s() invalid lightuserdata", im_func_name); } - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); - lua_pushstring(lua, hsb->name); - lua_setfield(lua, 1, LSB_LOGGER); - lua_pushstring(lua, hsb->hostname); - lua_setfield(lua, 1, LSB_HOSTNAME); - if (heka_encode_message_table(lsb, 1)) { return luaL_error(lua, "%s() failed: %s", im_func_name, lsb_get_error(lsb)); } size_t output_len = 0; const char *output = lsb_get_output(lsb, &output_len); + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); if (hsb->cb.aim(hsb->parent, output, output_len) != 0) { return luaL_error(lua, "%s() failed: rejected by the callback", im_func_name); @@ -402,6 +397,12 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) } lua_pop(lua, 1); // remove the Hostname + lua_getfield(lua, 1, "restricted_headers"); + if (lua_type(lua, -1) == LUA_TBOOLEAN) { + hsb->restricted_headers = lua_toboolean(lua, -1); + } + lua_pop(lua, 1); // remove the restricted_headers boolean + lua_pop(lua, 1); // remove the lsb_config table } @@ -652,6 +653,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, hsb->cb.aim = im; hsb->name = NULL; hsb->hostname = NULL; + hsb->restricted_headers = true; hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); if (!hsb->lsb) { @@ -666,7 +668,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, lsb_add_function(hsb->lsb, read_message, "read_message"); lsb_add_function(hsb->lsb, inject_message_analysis, "inject_message"); lsb_add_function(hsb->lsb, inject_payload, "inject_payload"); -// rename output to add_to_payload + // rename output to add_to_payload lua_getglobal(lua, "output"); lua_setglobal(lua, "add_to_payload"); lua_pushnil(lua); diff --git a/src/heka/sandbox_impl.h b/src/heka/sandbox_impl.h index 048940d..71413be 100644 --- a/src/heka/sandbox_impl.h +++ b/src/heka/sandbox_impl.h @@ -33,12 +33,13 @@ struct lsb_heka_sandbox { char *name; char *hostname; union { - lsb_heka_im_input iim; - lsb_heka_im_analysis aim; - lsb_heka_update_checkpoint ucp; + lsb_heka_im_input iim; + lsb_heka_im_analysis aim; + lsb_heka_update_checkpoint ucp; } cb; - char type; struct heka_stats stats; + char type; + bool restricted_headers; }; #endif diff --git a/src/heka/test/lua/aim.lua b/src/heka/test/lua/aim.lua index 07106e5..762df3f 100644 --- a/src/heka/test/lua/aim.lua +++ b/src/heka/test/lua/aim.lua @@ -7,6 +7,7 @@ require "string" -- Table tests local msgs = { {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {Timestamp = 1, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "ignore", Hostname = "spoof"}, } local err_msgs = { diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 4044b3f..1d8f7fd 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -47,13 +47,13 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, }; struct im_result results[] = { - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03iim\x4a\x02hn", .pb_len = 29, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03iim\x4a\x02hn", .pb_len = 29, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 156, .cp_numeric = NAN, .cp_string = "foo.log:123" }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 45, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x03iim\x4a\x02hn\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 165, .cp_numeric = NAN, .cp_string = "foo.log:123" }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x22\x03iim\x4a\x02hn\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 54, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x22\x03iim\x4a\x02hn\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 54, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x22\x03iim\x4a\x02hn\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 54, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, }; @@ -131,6 +131,7 @@ static int aim(void *parent, const char *pb, size_t pb_len) struct im_result results[] = { { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x1f\x7d\x09\x9e\xf5\x9d\x40\x1d\xa8\xaf\x6a\xff\xc3\x21\xeb\x42\x10\x80\x88\xe4\xaa\xa0\xa9\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 88, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x5b\x7d\xee\xa0\x02\xbc\x45\xbb\xaf\xa9\xcc\x2c\xdd\x65\xde\x45\x10\x80\x88\xdc\xad\xcd\xbf\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 110, .cp_numeric = NAN, .cp_string = NULL }, }; @@ -527,10 +528,13 @@ static char* test_im_input() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/iim.lua", NULL, "path = [[" TEST_LUA_PATH "]]\n" - "cpath = [[" TEST_LUA_CPATH "]]\n", &logger, iim); + "cpath = [[" TEST_LUA_CPATH "]]\n" + "Hostname = 'hn'\n" + "Logger = 'iim'\n", + &logger, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(9 == stats.im_cnt, "received %llu", stats.im_cnt); - mu_assert(440 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(494 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -544,8 +548,8 @@ static char* test_im_analysis() "Hostname = 'foo.com';Logger = 'aim'", &logger, aim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(3 == stats.im_cnt, "expected %llu", stats.im_cnt); - mu_assert(232 == stats.im_bytes, "expected %llu", stats.im_bytes); + mu_assert(4 == stats.im_cnt, "expected %llu", stats.im_cnt); + mu_assert(266 == stats.im_bytes, "expected %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_analysis failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -558,7 +562,9 @@ static char* test_encode_message() hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n" - "Hostname = 'sh';Logger = 'sl'", &logger, ucp); + "Hostname = 'sh'\n" + "Logger = 'sl'\n", + &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(172 == stats.out_max, "received %llu", stats.out_max); From 0494415002d00bbc80b6a5e11d0e6bbe659b13fa Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 14 Jun 2016 14:08:17 -0700 Subject: [PATCH 160/235] Add support for creating a high resolution time stamp when encoding messages --- CMakeLists.txt | 2 +- include/luasandbox/util/util.h | 11 ++++++++++- src/heka/message.c | 2 +- src/util/test/test_util.c | 17 ++++++++++++++++ src/util/util.c | 36 +++++++++++++++++++++++++++++++++- 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0469d68..3ba2128 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 22) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_PATCH 1) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 4966828..8211909 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -58,12 +58,21 @@ LSB_UTIL_EXPORT size_t lsb_lp2(unsigned long long x); LSB_UTIL_EXPORT char* lsb_read_file(const char *fn); /** - * Retrieves the highest resolution time available converted to nanoseconds + * Retrieves the highest resolution timer available converted to nanoseconds * * @return unsigned long long */ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); + +/** + * Retrieves the highest resolution time since Jan 1, 1970 converted to + * nanoseconds + * + * @return unsigned long long (time_ns) + */ +LSB_UTIL_EXPORT long long lsb_get_timestamp(); + /** * Sets the timezone environment variable for the time conversion functions * diff --git a/src/heka/message.c b/src/heka/message.c index 2962462..abd452c 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -891,7 +891,7 @@ lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) if (lua_isnumber(lsb->lua, -1)) { ts = (long long)lua_tonumber(lsb->lua, -1); } else { - ts = time(NULL) * 1000000000LL; + ts = lsb_get_timestamp(); } lua_pop(lsb->lua, 1); // remove timestamp diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index 7fa3e32..ec1e205 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -155,6 +155,22 @@ static char* benchmark_lsb_get_time() } +static char* benchmark_lsb_get_timestamp() +{ + int iter = 1000000; + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_get_timestamp(); + } + t = clock() - t; + printf("benchmark_lsb_get_timestamp(%d) - clock %g seconds\n", iter, + (double)t / CLOCKS_PER_SEC / iter); + long long delta = lsb_get_timestamp() / 1000000000LL - time(NULL); + mu_assert(delta > 0 ? delta < 1 : delta > -1, "delta %lld", delta); + return NULL; +} + + static char* all_tests() { mu_run_test(test_stub); @@ -165,6 +181,7 @@ static char* all_tests() mu_run_test(test_lsb_ungzip); mu_run_test(benchmark_lsb_get_time); + mu_run_test(benchmark_lsb_get_timestamp); return NULL; } diff --git a/src/util/util.c b/src/util/util.c index 7be3047..287dd0b 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -26,6 +26,7 @@ #endif #if defined(__MACH__) && defined(__APPLE__) +#include #include #include #endif @@ -99,7 +100,11 @@ unsigned long long lsb_get_time() QueryPerformanceCounter((LARGE_INTEGER *)&t); return (t / qpf * 1000000000ULL) + ((t % qpf) * 1000000000ULL / qpf); } else { - GetSystemTimeAsFileTime((FILETIME *)&t); + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + t = ft.dwHighDateTime; + t <<= 32; + t |= ft.dwLowDateTime; return t * 100ULL; } #else @@ -110,6 +115,35 @@ unsigned long long lsb_get_time() } +long long lsb_get_timestamp() +{ +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return ts.tv_sec * 1000000000LL + ts.tv_nsec; +#elif defined(__MACH__) && defined(__APPLE__) + clock_serv_t cclock; + mach_timespec_t mts; + host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); + clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + return mts.tv_sec * 1000000000LL + mts.tv_nsec; +#elif defined(_WIN32) + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + unsigned long long t = ft.dwHighDateTime; + t <<= 32; + t |= ft.dwLowDateTime; + t -= 16444736000000000ULL; // convert from Jan 1 1601 to Jan 1 1970 + return t * 100LL; +#else + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000000LL + tv.tv_usec * 1000LL; +#endif +} + + bool lsb_set_tz(const char *tz) { if (!tz) { From c8413be9036acf85260b7d9e0e2a1c309755386a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 15 Jun 2016 06:55:27 -0700 Subject: [PATCH 161/235] Grammar updates for strftime %f and Edge user agent support - Finish the work requested in #110 #106 --- CMakeLists.txt | 2 +- modules/lpeg/common_log_format.lua | 3 ++- modules/lpeg/date_time.lua | 1 + src/test/lua/lpeg_clf.lua | 2 ++ src/test/lua/lpeg_date_time.lua | 11 ++++++++++- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ba2128..a05942a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 22) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_PATCH 2) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/modules/lpeg/common_log_format.lua b/modules/lpeg/common_log_format.lua index 3da41ee..9e6f4c5 100644 --- a/modules/lpeg/common_log_format.lua +++ b/modules/lpeg/common_log_format.lua @@ -377,7 +377,8 @@ local ua_os_matchers = { } local ua_browser_matchers = { - {"Chrome" , ua_keyword("Chrome/")} + {"Edge" , ua_keyword("Edge/")} + , {"Chrome" , ua_keyword("Chrome/")} , {"Opera Mini" , ua_basic} , {"Opera Mobi" , ua_basic} , {"Opera" , ua_basic} diff --git a/modules/lpeg/date_time.lua b/modules/lpeg/date_time.lua index c80cf40..f1c959a 100644 --- a/modules/lpeg/date_time.lua +++ b/modules/lpeg/date_time.lua @@ -227,6 +227,7 @@ strftime_specifiers["y"] = l.Cg((l.digit * l.digit) / function (yy) return centu strftime_specifiers["d"] = date_mday strftime_specifiers["D"] = date_month * "/" * date_mday * "/" * strftime_specifiers["y"] strftime_specifiers["e"] = date_mday_sp +strftime_specifiers["f"] = l.Cg(l.digit^1 / ".%0" / tonumber, "sec_frac") strftime_specifiers["F"] = date_fullyear * "-" * date_month * "-" * date_mday strftime_specifiers["g"] = strftime_specifiers["y"] strftime_specifiers["G"] = date_fullyear diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index 440b4d8..e59b9a0 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -174,6 +174,7 @@ local function user_agent_normalization() ,"Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54" ,"Opera/12.02 (Android 4.1; Linux; Opera Mobi/ADR-1111101157; U; en-US) Presto/2.9.201 Version/12.02" ,"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14" + ,"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136" } local results = { {"Chrome" , 32 , "Windows 8"} @@ -193,6 +194,7 @@ local function user_agent_normalization() ,{"Opera Mini" , 10 , nil} ,{"Opera Mobi" , 12 , "Android"} ,{"Opera" , 12 , "Windows Vista"} + ,{"Edge" , 12 , "Windows 10"} } for i, v in ipairs(user_agents) do diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua index 57174cf..7721706 100644 --- a/src/test/lua/lpeg_date_time.lua +++ b/src/test/lua/lpeg_date_time.lua @@ -100,7 +100,7 @@ local function strftime_all() end local function strftime_composite() - local formats = {"%c","%D %T","%D %r","%d/%b/%Y:%H:%M:%S %z"} + local formats = {"%c", "%D %T", "%D %r", "%d/%b/%Y:%H:%M:%S %z"} local inputs = {"Mon Feb 10 16:46:36 2014", "02/10/14 16:46:36", "02/10/14 04:46:36 PM", "10/Feb/2014:16:46:36 +0000"} if os.date("%c"):find("^%d") then -- windows %c is non-standard inputs[1] = inputs[2] @@ -118,6 +118,14 @@ local function strftime_composite() end end +local function strftime_secfrac() + local g = dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S.%f") + local t = g:match("10/Feb/2014:16:46:36.124") + assert(t) + local ns = dt.time_to_ns(t) + assert(1392050796124000000 == ns, string.format("%d", ns)) +end + local function strftime_invalid() local r, g = pcall(dt.build_strftime_grammar, "%E") if r then @@ -135,6 +143,7 @@ function process() strftime_all() strftime_composite() strftime_invalid() + strftime_secfrac() return 0 end From 24da183bda2e5af3c02490f54b7c2032fad18928 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 15 Jun 2016 08:26:26 -0700 Subject: [PATCH 162/235] Add IE 11 #105 and Sync mobile #133 user agent parsing --- CMakeLists.txt | 2 +- modules/lpeg/common_log_format.lua | 29 ++++++++++++++++++----------- src/test/lua/lpeg_clf.lua | 6 ++++++ 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a05942a..4c94654 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 22) -set(CPACK_PACKAGE_VERSION_PATCH 2) +set(CPACK_PACKAGE_VERSION_PATCH 3) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/modules/lpeg/common_log_format.lua b/modules/lpeg/common_log_format.lua index 9e6f4c5..0ba25f8 100644 --- a/modules/lpeg/common_log_format.lua +++ b/modules/lpeg/common_log_format.lua @@ -346,7 +346,7 @@ end -- find the version of the product identified by the keyword local function ua_keyword(kw) return function(ua) - local i = ua:find(kw) + local i = ua:find(kw, 1, true) if i then local s = i + kw:len() if s then @@ -376,15 +376,21 @@ local ua_os_matchers = { , {"Windows NT 5.0" ,"Windows 2000" } } + +local fx_sync_android = ua_keyword("Firefox AndroidSync ") +local fx_sync_ios = ua_keyword("Firefox-iOS-Sync/") local ua_browser_matchers = { - {"Edge" , ua_keyword("Edge/")} - , {"Chrome" , ua_keyword("Chrome/")} - , {"Opera Mini" , ua_basic} - , {"Opera Mobi" , ua_basic} - , {"Opera" , ua_basic} - , {"MSIE" , ua_keyword("MSIE ")} - , {"Safari" , ua_basic} - , {"Firefox" , ua_keyword("Firefox/")} + {"Edge" , ua_keyword("Edge/")} + , {"Chrome" , ua_keyword("Chrome/")} + , {"Opera Mini" , ua_basic} + , {"Opera Mobi" , ua_basic} + , {"Opera" , ua_basic} + , {"MSIE" , ua_keyword("MSIE ")} + , {"Trident/7.0" , function() return 11, "MSIE" end} + , {"Safari" , ua_basic} + , {"Firefox AndroidSync" , function(ua) return fx_sync_android(ua), "FxSync", "Android" end} + , {"Firefox-iOS-Sync" , function(ua) return fx_sync_ios(ua), "FxSync", "iOS" end} + , {"Firefox" , ua_keyword("Firefox/")} } --[[ Public Interface --]] @@ -428,8 +434,9 @@ function normalize_user_agent(ua) for i, v in ipairs(ua_browser_matchers) do if ua:find(v[1], 1, true) then - browser = v[1] - version = v[2](ua) + version, browser, aos = v[2](ua) + if not browser then browser = v[1] end + if aos then os = aos end break end end diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua index e59b9a0..2ef68e2 100644 --- a/src/test/lua/lpeg_clf.lua +++ b/src/test/lua/lpeg_clf.lua @@ -175,6 +175,9 @@ local function user_agent_normalization() ,"Opera/12.02 (Android 4.1; Linux; Opera Mobi/ADR-1111101157; U; en-US) Presto/2.9.201 Version/12.02" ,"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14" ,"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136" + ,"Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko" + ,"Firefox AndroidSync 1.20.0.1.0 (Firefox)" + ,"Firefox-iOS-Sync/1.0 (Firefox)" } local results = { {"Chrome" , 32 , "Windows 8"} @@ -195,6 +198,9 @@ local function user_agent_normalization() ,{"Opera Mobi" , 12 , "Android"} ,{"Opera" , 12 , "Windows Vista"} ,{"Edge" , 12 , "Windows 10"} + ,{"MSIE" , 11 , "Windows 8.1"} + ,{"FxSync" , 1 , "Android"} + ,{"FxSync" , 1 , "iOS"} } for i, v in ipairs(user_agents) do From 2cd6126e0eb9cb06416ef55327a6515cf9779ce3 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 15 Jun 2016 13:22:00 -0700 Subject: [PATCH 163/235] Pull the host/hostname grammar into the lpeg.ip module #129 --- CMakeLists.txt | 2 +- modules/lpeg/ip_address.lua | 15 +++++++ modules/lpeg/mysql.lua | 9 +--- src/test/lua/lpeg_ip_address.lua | 70 ++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c94654..de753a7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) set(CPACK_PACKAGE_VERSION_MINOR 22) -set(CPACK_PACKAGE_VERSION_PATCH 3) +set(CPACK_PACKAGE_VERSION_PATCH 4) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/modules/lpeg/ip_address.lua b/modules/lpeg/ip_address.lua index 67b3571..c43fe4a 100644 --- a/modules/lpeg/ip_address.lua +++ b/modules/lpeg/ip_address.lua @@ -9,9 +9,13 @@ ### LPEG Grammars * `v4` - IPv4 address matcher * `v6` - IPv6 address matcher +* `hostname` - URI reg-name +* `host` - v6 | v4 | hostname +* `hostname_field` - returns the hostname as a Heka message field table `{value="127.0.0.1", representation="ipv4"}` --]] -- Imports +local string = require "string" local l = require "lpeg" l.locale(l) @@ -41,4 +45,15 @@ v6 = hg * hg * hg * hg * hg * hg * ls32 + (hg * hg^-5 + ":") * ":" * h16 + (hg * hg^-6 + ":") * ":" + +local unreserved = l.alnum + l.S"-._~" +local pct_encoded = l.P"%" * l.xdigit * l.xdigit +local sub_delims = l.S"!$&'()*+,;=" + +hostname = (unreserved + pct_encoded + sub_delims)^1 / string.lower +host = v6 + v4 + hostname +hostname_field = l.Ct(l.Cg(v6, "value") * l.Cg(l.Cc"ipv6", "representation")) + + l.Ct(l.Cg(v4, "value") * l.Cg(l.Cc"ipv4", "representation")) + + l.Ct(l.Cg(hostname, "value") * l.Cg(l.Cc"hostname", "representation")) + return M diff --git a/modules/lpeg/mysql.lua b/modules/lpeg/mysql.lua index 3ff2c10..1a5ccee 100644 --- a/modules/lpeg/mysql.lua +++ b/modules/lpeg/mysql.lua @@ -23,26 +23,19 @@ local ip = require "lpeg.ip_address" local M = {} setfenv(1, M) -- Remove external access to contain everything in the module -local pct_encoded = l.P"%" * l.xdigit * l.xdigit local unreserved = l.alnum + l.S"-._~" -local sub_delims = l.S"!$&'()*+,;=" - local space = l.space^1 local sep = l.P"\n" local sql_end = l.P";" * (l.P"\n" + -1) local line = (l.P(1) - sep)^0 * sep local float = l.digit^1 * "." * l.digit^1 / tonumber local integer = l.P"-"^-1 * l.digit^1 / tonumber -local host = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) - + l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation")) - + l.Ct(l.Cg((unreserved + pct_encoded + sub_delims)^1, "value") * l.Cg(l.Cc"hostname", "representation")) local time = l.P"# Time: " * line local user_legal = (1 - l.S"[]") -local host_legal = (l.alnum + l.S".-") local user_name = user_legal^0 * "[" * l.Cg(user_legal^0, "Username") * "]" -local host_name = host_legal^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" +local host_name = ip.hostname^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Query_time") diff --git a/src/test/lua/lpeg_ip_address.lua b/src/test/lua/lpeg_ip_address.lua index 3445bd9..13e62dc 100644 --- a/src/test/lua/lpeg_ip_address.lua +++ b/src/test/lua/lpeg_ip_address.lua @@ -91,10 +91,80 @@ local function ipv4_invalid() end end +local function hostname() + local tests = { + "localhost", + "example.com", + "127.org", + "example%20", + "UPPERCASE.COM", + "aA1-._~!$&'()*+,;=.com"} + for i,v in ipairs(tests) do + if not ip.hostname:match(v) then + error(v) + end + end +end + +local function hostname_invalid() + local grammar = ip.hostname * lpeg.P(-1) -- make it a full match + local tests = { + "example com", + "foo%AG", + "at@", + "colon:", + "lbracket[", + "rbracket]", + "hash#", + "quest?", + "slash/", + "lbrace{", + "rbrace}", + ""} + for i,v in ipairs(tests) do + if grammar:match(v) then + error(v) + end + end +end + +local function host() + local tests = { + "localhost", + "127.0.0.1", + "::1"} + for i,v in ipairs(tests) do + if not ip.host:match(v) then + error(v) + end + end +end + +local function host_field() + local tests = { + {"localhost", "hostname"}, + {"127.0.0.1", "ipv4"}, + {"::1", "ipv6"} + } + for i,v in ipairs(tests) do + local field = ip.hostname_field:match(v[1]) + if field then + assert(field.value == tests[i][1], field.value) + assert(field.representation == tests[i][2], field.representation) + else + error(v) + end + end +end + function process(tc) ipv6() ipv6_invalid() ipv4() ipv4_invalid() + hostname() + hostname_invalid() + host() + host_field() return 0 end From 45c92916f8bfdd62edb9dd974c2a422e32132889 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 15 Jun 2016 18:29:00 -0700 Subject: [PATCH 164/235] Doc updates --- docs/heka/{heka_json.md => json.md} | 0 gen_gh_pages.lua | 7 +++++++ modules/lpeg/ip_address.lua | 12 ++++++------ src/test/lua/lpeg_ip_address.lua | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) rename docs/heka/{heka_json.md => json.md} (100%) diff --git a/docs/heka/heka_json.md b/docs/heka/json.md similarity index 100% rename from docs/heka/heka_json.md rename to docs/heka/json.md diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index 0e5afd7..10e0dbf 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -195,6 +195,13 @@ local function output_menu(before, after, paths, version)
        • Shared Libraries
            • diff --git a/modules/lpeg/ip_address.lua b/modules/lpeg/ip_address.lua index c43fe4a..8f26326 100644 --- a/modules/lpeg/ip_address.lua +++ b/modules/lpeg/ip_address.lua @@ -11,7 +11,7 @@ * `v6` - IPv6 address matcher * `hostname` - URI reg-name * `host` - v6 | v4 | hostname -* `hostname_field` - returns the hostname as a Heka message field table `{value="127.0.0.1", representation="ipv4"}` +* `host_field` - returns the host as a Heka message field table `{value="127.0.0.1", representation="ipv4"}` --]] -- Imports @@ -50,10 +50,10 @@ local unreserved = l.alnum + l.S"-._~" local pct_encoded = l.P"%" * l.xdigit * l.xdigit local sub_delims = l.S"!$&'()*+,;=" -hostname = (unreserved + pct_encoded + sub_delims)^1 / string.lower -host = v6 + v4 + hostname -hostname_field = l.Ct(l.Cg(v6, "value") * l.Cg(l.Cc"ipv6", "representation")) - + l.Ct(l.Cg(v4, "value") * l.Cg(l.Cc"ipv4", "representation")) - + l.Ct(l.Cg(hostname, "value") * l.Cg(l.Cc"hostname", "representation")) +hostname = (unreserved + pct_encoded + sub_delims)^1 / string.lower +host = v6 + v4 + hostname +host_field = l.Ct(l.Cg(v6, "value") * l.Cg(l.Cc"ipv6", "representation")) + + l.Ct(l.Cg(v4, "value") * l.Cg(l.Cc"ipv4", "representation")) + + l.Ct(l.Cg(hostname, "value") * l.Cg(l.Cc"hostname", "representation")) return M diff --git a/src/test/lua/lpeg_ip_address.lua b/src/test/lua/lpeg_ip_address.lua index 13e62dc..e38a21b 100644 --- a/src/test/lua/lpeg_ip_address.lua +++ b/src/test/lua/lpeg_ip_address.lua @@ -147,7 +147,7 @@ local function host_field() {"::1", "ipv6"} } for i,v in ipairs(tests) do - local field = ip.hostname_field:match(v[1]) + local field = ip.host_field:match(v[1]) if field then assert(field.value == tests[i][1], field.value) assert(field.representation == tests[i][2], field.representation) From b1f0fe70306bfbf4944a421582f03f2ce2782198 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 19 Jun 2016 21:18:06 -0700 Subject: [PATCH 165/235] Add the correct path for the Lua headers in the sandbox build --- src/luasandbox_modules.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/luasandbox_modules.c b/src/luasandbox_modules.c index 6ac37e7..1404640 100644 --- a/src/luasandbox_modules.c +++ b/src/luasandbox_modules.c @@ -8,8 +8,8 @@ #include -#include "lauxlib.h" -#include "lua.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" #include "luasandbox/util/util.h" #include "luasandbox_impl.h" From 195474ad3d0caa2e0b95a337273042f86b9bbee8 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 22 Jun 2016 13:13:52 -0700 Subject: [PATCH 166/235] doc/format updates --- docs/heka/index.md | 4 ++-- include/luasandbox/util/util.h | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/heka/index.md b/docs/heka/index.md index 3ac826d..ec68e57 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -40,12 +40,12 @@ There are a few intentional changes between tho original Heka sandbox and this v #### Input Sandbox 1. A [Heka Stream Reader](stream_reader.html) Lua module was added. -1. A [Heka JSON](heka_json.html) Lua module was added. +1. A [Heka JSON](json.html) Lua module was added. #### Output Sandbox 1. [update_checkpoint](output.html#update_checkpoint) was added for batch and asynchronous processing. -1. A [Heka JSON](heka_json.html) Lua module was added. +1. A [Heka JSON](json.html) Lua module was added. ### Removals diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index 8211909..cd6d65d 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -30,7 +30,8 @@ #endif #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_NULL; @@ -90,7 +91,7 @@ LSB_UTIL_EXPORT bool lsb_set_tz(const char *tz); * * @return LSB_UTIL_EXPORT unsigned checksum */ -LSB_UTIL_EXPORT uint32_t lsb_crc32(const char* buf, size_t buf_len); +LSB_UTIL_EXPORT uint32_t lsb_crc32(const char *buf, size_t buf_len); /** * ZLIB Inflate the provided string into a new string @@ -103,7 +104,7 @@ LSB_UTIL_EXPORT uint32_t lsb_crc32(const char* buf, size_t buf_len); * @return char* Returned string (MUST be freed by the caller), NULL on failure */ LSB_UTIL_EXPORT char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, - size_t *r_len); + size_t *r_len); #ifdef __cplusplus } From 3eb600ce55ba023490f811000fa2997a58716b0d Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 22 Jun 2016 13:29:00 -0700 Subject: [PATCH 167/235] Pull in the new lua_circular_buffer module (CONTAINS BREAKING API CHANGES) --- CMakeLists.txt | 4 +-- cmake/externals.cmake | 2 +- src/heka/test/lua/encode_message.lua | 2 +- src/heka/test/test_sandbox.c | 2 +- src/test/lua/circular_buffer_delta.lua | 50 +++++--------------------- src/test/lua/output.lua | 11 +++++- src/test/lua/serialize.lua | 6 ++-- src/test/output/serialize.lua51.data | 4 ++- src/test/test_luasandbox.c | 39 +++++++++----------- 9 files changed, 46 insertions(+), 74 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de753a7..dbcf959 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,8 @@ cmake_minimum_required(VERSION 3.0 FATAL_ERROR) project(luasandbox C CXX) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 22) -set(CPACK_PACKAGE_VERSION_PATCH 4) +set(CPACK_PACKAGE_VERSION_MINOR 23) +set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") include(GNUInstallDirs) diff --git a/cmake/externals.cmake b/cmake/externals.cmake index 089dec2..bb2a758 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -164,7 +164,7 @@ add_dependencies(lua_bloom_filter luasandbox) externalproject_add( lua_circular_buffer GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG 85fc04e61e650ed65a7ba12ea02f8d307a5f1eab + GIT_TAG af8971042c6ea9733dce5bd2c3363191adb03155 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua index bb22422..990e37c 100644 --- a/src/heka/test/lua/encode_message.lua +++ b/src/heka/test/lua/encode_message.lua @@ -74,7 +74,7 @@ local msgs = { }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = cb}}, -- userdata - rv = '\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\141\1\10\3\107\101\121\16\1\42\131\1\123\34\116\105\109\101\34\58\48\44\34\114\111\119\115\34\58\50\44\34\99\111\108\117\109\110\115\34\58\49\44\34\115\101\99\111\110\100\115\95\112\101\114\95\114\111\119\34\58\49\44\34\99\111\108\117\109\110\95\105\110\102\111\34\58\91\123\34\110\97\109\101\34\58\34\67\111\108\117\109\110\95\49\34\44\34\117\110\105\116\34\58\34\99\111\117\110\116\34\44\34\97\103\103\114\101\103\97\116\105\111\110\34\58\34\115\117\109\34\125\93\125\10\110\97\110\10\110\97\110\10' + rv = '\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\158\1\10\3\107\101\121\16\1\42\148\1\123\34\116\105\109\101\34\58\48\44\34\114\111\119\115\34\58\50\44\34\99\111\108\117\109\110\115\34\58\49\44\34\115\101\99\111\110\100\115\95\112\101\114\95\114\111\119\34\58\49\44\34\99\111\108\117\109\110\95\105\110\102\111\34\58\91\123\34\110\97\109\101\34\58\34\67\111\108\117\109\110\95\49\34\44\34\117\110\105\116\34\58\34\99\111\117\110\116\34\44\34\97\103\103\114\101\103\97\116\105\111\110\34\58\34\115\117\109\34\125\93\44\34\97\110\110\111\116\97\116\105\111\110\115\34\58\91\93\125\10\110\97\110\10\110\97\110\10' }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {json = doc:make_field()}}, rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\23\10\4\106\115\111\110\16\1\42\13\123\34\102\111\111\34\58\34\98\97\114\34\125" diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_sandbox.c index 1d8f7fd..82ce15a 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_sandbox.c @@ -567,7 +567,7 @@ static char* test_encode_message() &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(172 == stats.out_max, "received %llu", stats.out_max); + mu_assert(189 == stats.out_max, "received %llu", stats.out_max); e = lsb_heka_destroy_sandbox(hsb); return NULL; } diff --git a/src/test/lua/circular_buffer_delta.lua b/src/test/lua/circular_buffer_delta.lua index 512dc6e..c9c1433 100644 --- a/src/test/lua/circular_buffer_delta.lua +++ b/src/test/lua/circular_buffer_delta.lua @@ -4,12 +4,12 @@ require "circular_buffer" -data = circular_buffer.new(3, 3, 1, true) +data = circular_buffer.new(3, 3, 1) local ADD_COL = data:set_header(1, "Add column") local SET_COL = data:set_header(2, "Set column", "count") local GET_COL = data:set_header(3, "Get column", "count", "sum") -local cb = circular_buffer.new(2, 2, 1, true) +local cb = circular_buffer.new(2, 2, 1) local SUM_COL = cb:set_header(1, "Sum column") local MIN_COL = cb:set_header(2, "Min", "count", "min") @@ -27,12 +27,6 @@ function report(tc) elseif tc == 1 then write_output(data:format("cbufd")) elseif tc == 2 then - local ts = 2e9 - if data:add(ts, ADD_COL, 0/0) then - data:set(ts, GET_COL, data:get(ts, ADD_COL)) - end - write_output(data:format("cbufd")) - elseif tc == 3 then -- the sum delta should reflect the difference -- the min delta should reflect the current value cb:set(0, SUM_COL, 3) @@ -41,46 +35,20 @@ function report(tc) cb:set(0, SUM_COL, 5) cb:set(0, MIN_COL, 5) write_output(cb:format("cbufd")) - elseif tc == 4 then - cb:add(0, SUM_COL, 3) - cb:add(0, MIN_COL, 3) - write_output(cb:format("cbufd")) - elseif tc == 5 then - cb:add(0, SUM_COL, 1) - cb:set(0, SUM_COL, 0/0) - cb:add(0, MIN_COL, 1) - write_output(cb:format("cbufd")) - elseif tc == 6 then - cb:add(0, SUM_COL, 1) - cb:add(0, SUM_COL, 0/0) - cb:add(0, MIN_COL, 1) - write_output(cb:format("cbufd")) - elseif tc == 7 then - cb:add(0, SUM_COL, 1) - cb:set(0, MIN_COL, 0/0) - write_output(cb:format("cbufd")) - elseif tc == 8 then - cb:add(0, SUM_COL, 1) - cb:set(0, MIN_COL, 9) - cb:add(0, MIN_COL, 0/0) - write_output(cb:format("cbufd")) - elseif tc == 9 then - cb:set(0, SUM_COL, 0) - cb:set(0, MIN_COL, 0) - write_output(cb:format("cbufd")) - elseif tc == 10 then - cb:add(0, SUM_COL, 0) - cb:add(0, MIN_COL, 0) - write_output(cb:format("cbufd")) - elseif tc == 11 then + elseif tc == 3 then cb:fromstring("1 1 1 2 3 4 0 3 4") write_output(cb:format("cbufd")) - elseif tc == 12 then + elseif tc == 4 then data:set(0, ADD_COL, 1/0) data:set(0, ADD_COL, 1/0) data:set(0, SET_COL, -1/0) data:add(0, SET_COL, 1) data:set(0, GET_COL, data:get(0, ADD_COL)) write_output(data:format("cbufd")) + elseif tc == 5 then + cb:annotate(1e9, 1, "info", "delta anno") + write_output(cb:format("cbufd")) -- annotation delta only + elseif tc == 6 then + write_output(cb:format("cbufd")) -- no delta end end diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index a2b63e0..6d751c7 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -5,12 +5,21 @@ require "circular_buffer" local cbuf = circular_buffer.new(1440, 3, 60) +local test = circular_buffer.new(2, 2, 60) function process(tc) if tc == 0 then -- lua types write_output(1.2, " string ", nil, " ", true, " ", false) elseif tc == 1 then -- cbuf - write_output("annotation\n", cbuf) + write_output(cbuf) + elseif tc == 2 then + test:annotate(0, 1, "info", "annotation\"\t\b\r\n\240 end") + test:annotate(60e9, 2, "alert", "alert") + test:annotate(120e9, 2, "info", "out of range, should be ignored") + write_output(test) + elseif tc == 3 then + test:set(120e9, 1, 0/0) -- advance the buffer (pruning the annotation) + write_output(test) end return 0 end diff --git a/src/test/lua/serialize.lua b/src/test/lua/serialize.lua index 92921c0..0eac4df 100644 --- a/src/test/lua/serialize.lua +++ b/src/test/lua/serialize.lua @@ -41,8 +41,10 @@ cycleb["a"] = cyclea data = circular_buffer.new(3,3,1) -delta = circular_buffer.new(2,1,1, true) +delta = circular_buffer.new(2,1,1) delta:add(0, 1, 2) +delta:annotate(0, 1, "info", "annotation\"\t\b\r\n\240 end") +delta:annotate(1e9, 1, "alert", "boom") dataRef = data @@ -53,5 +55,3 @@ end function timer_event(ns) end - - diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index 561570a..1535688 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -5,8 +5,10 @@ _G["dataRef"]:set_header(1, "Column_1", "count", "sum") _G["dataRef"]:set_header(2, "Column_2", "count", "sum") _G["dataRef"]:set_header(3, "Column_3", "count", "sum") _G["dataRef"]:fromstring("2 2 nan nan nan nan nan nan nan nan nan") -if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end +if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1) end _G["delta"]:set_header(1, "Column_1", "count", "sum") +_G["delta"]:annotate(1e+09, 1, "alert", "boom", true) +_G["delta"]:annotate(0, 1, "info", "annotation\"\t\b\r\n end", true) _G["delta"]:fromstring("1 1 2 nan 0 2") _G["data"] = _G["dataRef"] _G["rate"] = 0.12345678 diff --git a/src/test/test_luasandbox.c b/src/test/test_luasandbox.c index 97d22db..47eb4e8 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_luasandbox.c @@ -45,6 +45,7 @@ static const char *test_cfg = "os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'}\n" "}\n" "disable_modules = {io = 1, coroutine = 1}\n" + "log_level = 7\n" MODULE_PATH ; @@ -485,14 +486,16 @@ static char* test_output() { const char *outputs[] = { "1.2 string nil true false" - , "" + , "" // this cbuf output is not verified, it is used for benchmarking + , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":60,\"column_info\":[{\"name\":\"Column_1\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Column_2\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[{\"x\":0,\"col\":1,\"shortText\":\"i\",\"text\":\"annotation\\\"\\t\\b\\r\\n end\"},{\"x\":60000,\"col\":2,\"shortText\":\"a\",\"text\":\"alert\"}]}\nnan\tnan\nnan\tnan\n" + , "{\"time\":60,\"rows\":2,\"columns\":2,\"seconds_per_row\":60,\"column_info\":[{\"name\":\"Column_1\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Column_2\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[{\"x\":60000,\"col\":2,\"shortText\":\"a\",\"text\":\"alert\"}]}\nnan\tnan\nnan\tnan\n" , NULL }; lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, "circular_buffer.preserve"); + lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); lsb_add_function(sb, &write_output, "write_output"); @@ -601,10 +604,10 @@ static char* test_cbuf() { const char *output_file = "circular_buffer.preserve"; const char *outputs[] = { - "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\tnan\tnan\nnan\tnan\tnan\nnan\tnan\tnan\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" - , "{\"time\":2,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n3\t1\t3\nnan\tnan\tnan\n1\t1\t1\n" - , "{\"time\":8,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\tnan\tnan\nnan\tnan\tnan\n1\t1\t1\n" + "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\nnan\tnan\tnan\nnan\tnan\tnan\nnan\tnan\tnan\n" + , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" + , "{\"time\":2,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n3\t1\t3\nnan\tnan\tnan\n1\t1\t1\n" + , "{\"time\":8,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\nnan\tnan\tnan\nnan\tnan\tnan\n1\t1\t1\n" , NULL }; @@ -657,25 +660,15 @@ static char* test_cbuf_delta() { const char *output_file = "circular_buffer_delta.preserve"; const char *outputs[] = { - "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" -#ifdef LUA_JIT - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n0\t1\t1\t1\n1\t2\t1\t2\n2\t3\t1\t3\n" -#else - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t2\t1\t2\n2\t3\t1\t3\n0\t1\t1\t1\n" -#endif - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" + "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" + , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n0\t1\t1\t1\n1\t2\t1\t2\n2\t3\t1\t3\n" + , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" , "" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n2\tnan\tnan\tnan\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t2\t5\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t3\t8\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\tnan\t9\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\tnan\t10\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t1\tnan\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t1\tnan\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t-2\t0\n" + , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}],\"annotations\":[]}\n0\t2\t5\n" + , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}],\"annotations\":[]}\n0\t3\t4\n" + , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n0\tinf\t-inf\tinf\n" + , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}],\"annotations\":[{\"x\":1000,\"col\":1,\"shortText\":\"i\",\"text\":\"delta anno\"}]}\n" , "" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t3\t4\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n0\tinf\t-inf\tinf\n" , NULL }; From 736387a375f0234a337170981ce2e0f3a2bd157e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 1 Jul 2016 11:12:32 -0700 Subject: [PATCH 168/235] Update for the 1.0 release Restructure the Lua C module build/test - remove the built-in modules (expose the needed data through the API) - move the sandbox extension tests into the module where possible - create a test library to avoid duplication of test scaffolding --- CMakeLists.txt | 47 +- cmake/CMakeLists.txt.cjson | 45 + cmake/CMakeLists.txt.lpeg | 13 +- cmake/CMakeLists.txt.lua_geoip | 21 +- cmake/CMakeLists.txt.lua_openssl | 21 +- cmake/CMakeLists.txt.lua_postgres | 17 +- cmake/CMakeLists.txt.lua_sec | 21 +- cmake/CMakeLists.txt.lua_snappy | 15 +- cmake/CMakeLists.txt.sax | 35 + cmake/CMakeLists.txt.struct | 30 + cmake/FindLua.cmake | 129 +-- cmake/externals.cmake | 242 ++--- cmake/luasandboxConfig.cmake.in | 2 +- cmake/mozsvc.cmake | 5 +- docs/heka/analysis.md | 2 - docs/heka/index.md | 5 +- docs/heka/input.md | 64 +- docs/heka/json.md | 208 ----- docs/heka/kafka_consumer.md | 49 - docs/heka/kafka_producer.md | 118 --- docs/heka/output.md | 32 +- docs/heka/stream_reader.md | 63 -- docs/lsb_compression.md | 24 - docs/lsb_hash.md | 23 - gen_gh_pages.lua | 20 +- include/luasandbox.h | 1 + include/luasandbox/heka/sandbox.h | 21 + .../luasandbox/heka/stream_reader.h | 23 +- {src => include/luasandbox}/test/mu_test.h | 4 +- include/luasandbox/test/sandbox.h | 51 ++ include/luasandbox/util/heka_message.h | 2 +- include/luasandbox/util/util.h | 24 - sandboxes/heka/input/heka_kafka.lua | 4 +- sandboxes/heka/input/heka_stdin.lua | 3 +- sandboxes/heka/input/heka_tcp.lua | 4 +- .../heka/output/elasticsearch_bulk_api.lua | 4 +- sandboxes/heka/output/heka_kafka.lua | 4 +- src/CMakeLists.txt | 7 +- src/heka/CMakeLists.txt | 9 - src/heka/kafka.c | 753 ---------------- src/heka/kafka_impl.h | 20 - src/heka/message.c | 8 +- src/heka/rapidjson.cpp | 721 --------------- src/heka/rapidjson_impl.h | 36 - src/heka/sandbox.c | 148 ++- src/heka/stream_reader.c | 115 +-- src/heka/test/CMakeLists.txt | 10 +- src/heka/test/lua/encode_message.lua | 8 +- src/heka/test/lua/heka_json.lua | 312 ------- src/heka/test/lua/iim.lua | 18 +- src/heka/test/lua/input.lua | 1 + src/heka/test/lua/kafka_consumer.lua | 79 -- src/heka/test/lua/kafka_producer.lua | 75 -- src/heka/test/lua/output.lua | 1 + src/heka/test/test.h.in | 7 +- .../{test_sandbox.c => test_heka_sandbox.c} | 21 +- src/heka/test/test_kafka.c | 115 --- src/luasandbox.c | 31 +- src/luasandbox_defines.h | 6 + src/luasandbox_impl.h | 21 - src/luasandbox_modules.c | 94 -- src/test/CMakeLists.txt | 20 +- src/test/lua/bloom_filter.lua | 26 - src/test/lua/bloom_filter_benchmark.lua | 17 - src/test/lua/circular_buffer.lua | 25 - src/test/lua/circular_buffer_add.lua | 12 - src/test/lua/circular_buffer_delta.lua | 54 -- src/test/lua/circular_buffer_errors.lua | 12 - src/test/lua/cuckoo_filter.lua | 30 - src/test/lua/cuckoo_filter_benchmark.lua | 17 - src/test/lua/hyperloglog.lua | 21 - src/test/lua/lsb_compression.lua | 29 - src/test/lua/lsb_hash.lua | 25 - src/test/lua/output.lua | 19 +- src/test/lua/output_errors.lua | 2 +- src/test/output/serialize.data | 72 -- src/test/output/serialize.lua51.data | 2 +- src/test/sandbox.c | 142 +++ ...st_luasandbox.c => test_generic_sandbox.c} | 848 ++---------------- src/util/CMakeLists.txt | 4 - src/util/heka_message.c | 2 +- src/util/test/test_heka_message.c | 2 +- src/util/test/test_heka_message_matcher.c | 2 +- src/util/test/test_input_buffer.c | 4 +- src/util/test/test_output_buffer.c | 2 +- src/util/test/test_protobuf.c | 3 +- src/util/test/test_running_stats.c | 2 +- src/util/test/test_string_matcher.c | 2 +- src/util/test/test_util.c | 67 +- src/util/util.c | 76 -- 90 files changed, 898 insertions(+), 4653 deletions(-) create mode 100644 cmake/CMakeLists.txt.cjson create mode 100644 cmake/CMakeLists.txt.sax create mode 100644 cmake/CMakeLists.txt.struct delete mode 100644 docs/heka/json.md delete mode 100644 docs/heka/kafka_consumer.md delete mode 100644 docs/heka/kafka_producer.md delete mode 100644 docs/heka/stream_reader.md delete mode 100644 docs/lsb_compression.md delete mode 100644 docs/lsb_hash.md rename src/heka/stream_reader_impl.h => include/luasandbox/heka/stream_reader.h (55%) rename {src => include/luasandbox}/test/mu_test.h (98%) create mode 100644 include/luasandbox/test/sandbox.h delete mode 100644 src/heka/kafka.c delete mode 100644 src/heka/kafka_impl.h delete mode 100644 src/heka/rapidjson.cpp delete mode 100644 src/heka/rapidjson_impl.h delete mode 100644 src/heka/test/lua/heka_json.lua delete mode 100644 src/heka/test/lua/kafka_consumer.lua delete mode 100644 src/heka/test/lua/kafka_producer.lua rename src/heka/test/{test_sandbox.c => test_heka_sandbox.c} (97%) delete mode 100644 src/heka/test/test_kafka.c delete mode 100644 src/luasandbox_modules.c delete mode 100644 src/test/lua/bloom_filter.lua delete mode 100644 src/test/lua/bloom_filter_benchmark.lua delete mode 100644 src/test/lua/circular_buffer.lua delete mode 100644 src/test/lua/circular_buffer_add.lua delete mode 100644 src/test/lua/circular_buffer_delta.lua delete mode 100644 src/test/lua/circular_buffer_errors.lua delete mode 100644 src/test/lua/cuckoo_filter.lua delete mode 100644 src/test/lua/cuckoo_filter_benchmark.lua delete mode 100644 src/test/lua/hyperloglog.lua delete mode 100644 src/test/lua/lsb_compression.lua delete mode 100644 src/test/lua/lsb_hash.lua delete mode 100644 src/test/output/serialize.data create mode 100644 src/test/sandbox.c rename src/test/{test_luasandbox.c => test_generic_sandbox.c} (50%) diff --git a/CMakeLists.txt b/CMakeLists.txt index dbcf959..7418805 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,10 +3,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox C CXX) +project(luasandbox C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 23) +set(CPACK_PACKAGE_VERSION_MINOR 24) set(CPACK_PACKAGE_VERSION_PATCH 0) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") @@ -35,41 +35,8 @@ if(LUA_JIT) add_definitions(-DLUA_JIT) endif() -option(LUA_SNAPPY "Include the Lua Snappy module (only used to support a deprecated Mozilla telemetry data format)" off) - find_library(LIBM_LIBRARY m) -find_package(ZLIB) -if (ZLIB_FOUND) - add_definitions(-DHAVE_ZLIB) -endif() - -find_library(LIBRTKAFKA_LIBRARY rdkafka) -if (LIBRTKAFKA_LIBRARY) - add_definitions(-DHAVE_KAFKA) -else() - message(STATUS "librdkafka was not found, dependent modules will not be built") -endif() - -find_package(PostgreSQL) -if (NOT PostgreSQL_FOUND) - message(STATUS "PostgreSQL was not found, dependent modules will not be built") -endif() - -if (APPLE) - set(OPENSSL_ROOT_DIR /opt/local) -endif() - -find_package(OpenSSL) -if (NOT OPENSSL_FOUND) - message(STATUS "OpenSSL was not found, dependent modules will not be built") -endif() - -find_library(GEOIP_LIBRARY GeoIP) -if (NOT GEOIP_LIBRARY) - message(STATUS "GeoIP was not found, dependent modules will not be built") -endif() - find_package(Git REQUIRED) set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") @@ -84,7 +51,7 @@ if(HAVE_CLOCK_GETTIME) add_definitions(-DHAVE_CLOCK_GETTIME) endif() -include_directories("${EP_BASE}/include") +include_directories("${EP_BASE}/inst/include") install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/modules COMPONENT core) install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT core) install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandboxes/" DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/sandboxes COMPONENT core) @@ -101,5 +68,9 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake ${CMAKE_CURRENT add_subdirectory(src) add_custom_target(copy_includes -COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/include) -add_dependencies(luasandboxutil copy_includes) +COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/inst/include) +add_dependencies(${LUA_PROJECT} copy_includes) +add_dependencies(luasandboxutil ${LUA_PROJECT}) +add_dependencies(luasandbox luasandboxutil) +add_dependencies(luasandboxtest luasandbox) +add_dependencies(luasandboxheka luasandboxtest) diff --git a/cmake/CMakeLists.txt.cjson b/cmake/CMakeLists.txt.cjson new file mode 100644 index 0000000..a0909ad --- /dev/null +++ b/cmake/CMakeLists.txt.cjson @@ -0,0 +1,45 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(cjson C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-Wall -fPIC") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) +set(FPCONV_SOURCES fpconv.c) + +# Handle platforms missing isinf() macro (Eg, some Solaris systems). +include(CheckSymbolExists) +CHECK_SYMBOL_EXISTS(isinf math.h HAVE_ISINF) +if(NOT HAVE_ISINF) + add_definitions(-DUSE_INTERNAL_ISINF) +endif() + +if(WIN32) + # Windows sprintf()/strtod() handle NaN/inf differently. Not supported. + add_definitions(-DDISABLE_INVALID_NUMBERS) +endif() + +add_definitions(-DLUA_SANDBOX -DDIST_VERSION="2.1.0") +add_library(cjson SHARED lua_cjson.c strbuf.c fpconv.c lua_cjson.def) +target_link_libraries(cjson ${LUASANDBOX_LUASB_LIBRARY}) +if(LIBM_LIBRARY) + target_link_libraries(cjson ${LIBM_LIBRARY}) +endif() +install(TARGETS cjson DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lpeg b/cmake/CMakeLists.txt.lpeg index 5c39d13..2caf5c2 100644 --- a/cmake/CMakeLists.txt.lpeg +++ b/cmake/CMakeLists.txt.lpeg @@ -14,12 +14,15 @@ else() set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") endif() -include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox) -find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) - -set(INSTALL_PATH lib/lua) +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) set(CMAKE_SHARED_LIBRARY_PREFIX "") +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) + file(WRITE lpeg.def "EXPORTS\nluaopen_lpeg\n") set(LPEG_SRC lpcap.c @@ -31,6 +34,6 @@ lpeg.def ) add_library(lpeg SHARED ${LPEG_SRC}) -target_link_libraries(lpeg ${LUA_LIBRARY}) +target_link_libraries(lpeg ${LUASANDBOX_LUASB_LIBRARY}) install(TARGETS lpeg DESTINATION ${INSTALL_PATH}) install(FILES re.lua DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lua_geoip b/cmake/CMakeLists.txt.lua_geoip index 1a3f089..8f5ab27 100644 --- a/cmake/CMakeLists.txt.lua_geoip +++ b/cmake/CMakeLists.txt.lua_geoip @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luageoip C) +project(geoip C) if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") @@ -14,12 +14,17 @@ else() set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") endif() -include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox) -find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) - -set(INSTALL_PATH lib/lua) +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) set(CMAKE_SHARED_LIBRARY_PREFIX "") +find_library(GEOIP_LIBRARY GeoIP REQUIRED) + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) + file(WRITE geoip.def "EXPORTS\nluaopen_geoip") set(GEOIP_SRC src/lua-geoip.c @@ -42,13 +47,13 @@ country.def ) add_library(geoip SHARED ${GEOIP_SRC}) -target_link_libraries(geoip ${LUA_LIBRARY} ${GEOIP_LIBRARY}) +target_link_libraries(geoip ${LUASANDBOX_LUASB_LIBRARY} ${GEOIP_LIBRARY}) install(TARGETS geoip DESTINATION ${INSTALL_PATH}) add_library(city SHARED ${CITY_SRC}) -target_link_libraries(city ${LUA_LIBRARY} ${GEOIP_LIBRARY}) +target_link_libraries(city ${LUASANDBOX_LUASB_LIBRARY} ${GEOIP_LIBRARY}) install(TARGETS city DESTINATION ${INSTALL_PATH}/geoip) add_library(country SHARED ${COUNTRY_SRC}) -target_link_libraries(country ${LUA_LIBRARY} ${GEOIP_LIBRARY}) +target_link_libraries(country ${LUASANDBOX_LUASB_LIBRARY} ${GEOIP_LIBRARY}) install(TARGETS country DESTINATION ${INSTALL_PATH}/geoip) diff --git a/cmake/CMakeLists.txt.lua_openssl b/cmake/CMakeLists.txt.lua_openssl index eae31a6..ca7f74c 100644 --- a/cmake/CMakeLists.txt.lua_openssl +++ b/cmake/CMakeLists.txt.lua_openssl @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luaopenssl C) +project(openssl C) if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") @@ -14,12 +14,21 @@ else() set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") endif() -include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/deps) -find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) - -set(INSTALL_PATH lib/lua) +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) set(CMAKE_SHARED_LIBRARY_PREFIX "") +find_package(OpenSSL REQUIRED) + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/deps) + +if (APPLE) + set(OPENSSL_ROOT_DIR /opt/local) +endif() + file(WRITE luaopenssl.def "EXPORTS\nluaopen_openssl\n") set(LUAOPENSSL_SRC src/asn1.c @@ -60,5 +69,5 @@ luaopenssl.def ) add_library(openssl SHARED ${LUAOPENSSL_SRC}) -target_link_libraries(openssl ${LUA_LIBRARY} ${OPENSSL_LIBRARIES}) +target_link_libraries(openssl ${LUASANDBOX_LUASB_LIBRARY} ${OPENSSL_LIBRARIES}) install(TARGETS openssl DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lua_postgres b/cmake/CMakeLists.txt.lua_postgres index 5e6bb13..ba8a6ba 100644 --- a/cmake/CMakeLists.txt.lua_postgres +++ b/cmake/CMakeLists.txt.lua_postgres @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luapostgres C) +project(postgres C) if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") @@ -14,12 +14,17 @@ else() set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") endif() -include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox ${PostgreSQL_INCLUDE_DIRS}) -find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) - -set(INSTALL_PATH lib/lua) +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) set(CMAKE_SHARED_LIBRARY_PREFIX "") +find_package(PostgreSQL REQUIRED) + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox ${PostgreSQL_INCLUDE_DIRS}) + file(WRITE postgres.def "EXPORTS\nluaopen_postgres") set(POSTGRES_SRC src/ls_postgres.c @@ -28,5 +33,5 @@ postgres.def ) add_library(postgres SHARED ${POSTGRES_SRC}) -target_link_libraries(postgres ${LUA_LIBRARY} ${PostgreSQL_LIBRARIES}) +target_link_libraries(postgres ${LUASANDBOX_LUASB_LIBRARY} ${PostgreSQL_LIBRARIES}) install(TARGETS postgres DESTINATION ${INSTALL_PATH}/luasql) diff --git a/cmake/CMakeLists.txt.lua_sec b/cmake/CMakeLists.txt.lua_sec index dd2c12c..0f0c84f 100644 --- a/cmake/CMakeLists.txt.lua_sec +++ b/cmake/CMakeLists.txt.lua_sec @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasec C) +project(ssl C) if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") @@ -13,12 +13,21 @@ else() set(CMAKE_C_FLAGS_RELEASE "-O2") endif() -include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/src) -find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) - -set(INSTALL_PATH lib/lua) +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) set(CMAKE_SHARED_LIBRARY_PREFIX "") +find_package(OpenSSL REQUIRED) + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/src) + +if (APPLE) + set(OPENSSL_ROOT_DIR /opt/local) +endif() + set(LUASOCKET_SRC src/luasocket/buffer.c src/luasocket/io.c @@ -37,7 +46,7 @@ luasec.def ) add_library(ssl SHARED ${LUASEC_SRC}) -target_link_libraries(ssl luasocket ${LUA_LIBRARY} ${OPENSSL_LIBRARIES}) +target_link_libraries(ssl luasocket ${LUASANDBOX_LUASB_LIBRARY} ${OPENSSL_LIBRARIES}) install(TARGETS ssl DESTINATION ${INSTALL_PATH}) install(FILES src/ssl.lua DESTINATION ${INSTALL_PATH}) install(FILES src/https.lua DESTINATION ${INSTALL_PATH}/ssl) diff --git a/cmake/CMakeLists.txt.lua_snappy b/cmake/CMakeLists.txt.lua_snappy index 2534503..6cd2414 100644 --- a/cmake/CMakeLists.txt.lua_snappy +++ b/cmake/CMakeLists.txt.lua_snappy @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasnappy C CXX) +project(snappy C CXX) if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") @@ -14,12 +14,15 @@ else() set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") endif() -include_directories(${LUA_SANDBOX_INCLUDE}/luasandbox) -find_library(LUA_LIBRARY luasb PATHS ${EP_BASE}/lib NO_DEFAULT_PATH) - -set(INSTALL_PATH lib/lua) +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) set(CMAKE_SHARED_LIBRARY_PREFIX "") +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) + file(WRITE snappy.def "EXPORTS\nluaopen_snappy") set(SNAPPY_SRC lua-snappy.cc @@ -31,5 +34,5 @@ snappy.def ) add_library(snappy SHARED ${SNAPPY_SRC}) -target_link_libraries(snappy ${LUA_LIBRARY}) +target_link_libraries(snappy ${LUASANDBOX_LUASB_LIBRARY}) install(TARGETS snappy DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.sax b/cmake/CMakeLists.txt.sax new file mode 100644 index 0000000..6839ce6 --- /dev/null +++ b/cmake/CMakeLists.txt.sax @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(sax C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3 /WX") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) +include_directories(${CMAKE_SOURCE_DIR}/include) + +find_library(LIBM_LIBRARY m) + +add_definitions(-DLUA_SANDBOX -DDIST_VERSION="0.5.0") +add_library(sax SHARED src/symtseries.c lua/lua_sax.c lua/lua_sax.def) +target_link_libraries(sax ${LUASANDBOX_LUASB_LIBRARY}) +if(LIBM_LIBRARY) + target_link_libraries(sax ${LIBM_LIBRARY}) +endif() +install(TARGETS sax DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.struct b/cmake/CMakeLists.txt.struct new file mode 100644 index 0000000..6900d18 --- /dev/null +++ b/cmake/CMakeLists.txt.struct @@ -0,0 +1,30 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) +project(struct C) + +if(MSVC) + set(CMAKE_C_FLAGS "/W3") + set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") +else() + set(CMAKE_C_FLAGS "-D_POSIX_SOURCE -fPIC") + set(CMAKE_C_FLAGS_RELEASE "-O2") + set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") +endif() + +include(GNUInstallDirs) +if(WIN32) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +endif() +set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) +set(CMAKE_SHARED_LIBRARY_PREFIX "") + +include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) + +file(WRITE struct.def "EXPORTS\nluaopen_struct") + +add_library(struct SHARED struct.c struct.def) +target_link_libraries(struct ${LUASANDBOX_LUASB_LIBRARY}) +install(TARGETS struct DESTINATION ${INSTALL_PATH}) diff --git a/cmake/FindLua.cmake b/cmake/FindLua.cmake index 2043318..72a77cb 100644 --- a/cmake/FindLua.cmake +++ b/cmake/FindLua.cmake @@ -1,125 +1,4 @@ -# Locate Lua library -# This module defines -# LUA_EXECUTABLE, if found -# LUA_FOUND, if false, do not try to link to Lua -# LUA_LIBRARIES -# LUA_INCLUDE_DIR, where to find lua.h -# LUA_VERSION_STRING, the version of Lua found (since CMake 2.8.8) -# -# Note that the expected include convention is -# #include "lua.h" -# and not -# #include -# This is because, the lua location is not standardized and may exist -# in locations other than lua/ - -#============================================================================= -# Copyright 2007-2009 Kitware, Inc. -# Modified to support Lua 5.2 by LuaDist 2012 -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) -# -# The required version of Lua can be specified using the -# standard syntax, e.g. FIND_PACKAGE(Lua 5.1) -# Otherwise the module will search for any available Lua implementation - - -if (LUA_SANDBOX_INCLUDE) - set(LUA_INCLUDE_DIR ${EP_BASE}/include/luasandbox) - set(LIB_PATH ${EP_BASE}/lib) - find_library(LUA_LIBRARY luasb PATHS ${LIB_PATH} NO_DEFAULT_PATH) -else() - # Always search for non-versioned lua first (recommended) - SET(_POSSIBLE_LUA_INCLUDE include include/lua) - SET(_POSSIBLE_LUA_EXECUTABLE lua) - SET(_POSSIBLE_LUA_LIBRARY lua) - - # Determine possible naming suffixes (there is no standard for this) - IF(Lua_FIND_VERSION_MAJOR AND Lua_FIND_VERSION_MINOR) - SET(_POSSIBLE_SUFFIXES "${Lua_FIND_VERSION_MAJOR}${Lua_FIND_VERSION_MINOR}" "${Lua_FIND_VERSION_MAJOR}.${Lua_FIND_VERSION_MINOR}" "-${Lua_FIND_VERSION_MAJOR}.${Lua_FIND_VERSION_MINOR}") - ELSE(Lua_FIND_VERSION_MAJOR AND Lua_FIND_VERSION_MINOR) - SET(_POSSIBLE_SUFFIXES "52" "5.2" "-5.2" "51" "5.1" "-5.1") - ENDIF(Lua_FIND_VERSION_MAJOR AND Lua_FIND_VERSION_MINOR) - - # Set up possible search names and locations - FOREACH(_SUFFIX ${_POSSIBLE_SUFFIXES}) - LIST(APPEND _POSSIBLE_LUA_INCLUDE "include/lua${_SUFFIX}") - LIST(APPEND _POSSIBLE_LUA_EXECUTABLE "lua${_SUFFIX}") - LIST(APPEND _POSSIBLE_LUA_LIBRARY "lua${_SUFFIX}") - ENDFOREACH(_SUFFIX) - - # Find the lua executable - FIND_PROGRAM(LUA_EXECUTABLE - NAMES ${_POSSIBLE_LUA_EXECUTABLE} - ) - - # Find the lua header - FIND_PATH(LUA_INCLUDE_DIR lua.h - HINTS - $ENV{LUA_DIR} - PATH_SUFFIXES ${_POSSIBLE_LUA_INCLUDE} - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /sw # Fink - /opt/local # DarwinPorts - /opt/csw # Blastwave - /opt - ) - - # Find the lua library - FIND_LIBRARY(LUA_LIBRARY - NAMES ${_POSSIBLE_LUA_LIBRARY} - HINTS - $ENV{LUA_DIR} - PATH_SUFFIXES lib64 lib - PATHS - ~/Library/Frameworks - /Library/Frameworks - /usr/local - /usr - /sw - /opt/local - /opt/csw - /opt - ) -endif() - -IF(LUA_LIBRARY) - # include the math library for Unix - IF(UNIX AND NOT APPLE) - FIND_LIBRARY(LUA_MATH_LIBRARY m) - SET( LUA_LIBRARIES "${LUA_LIBRARY};${LUA_MATH_LIBRARY}" CACHE STRING "Lua Libraries") - # For Windows and Mac, don't need to explicitly include the math library - ELSE(UNIX AND NOT APPLE) - SET( LUA_LIBRARIES "${LUA_LIBRARY}" CACHE STRING "Lua Libraries") - ENDIF(UNIX AND NOT APPLE) -ENDIF(LUA_LIBRARY) - -# Determine Lua version -IF(LUA_INCLUDE_DIR AND EXISTS "${LUA_INCLUDE_DIR}/lua.h") - FILE(STRINGS "${LUA_INCLUDE_DIR}/lua.h" lua_version_str REGEX "^#define[ \t]+LUA_RELEASE[ \t]+\"Lua .+\"") - - STRING(REGEX REPLACE "^#define[ \t]+LUA_RELEASE[ \t]+\"Lua ([^\"]+)\".*" "\\1" LUA_VERSION_STRING "${lua_version_str}") - UNSET(lua_version_str) -ENDIF() - -INCLUDE(FindPackageHandleStandardArgs) -# handle the QUIETLY and REQUIRED arguments and set LUA_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(Lua - REQUIRED_VARS LUA_LIBRARIES LUA_INCLUDE_DIR - VERSION_VAR LUA_VERSION_STRING) - -MARK_AS_ADVANCED(LUA_INCLUDE_DIR LUA_LIBRARIES LUA_LIBRARY LUA_MATH_LIBRARY LUA_EXECUTABLE) - +set(LUA_FOUND TRUE) +set(LUA_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY} ${LUASANDBOX_LUASB_LIBRARY}) +set(LUA_INCLUDE_DIR ${LUASANDBOX_INCLUDE_DIR}/luasandbox) +set(LUA_VERSION_STRING "5.1.5") diff --git a/cmake/externals.cmake b/cmake/externals.cmake index bb2a758..5b98a0b 100644 --- a/cmake/externals.cmake +++ b/cmake/externals.cmake @@ -4,26 +4,35 @@ include(ExternalProject) string(REPLACE ")$" "|INSTALL_ARGS)$" _ep_keywords_ExternalProject_Add ${_ep_keywords_ExternalProject_Add}) +set(EP_BASE ${CMAKE_BINARY_DIR}/ep_base) get_filename_component(GIT_PATH ${GIT_EXECUTABLE} PATH) -set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") +set(LUASANDBOX_LUASB_LIBRARY ${EP_BASE}/inst/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(LUASANDBOX_UTIL_LIBRARY ${EP_BASE}/../src/util/${CMAKE_SHARED_LIBRARY_PREFIX}luasandboxutil${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(LUASANDBOX_LIBRARY ${EP_BASE}/../src/${CMAKE_SHARED_LIBRARY_PREFIX}luasandbox${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(LUASANDBOX_TEST_LIBRARY ${EP_BASE}/../src/test/${CMAKE_SHARED_LIBRARY_PREFIX}luasandboxtest${CMAKE_SHARED_LIBRARY_SUFFIX}) +set(LUASANDBOX_HEKA_LIBRARY ${EP_BASE}/../src/heka/${CMAKE_SHARED_LIBRARY_PREFIX}luasandboxheka${CMAKE_SHARED_LIBRARY_SUFFIX}) + set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} --DCMAKE_INSTALL_PREFIX=${EP_BASE} --DEP_BASE=${EP_BASE} --DLUA_SANDBOX_INCLUDE=${CMAKE_SOURCE_DIR}/include --DUSE_RPATH=false +-DCMAKE_INSTALL_PREFIX=${EP_BASE}/inst +-DLUASANDBOX_INCLUDE_DIR=${EP_BASE}/inst/include +-DLUASANDBOX_LUASB_LIBRARY=${LUASANDBOX_LUASB_LIBRARY} +-DLUASANDBOX_UTIL_LIBRARY=${LUASANDBOX_UTIL_LIBRARY} +-DLUASANDBOX_LIBRARY=${LUASANDBOX_LIBRARY} +-DLUASANDBOX_TEST_LIBRARY=${LUASANDBOX_TEST_LIBRARY} +-DLUASANDBOX_HEKA_LIBRARY=${LUASANDBOX_HEKA_LIBRARY} +-DUSE_RPATH=false # todo remove after the LUA_SOCKET build is updated to https://github.com/diegonehab/luasocket --no-warn-unused-cli) -set(LUA_INCLUDE_DIR "${EP_BASE}/include/${PROJECT_NAME}") set(INST_ARGS "install/strip") if(MSVC) set(INST_ARGS "install") endif() -if (LUA_JIT) +if(LUA_JIT) message(FATAL_ERROR, "LuaJIT support has not been added back in yet, issue #66") else() set(LUA_PROJECT "lua-5_1_5") @@ -35,7 +44,7 @@ else() INSTALL_ARGS ${INST_ARGS} ) add_library(luasb SHARED IMPORTED) - set_target_properties(luasb PROPERTIES IMPORTED_LOCATION "${EP_BASE}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}") + set_target_properties(luasb PROPERTIES IMPORTED_LOCATION ${LUASANDBOX_LUASB_LIBRARY}) #install(TARGETS luasb DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core OPTIONAL) #http://public.kitware.com/Bug/view.php?id=14311&nbn=6 endif() @@ -48,161 +57,188 @@ externalproject_add( CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} ) -add_dependencies(lua_lpeg ${LUA_PROJECT}) +add_dependencies(lua_lpeg luasandboxheka) + +## sandbox enhanced modules externalproject_add( - lua_cjson - GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG b24f724a4982531e392a88679b3783fdc5361c5d + lua_bloom_filter + GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git + GIT_TAG dbfd90e313be86fe1864fe0e18f6e9d5cb0e6f16 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} + TEST_AFTER_INSTALL 1 +) +add_dependencies(lua_bloom_filter luasandboxheka) + +externalproject_add( + lua_circular_buffer + GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git + GIT_TAG f86e296bb0fafbc8529d74336cc17ff18d8a9e75 CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} + TEST_AFTER_INSTALL 1 ) -add_dependencies(lua_cjson ${LUA_PROJECT}) +add_dependencies(lua_circular_buffer luasandboxheka) externalproject_add( - lua_struct - GIT_REPOSITORY https://github.com/trink/struct.git - GIT_TAG b7e9b87d1ee36a5e22c6749be0959b45858beaad - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + lua_hyperloglog + GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git + GIT_TAG eb1e27c00c206ceffaab6787b4f54c59d4451a5c CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} + TEST_AFTER_INSTALL 1 ) -add_dependencies(lua_struct ${LUA_PROJECT}) +add_dependencies(lua_hyperloglog luasandboxheka) externalproject_add( - lua_socket - GIT_REPOSITORY https://github.com/LuaDist/luasocket.git - GIT_TAG b97ed47e7a01e0d2523809efe11676333a85dcab - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io + lua_cuckoo_filter + GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git + GIT_TAG a5c031fe70008d0f3d347371a7d7ae68acaba575 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} INSTALL_ARGS ${INST_ARGS} + TEST_AFTER_INSTALL 1 ) -add_dependencies(lua_socket ${LUA_PROJECT}) +add_dependencies(lua_cuckoo_filter luasandboxheka) + +externalproject_add( + lua_sax + GIT_REPOSITORY https://github.com/trink/symtseries.git + GIT_TAG bd2426c93d0d55977ca7b21354105463b0470a49 + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.sax /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} +) +add_dependencies(lua_sax luasandboxheka) + +externalproject_add( + lua_cjson + GIT_REPOSITORY https://github.com/trink/lua-cjson.git + GIT_TAG b24f724a4982531e392a88679b3783fdc5361c5d + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.cjson /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} +) +add_dependencies(lua_cjson luasandboxheka) + +set(OPTIONAL_MODULES LUA_RJSON LUA_STRUCT LUA_KAFKA LUA_SOCKET LUA_SEC LUA_OPENSSL LUA_POSTGRES LUA_GEOIP LUA_SNAPPY) +foreach(module_name IN LISTS OPTIONAL_MODULES) + if(ALL_OPTIONAL_MODULES) + option(${module_name} "Include the ${module_name} module" on) + else() + option(${module_name} "Include the ${module_name} module" off) + endif() +endforeach() + +# Optionally Built modules + +if(LUA_RJSON) + externalproject_add( + lua_rjson + GIT_REPOSITORY https://github.com/mozilla-services/lua_rjson.git + GIT_TAG 664351d4b19011375cef41365000201b2728be20 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} + TEST_AFTER_INSTALL 1 + ) + add_dependencies(lua_rjson luasandboxheka) +endif() + +if(LUA_KAFKA) + externalproject_add( + lua_kafka + GIT_REPOSITORY https://github.com/mozilla-services/lua_kafka.git + GIT_TAG 27b63f1e3f56ff641661e983e70e79062511d091 + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} + # TEST_AFTER_INSTALL 1 # requires a running kafka installation + ) + add_dependencies(lua_kafka luasandboxheka) +endif() + +if(LUA_STRUCT) + externalproject_add( + lua_struct + URL http://www.inf.puc-rio.br/~roberto/struct/struct-0.2.tar.gz + URL_MD5 99384bf1f54457ec9f796ad0b539d19c + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.struct /CMakeLists.txt + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} + INSTALL_ARGS ${INST_ARGS} + ) + add_dependencies(lua_struct luasandboxheka) +endif() -if (OPENSSL_FOUND) +if(LUA_SOCKET) # todo move off the LuaDist version to https://github.com/diegonehab/luasocket + externalproject_add( + lua_socket + GIT_REPOSITORY https://github.com/LuaDist/luasocket.git + GIT_TAG b97ed47e7a01e0d2523809efe11676333a85dcab + UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake + CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/lua_socket + INSTALL_ARGS ${INST_ARGS} + ) + externalproject_add_step(lua_socket copy_install + COMMAND ${CMAKE_COMMAND} -E copy_directory ${EP_BASE}/lua_socket/lib/lua ${EP_BASE}/inst/lib/luasandbox/io_modules + DEPENDEES install) + add_dependencies(lua_socket luasandboxheka) +endif() + +if(LUA_SEC) externalproject_add( lua_sec GIT_REPOSITORY https://github.com/brunoos/luasec.git GIT_TAG 20443861ebc3f6498ee7d9c70fbdaa059bec15e1 UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_sec /CMakeLists.txt CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -DOPENSSL_LIBRARIES=${OPENSSL_LIBRARIES} INSTALL_ARGS ${INST_ARGS} ) - add_dependencies(lua_sec ${LUA_PROJECT}) + add_dependencies(lua_sec luasandboxheka) +endif() +if(LUA_OPENSSL) externalproject_add( lua_openssl GIT_REPOSITORY https://github.com/zhaozg/lua-openssl.git GIT_TAG 2e8930c5e13b52705ba9c390110e84b1f4748ba9 UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_openssl /CMakeLists.txt CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR} - -DOPENSSL_LIBRARIES=${OPENSSL_LIBRARIES} INSTALL_ARGS ${INST_ARGS} ) - add_dependencies(lua_openssl ${LUA_PROJECT}) + add_dependencies(lua_openssl luasandboxheka) endif() -if (PostgreSQL_FOUND) +if(LUA_POSTGRES) externalproject_add( lua_postgres GIT_REPOSITORY https://github.com/LuaDist/luasql-postgresql.git GIT_TAG 29a3aa1964aeac93323ec5d1446ac7d32ec700df UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_postgres /CMakeLists.txt CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - -DPostgreSQL_INCLUDE_DIRS=${PostgreSQL_INCLUDE_DIRS} - -DPostgreSQL_LIBRARIES=${PostgreSQL_LIBRARIES} INSTALL_ARGS ${INST_ARGS} ) - add_dependencies(lua_postgres ${LUA_PROJECT}) + add_dependencies(lua_postgres luasandboxheka) endif() -if (GEOIP_LIBRARY) +if(LUA_GEOIP) externalproject_add( lua_geoip GIT_REPOSITORY https://github.com/agladysh/lua-geoip.git GIT_TAG a07d261d8a2c7ff854fe6cd72cb8c2e16ec638ff UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_geoip /CMakeLists.txt CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io - -DGEOIP_LIBRARY=${GEOIP_LIBRARY} INSTALL_ARGS ${INST_ARGS} ) - add_dependencies(lua_geoip ${LUA_PROJECT}) + add_dependencies(lua_geoip luasandboxheka) endif() -if (LUA_SNAPPY) +if(LUA_SNAPPY) externalproject_add( lua_snappy GIT_REPOSITORY https://github.com/forhappy/lua-snappy.git GIT_TAG 6b4f3f6736857d5c72aa6ed0ec566af6e222278d UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_snappy /CMakeLists.txt CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - -DCMAKE_INSTALL_PREFIX=${EP_BASE}/io INSTALL_ARGS ${INST_ARGS} ) - add_dependencies(lua_snappy ${LUA_PROJECT}) + add_dependencies(lua_snappy luasandboxheka) endif() - - -# sandbox enhanced modules - -externalproject_add( - lua_bloom_filter - GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG 255af01bc127fccbd11fc5e57218a5fcc8659c69 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_bloom_filter luasandbox) - -externalproject_add( - lua_circular_buffer - GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG af8971042c6ea9733dce5bd2c3363191adb03155 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_circular_buffer luasandbox) - -externalproject_add( - lua_hyperloglog - GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG 792b11dd811d89d6f0ad973b04a742c02c4ef7b5 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_hyperloglog luasandbox) - -externalproject_add( - lua_cuckoo_filter - GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG bc7f406b3997fc9432bb5d1db96cad902bd3283a - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_cuckoo_filter luasandbox) - -externalproject_add( - lua_sax - GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG bd2426c93d0d55977ca7b21354105463b0470a49 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_sax luasandbox) - -externalproject_add( - rapidjson - GIT_REPOSITORY https://github.com/miloyip/rapidjson.git - GIT_TAG ed7efe6289cfa1fafc354aaa8a29fcd31c1607fd - CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo "no configure" - BUILD_COMMAND ${CMAKE_COMMAND} -E echo "no build" - INSTALL_COMMAND ${CMAKE_COMMAND} -E echo "no install" -) -include_directories(${EP_BASE}/Source/rapidjson/include) diff --git a/cmake/luasandboxConfig.cmake.in b/cmake/luasandboxConfig.cmake.in index 1b068e0..601ba86 100644 --- a/cmake/luasandboxConfig.cmake.in +++ b/cmake/luasandboxConfig.cmake.in @@ -9,6 +9,6 @@ FIND_LIBRARY(LUASANDBOX_LUASB_LIBRARY NAMES luasb PATHS ${LUASANDB FIND_LIBRARY(LUASANDBOX_UTIL_LIBRARY NAMES luasandboxutil PATHS ${LUASANDBOX_LIB_DIR}) FIND_LIBRARY(LUASANDBOX_LIBRARY NAMES luasandbox PATHS ${LUASANDBOX_LIB_DIR}) FIND_LIBRARY(LUASANDBOX_HEKA_LIBRARY NAMES luasandboxheka PATHS ${LUASANDBOX_LIB_DIR}) -FIND_PATH(LUASANDBOX_MODULES NAMES modules PATHS ${LUASANDBOX_LIB_DIR}/luasandbox) +FIND_LIBRARY(LUASANDBOX_TEST_LIBRARY NAMES luasandboxtest PATHS ${LUASANDBOX_LIB_DIR}) set(LUASANDBOX_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY} ${LUASANDBOX_LUASB_LIBRARY}) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index eba9c51..b4b2ce7 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -24,10 +24,7 @@ if(MSVC) else() # Predefined Macros: clang|gcc -dM -E -x c /dev/null # Compiler options: http://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html#Invoking-GCC - set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra") - if (NOT WIN32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC -fvisibility=hidden") - endif() + set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra -fPIC -fvisibility=hidden") set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") set(CMAKE_C_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index 4eb2136..0d9b453 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -18,8 +18,6 @@ Hindsight reference implementation. - the entire module is inaccessible - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - getenv, execute, exit, remove, rename, setlocale, tmpname -- [lsb_compression](../lsb_compression.html) - - the entire module is inaccessible ## Required Lua Functions (called by the host) diff --git a/docs/heka/index.md b/docs/heka/index.md index ec68e57..62ec350 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -39,13 +39,12 @@ There are a few intentional changes between tho original Heka sandbox and this v #### Input Sandbox -1. A [Heka Stream Reader](stream_reader.html) Lua module was added. -1. A [Heka JSON](json.html) Lua module was added. +1. A [create_stream_reader](input.html#create_stream_reader) function was added. #### Output Sandbox 1. [update_checkpoint](output.html#update_checkpoint) was added for batch and asynchronous processing. -1. A [Heka JSON](json.html) Lua module was added. +1. [create_message_matcher](output.html#create_message_matcher) function was added. ### Removals diff --git a/docs/heka/input.md b/docs/heka/input.md index 7de7df1..de627f1 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -64,6 +64,59 @@ to the appropriate configuration value. *Return* * none - throws an error on invalid input +### create_stream_reader +Creates a Heka stream reader to enable parsing of a framed Heka protobuf stream +in a Lua sandbox. See: +[Example of a Heka protobuf reader](#example-of-a-heka-protobuf-stdin-reader) + +*Arguments* +* name (string) - name of the stream reader (used in the log) + +*Return* +* hsr (userdata) - Heka stream reader or an error is thrown + +#### Heka Stream Reader Methods + +##### find_message + +Locates a Heka message within the stream. + +```lua +local found, consumed, need = hsr:find_message(buf) + +``` + +*Arguments* +* buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data or a userdate file object +* decode (bool default: true) - true if the framed message should be protobuf decoded + +*Return* +* found (bool) - true if a message was found +* consumed (number) - number of bytes consumed so the offset can be tracked for checkpointing purposes +* need/read (number) - number of bytes needed to complete the message or fill the underlying buffer + or in the case of a file object the number of bytes added to the buffer + +##### decode_message + +Converts a Heka protobuf encoded message string into a stream reader representation. +Note: this operation clears the internal stream reader buffer. + +*Arguments* +* heka_pb (string) - Heka protobuf binary string + +*Return* +* none - throws an error on failure + +##### read_message + +Provides access to the Heka message data within the reader object. + +```lua +local ts = hsr:read_message("Timestamp") + +``` +See [read_message](analysis.html#read_message) for details. + ## Modes of Operation ### Run Once @@ -98,7 +151,7 @@ end called again. * The `instruction_limit` configuration can be set if desired. -#### Example startup ping +#### Example heartbeat ping ```lua -- cfg ticker_interval = 60 @@ -150,21 +203,24 @@ instruction_limit = 0 --]] + local stdin = require "io".stdin -require "heka_stream_reader" +require "string" -local hsr = heka_stream_reader.new(read_config("Logger")) +local hsr = create_stream_reader(read_config("Logger")) function process_message() + local cnt = 0 local found, consumed, read repeat repeat found, consumed, read = hsr:find_message(stdin) if found then inject_message(hsr) + cnt = cnt + 1 end until not found until read == 0 - return 0 + return 0, string.format("processed %d messages", cnt) end ``` diff --git a/docs/heka/json.md b/docs/heka/json.md deleted file mode 100644 index 30cefc1..0000000 --- a/docs/heka/json.md +++ /dev/null @@ -1,208 +0,0 @@ -# Heka JSON Module - -Allows for JSON-Schema validation and more efficient manipulation of large JSON -structures where only a small amount of the data is consumed by the plugin. -Schema pattern matching is restricted to a subset of regex decribed in the -Regular Expression section of the -[RapidJSON Schema Documentation](http://rapidjson.org/md_doc_schema.html). - -## Availability - -Input/Output plugins only. - -## API Functions - -### parse - -Creates a Heka JSON Document from a string. - -```lua -local ok, doc = pcall(heka_json.parse, '{"foo":"bar"}') -assert(ok, doc) - -``` -*Arguments* -* JSON (string) - JSON string to parse - -*Return* -* doc (userdata) - Heka JSON document or an error is thrown - -### parse_schema - -Creates a Heka JSON Schema. - -```lua -local ok, doc = pcall(heka_json.parse_schema, '{"type":"array","minItems": 1,"oneOf": [{"items": {"type":"number"}}]}') -assert(ok, doc) - -``` -*Arguments* -* JSON (string) - JSON schema string to parse - -*Return* -* schema (userdata) - Heka JSON schema or an error is thrown - -### parse_message - -Creates a Heka JSON Document from a message variable. - -```lua -local ok, doc = pcall(heka_json.parse_message, "Fields[myjson]") -assert(ok, doc) - -``` -*Arguments* -* heka_stream_reader (userdata) - require only for Input plugins since there is - no active message available. -* variableName (string) - * Payload - * Fields[*name*] -* fieldIndex (unsigned) - optional, only used in combination with the Fields - variableName use to retrieve a specific instance of a repeated field name; - zero indexed -* arrayIndex (unsigned) - optional, only used in combination with the Fields - variableName use to retrieve a specific element out of a field containing an - array; zero indexed - -*Return* -* doc (userdata) - Heka JSON document or an error is thrown - -## Heka JSON Document API Methods - -### validate - -Checks that the JSON document conforms to the specified schema. - -```lua -local ok, err = doc:validate(schema) -assert(ok, err) - -``` -*Arguments* -* heka_schema (userdata) - a compiled schema to validate against - -*Return* -* ok (bool) - true if valid -* err (string) - error message on failure - -### find - -Searches for and returns a value in the JSON structure. - -```lua -local v = doc:find("obj", "arr", 0, "foo") -assert(v, "not found") - -``` -*Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document -* key (string, number) - object key, or array index -* keyN (string, number) - final object key, or array index - -*Return* -* value (lightuserdata) - handle to be passed to other methods, nil if not found - -### remove - -Searches for and NULL's out the resulting value in the JSON structure returning the -extracted Heka JSON document. - -```lua -local rv = doc:remove("obj", "arr") -assert(rv, "not found") -rv:size() -- number of elements in the extracted array - -``` -*Arguments* -* value (lightuserdata) - optional, when not specified the function is applied to document -* key (string, number) - object key, or array index -* keyN (string, number) - final object key, or array index - -*Return* -* doc (userdata) - the object removed from the original JSON or nil - -### value - -Returns the primitive value of the JSON element. - -```lua -local v = doc:find("obj", "arr", 0, "foo") -local str = doc:value(v) -assert("bar" == str, tostring(str)) - -``` -*Arguments* -* value (lightuserdata, nil) - optional, when not specified the function is applied to document - (accepts nil for easier nesting without having to test the inner expression) - e.g., str = doc:value(doc:find("foo")) or "my default" - -*Return* -* primitive - string, number, bool, nil or throws an error if not convertable (object, array) - -### type - -Returns the type of the value in the JSON structure. - -```lua -local t = doc:type() -assert(t == "object", t) - -``` -*Arguments* -* value (lightuserdata, nil) - optional, when not specified the function is applied to document - (accepts nil for easier nesting without having to test the inner expression) - -*Return* -* type (string, nil) - "string", "number", "boolean", "object", "array" or "null" - -### iter - -Retrieves an interator function for an object/array. - -```lua -local v = doc:find("obj", "arr") -for i,v in doc:iter(v) do --- ... -end -``` -*Arguments* -* value (lightuserdata, nil) - optional, when not specified the function is applied to document - (accepts nil for API consistency) - -*Return* -* iter (function, nil) - iterator function returning an index/value for arrays or a key/value for - objects. Throws an error on primitive types. - -### size - -Returns the size of the value. -```lua -local v = doc:find("obj", "arr") -local n = doc:size(v) - -``` -*Arguments* -* value (lightuserdata, nil) - optional, when not specified the function is applied to document - (accepts nil for easier nesting without having to test the inner expression) - -*Return* -* size (number, nil) - Number of element in an array/object or the length of the string. - Throws an error on numeric, boolean and null types. - -### make_field - -Helper function to wrap the lightuserdata so it can be used in an inject_message field. - -```lua -local msg = {Fields = {}} -local v = doc:find("obj", "arr") -msg.Fields.array = doc:make_field(v) -- set array to the JSON string representation of "arr" -inject_message(msg) - -``` -*Arguments* -* value (lightuserdata, nil) - optional, when not specified the function is applied to document - (accepts nil for easier nesting without having to test the inner expression) - -*Return* -* field (table, nil) - i.e., `{value = v, userdata = doc}` diff --git a/docs/heka/kafka_consumer.md b/docs/heka/kafka_consumer.md deleted file mode 100644 index 5fe75fb..0000000 --- a/docs/heka/kafka_consumer.md +++ /dev/null @@ -1,49 +0,0 @@ -# Heka Kafka Consumer - -Kafka Lua module for input sandbox plugins. - -## Functions - -### new - -Creates a Heka Kafka consumer. - -```lua -local brokerlist = "localhost:9092" -local topics = {"test"} -local consumer_conf = {["group.id"] = "test_g1"}) -local topic_conf = nil -local consumer = heka_kafka_consumer.new(brokerlist, topics, consumer_conf, topic_conf) - -``` - -*Arguments* -* brokerlist (string) - [librdkafka broker string](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205) -* topics (array of 'topic[:partition]' strings) - Balanced consumer group mode a - consumer can only subscribe on topics, not topics:partitions. The partition - syntax is only used for manual assignments (without balanced consumer groups). -* consumer_conf (table) - must contain 'group.id' see: [librdkafka consumer configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties) -* topic_conf (table, optional) - [librdkafka topic configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties) - -*Return* -* consumer (userdata) - Kafka consumer or an error is thrown - -## Methods - -### receive - -Receives a message from the specified Kafka topic(s). - -```lua -local msg, topic, partition, key = consumer:receive() - -``` - -*Arguments* -* none - -*Return* -* msg (string) - Kafka message payload -* topic (string) - Topic name the message was received from -* partition (number) - Topic partition the message was received from -* key (string) - Message key (if available) diff --git a/docs/heka/kafka_producer.md b/docs/heka/kafka_producer.md deleted file mode 100644 index 036c57c..0000000 --- a/docs/heka/kafka_producer.md +++ /dev/null @@ -1,118 +0,0 @@ -# Heka Kafka Producer - -Kafka Lua module for output sandboxes. - -## Functions - -### new - -Creates a Heka Kafka producer. - -```lua -local brokerlist = "localhost:9092" -local producer_conf = producer_conf = { - ["queue.buffering.max.messages"] = 20000, - ["batch.num.messages"] = 200, - ["message.max.bytes"] = 1024 * 1024, - ["queue.buffering.max.ms"] = 10, - ["topic.metadata.refresh.interval.ms"] = -1, -} -local producer = heka_kafka_producer.new(brokerlist, producer_conf) - -``` - -*Arguments* -* brokerlist (string) - [librdkafka broker string](https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205) -* producer_conf (table) - [librdkafka producer configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties) - -*Return* -* producer (userdata) - Kafka producer or an error is thrown - -## Methods - -### create_topic - -Creates a topic to be used by a producer, no-op if the topic already exists. - -```lua -producer:create_topic(topic) -- creates the topic if it does not exist - -``` - -*Arguments* -* topic (string) - Name of the topic - -*Return* -* none - - -### has_topic - -Tests if a producer is managing a topic. - -```lua -local b = producer:has_topic(topic) - -``` - -*Arguments* -* topic (string) - Name of the topic - -*Return* -* bool - True if the producer is managing a topic with the specificed name - - -### destroy_topic - -Removes a topic from the producer. - -```lua -producer:destroy_topic(topic) - -``` - -*Arguments* -* topic (string) - Name of the topic - -*Return* -* none - no-op on non-existent topic - - -### send - -Sends a message using the specified topic. - -```lua -local ret = producer:send(topic, -1, sequence_id, payload) - -``` - -*Arguments* -* topic (string) - Name of the topic -* partition (number) - Topic partition number (-1 for automatic assignment) -* sequence_id (lightuserdata) - Opaque pointer for checkpointing (passed into process_message) -* payload (string or (nil/none)) - Use nil/none to send the current active message while in - the process_message call - -*Return* -* ret (number) - 0 on success or errno - - ENOBUFS (105) maximum number of outstanding messages has been reached - - EMSGSIZE (90) message is larger than configured max size - - ESRCH (2) requested partition is unknown in the Kafka cluster - - ENOENT (3) topic is unknown in the Kafka cluster - -### poll - -Polls the provided Kafka producer for events and invokes callback. This should -be called after every send. - -```lua -producer:poll() - -``` - -*Arguments* -* none - -*Return* -* none diff --git a/docs/heka/output.md b/docs/heka/output.md index 0afa04c..758caed 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -413,47 +413,39 @@ producer_conf = { -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "kafka.producer" -require "kafka.topic" - -local brokers = read_config("brokers") or error("brokers must be set") +local brokerlist = read_config("brokerlist") or error("brokerlist must be set") local topic_constant = read_config("topic_constant") local topic_variable = read_config("topic_variable") or "Logger" local producer_conf = read_config("producer_conf") -local producer = kafka.producer.new(brokers, producer_conf) -local topics = {} +local producer = kafka.producer(brokerlist, producer_conf) function process_message(sequence_id) - local var = topic_constant - if not var then - var = read_message(topic_variable) or "unknown" - end - - local topic = topics[var] + local topic = topic_constant if not topic then - topic = kafka.topic.new(producer, var) - topics[var] = topic + topic = read_message(topic_variable) or "unknown" end + producer:create_topic(topic) -- creates the topic if it does not exist - producer:poll() -- calls async_checkpoint_update - local ret = topic:send(0, read_message("raw"), sequence_id) + producer:poll() + local ret = producer:send(topic, -1, sequence_id) -- sends the current message if ret ~= 0 then if ret == 105 then - return -3 -- queue full retry + return -3, "queue full" -- retry elseif ret == 90 then - return -1 -- message too large + return -1, "message too large" -- fail elseif ret == 2 then - error("unknown topic: " .. var) + error("unknown topic: " .. topic) elseif ret == 3 then error("unknown partition") end end + return -5 -- asynchronous checkpoint management end function timer_event(ns) - producer:poll() -- calls update_checkpoint + producer:poll() end ``` diff --git a/docs/heka/stream_reader.md b/docs/heka/stream_reader.md deleted file mode 100644 index 71d8e5c..0000000 --- a/docs/heka/stream_reader.md +++ /dev/null @@ -1,63 +0,0 @@ -# Heka Stream Reader Module - -Enables parsing of a framed Heka protobuf stream in a Lua sandbox. See: -[Example of a Heka protobuf reader](input.html#example-of-a-heka-protobuf-stdin-reader) - -## Functions - -### new - -Creates a Heka stream reader. - -```lua -local hsr = heka_stream_reader.new("stdin") - -``` - -*Arguments* -* name (string) - name of the stream reader (used in the log) - -*Return* -* hsr (userdata) - Heka stream reader or an error is thrown - -## Methods - -### find_message - -Locates a Heka message within the stream. - -```lua -local found, consumed, need = hsr:find_message(buf) - -``` - -*Arguments* -* buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data or a userdate file object -* decode (bool default: true) - true if the framed message should be protobuf decoded - -*Return* -* found (bool) - true if a message was found -* consumed (number) - number of bytes consumed so the offset can be tracked for checkpointing purposes -* need/read (number) - number of bytes needed to complete the message or fill the underlying buffer - or in the case of a file object the number of bytes added to the buffer - -### decode_message - -Converts a Heka protobuf encoded message string into a stream reader representation. -Note: this operation clears the internal stream reader buffer. - -*Arguments* -* heka_pb (string) - Heka protobuf binary string - -*Return* -* none - throws an error on failure - -### read_message - -Provides access to the Heka message data within the reader object. - -```lua -local ts = hsr:read_message("Timestamp") - -``` -See [read_message](analysis.html#read_message) for details. diff --git a/docs/lsb_compression.md b/docs/lsb_compression.md deleted file mode 100644 index 7522efe..0000000 --- a/docs/lsb_compression.md +++ /dev/null @@ -1,24 +0,0 @@ -# Lua Sandbox Compression Module - -## Overview -Built-in Lua Sandbox module for various types of compresssion. - - -### Example Usage -```lua -require "lsb_compression" -local inflated = lsb_compression.ungzip("\031\139\008\000\000\000\000\000\000\003\075\203\207\079\074\044\226\002\000\071\151\044\178\007\000\000\000") --- inflated == "foobar\n" -``` - -### Functions - -#### ungzip -Inflates a gzipped string (only availble if libz is installed on the system). - -*Arguments* -- bytes (string) - gzip compressed string to inflate -- max_size (number, nil) - maximum size the of the resulting string (default 0 (unlimited)) - -*Return* -- inflated (string) - nil if bytes are not a gzip string or the output would exceed `max_size` diff --git a/docs/lsb_hash.md b/docs/lsb_hash.md deleted file mode 100644 index 3b0bb4f..0000000 --- a/docs/lsb_hash.md +++ /dev/null @@ -1,23 +0,0 @@ -# Lua Sandbox Hash Module - -## Overview -Built-in Lua Sandbox module for various types of hashing. - -### Example Usage -```lua -require "lsb_hash" -local crc = lsb_hash.crc32("foobar") --- crc == 2666930069 -``` - -### Functions - -#### crc32 - -CRC-32 checksum (only availble if libz is installed on the system) . - -*Arguments* -- value (string) - string to checksum - -*Return* -- checksum (number) - unsigned int diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index 10e0dbf..be340bb 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -191,18 +191,6 @@ local function output_menu(before, after, paths, version)
            • C Modules
              • diff --git a/include/luasandbox.h b/include/luasandbox.h index c631a0a..f651d11 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -29,6 +29,7 @@ #define LSB_SHUTTING_DOWN "shutting down" #define LSB_CONFIG_TABLE "lsb_config" +#define LSB_THIS_PTR "lsb_this_ptr" #define LSB_MEMORY_LIMIT "memory_limit" #define LSB_INSTRUCTION_LIMIT "instruction_limit" #define LSB_OUTPUT_LIMIT "output_limit" diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 876d068..b6a38fa 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -31,6 +31,7 @@ #define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" #define LSB_HEKA_UPDATE_CHECKPOINT "update_checkpoint" +#define LSB_HEKA_THIS_PTR "lsb_heka_this_ptr" enum lsb_heka_pm_rv { LSB_HEKA_PM_SENT = 0, @@ -325,6 +326,26 @@ LSB_HEKA_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); */ LSB_HEKA_EXPORT bool lsb_heka_is_running(lsb_heka_sandbox *hsb); +/** + * Retrieve the currently active sandbox message. This call returns a handle to + * internal data and is not thread safe. + * * + * @param hsb Heka sandbox + * + * @return const lsb_heka_message* NULL if there is no active message + */ +LSB_HEKA_EXPORT const lsb_heka_message* +lsb_heka_get_message(lsb_heka_sandbox *hsb); + +/** + * Retrieve the sandbox tyye. + * * + * @param hsb Heka sandbox + * + * @return char Heka sandbox type identifer + */ +LSB_HEKA_EXPORT char lsb_heka_get_type(lsb_heka_sandbox *hsb); + #ifdef __cplusplus } #endif diff --git a/src/heka/stream_reader_impl.h b/include/luasandbox/heka/stream_reader.h similarity index 55% rename from src/heka/stream_reader_impl.h rename to include/luasandbox/heka/stream_reader.h index bc08660..18f83d0 100644 --- a/src/heka/stream_reader_impl.h +++ b/include/luasandbox/heka/stream_reader.h @@ -6,20 +6,13 @@ /** Hindsight Heka stream reader structures @file */ -#ifndef luasandbox_heka_stream_reader_impl_h_ -#define luasandbox_heka_stream_reader_impl_h_ +#ifndef luasandbox_heka_stream_reader_h_ +#define luasandbox_heka_stream_reader_h_ -#ifdef __cplusplus -extern "C" -{ -#endif - -#include "luasandbox/lua.h" -#include "luasandbox/util/input_buffer.h" -#include "luasandbox/util/heka_message.h" +#include "../util/heka_message.h" +#include "../util/input_buffer.h" -extern const char *mozsvc_heka_stream_reader; -extern const char *mozsvc_heka_stream_reader_table; +#define LSB_HEKA_STREAM_READER "lsb.heka_stream_reader" typedef struct heka_stream_reader { @@ -28,10 +21,4 @@ typedef struct heka_stream_reader lsb_input_buffer buf; } heka_stream_reader; -int luaopen_heka_stream_reader(lua_State *lua); - -#ifdef __cplusplus -} -#endif - #endif diff --git a/src/test/mu_test.h b/include/luasandbox/test/mu_test.h similarity index 98% rename from src/test/mu_test.h rename to include/luasandbox/test/mu_test.h index fcaa293..e8d6bc0 100644 --- a/src/test/mu_test.h +++ b/include/luasandbox/test/mu_test.h @@ -11,7 +11,9 @@ #include -#include "../luasandbox_defines.h" +#ifdef _WIN32 +#define snprintf _snprintf +#endif #if defined(_MSC_VER) #define PRIuSIZE "Iu" diff --git a/include/luasandbox/test/sandbox.h b/include/luasandbox/test/sandbox.h new file mode 100644 index 0000000..3009e54 --- /dev/null +++ b/include/luasandbox/test/sandbox.h @@ -0,0 +1,51 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Test interface for the generic lua sandbox @file */ + +#ifndef luasandbox_test_sandbox_h_ +#define luasandbox_test_sandbox_h_ + +#include + +#include "../../luasandbox.h" +#include "../error.h" + +#ifdef _WIN32 +#define LSB_TEST_MODULE_PATH "path = '.\\\\?.lua';cpath = '.\\\\?.dll'\n" +#ifdef luasandboxtest_EXPORTS +#define LSB_TEST_EXPORT __declspec(dllexport) +#else +#define LSB_TEST_EXPORT __declspec(dllimport) +#endif +#else +#define LSB_TEST_MODULE_PATH "path = './?.lua';cpath = './?.so'\n" +#if __GNUC__ >= 4 +#define LSB_TEST_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_TEST_EXPORT +#endif +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif +#include "../lua.h" + +LSB_TEST_EXPORT extern const char *lsb_test_output; +LSB_TEST_EXPORT extern size_t lsb_test_output_len; +LSB_TEST_EXPORT extern lsb_logger lsb_test_logger; + +LSB_TEST_EXPORT int lsb_test_process(lsb_lua_sandbox *lsb, double ts); +LSB_TEST_EXPORT int lsb_test_report(lsb_lua_sandbox *lsb, double tc); +LSB_TEST_EXPORT int lsb_test_write_output(lua_State *lua); + +#ifdef __cplusplus +} +#endif + +#endif \ No newline at end of file diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index 0af3158..6d7e60f 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -186,7 +186,7 @@ LSB_UTIL_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, * * @return bool True on success */ -LSB_UTIL_EXPORT bool lsb_read_heka_field(lsb_heka_message *m, +LSB_UTIL_EXPORT bool lsb_read_heka_field(const lsb_heka_message *m, lsb_const_string *name, int fi, int ai, diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h index cd6d65d..bccc66a 100644 --- a/include/luasandbox/util/util.h +++ b/include/luasandbox/util/util.h @@ -65,7 +65,6 @@ LSB_UTIL_EXPORT char* lsb_read_file(const char *fn); */ LSB_UTIL_EXPORT unsigned long long lsb_get_time(); - /** * Retrieves the highest resolution time since Jan 1, 1970 converted to * nanoseconds @@ -83,29 +82,6 @@ LSB_UTIL_EXPORT long long lsb_get_timestamp(); */ LSB_UTIL_EXPORT bool lsb_set_tz(const char *tz); -/** - * ZLIB CRC-32 checksum - * - * @param buf - array of bytes - * @param buf_len - length of the array - * - * @return LSB_UTIL_EXPORT unsigned checksum - */ -LSB_UTIL_EXPORT uint32_t lsb_crc32(const char *buf, size_t buf_len); - -/** - * ZLIB Inflate the provided string into a new string - * - * @param s String to ungzip - * @param s_len Length of the string to ungzip - * @param max_len Maximum length the buffer is allowed to grow to (0 for - * unlimited) - * @param r_len (optional) Length of the returned string - * @return char* Returned string (MUST be freed by the caller), NULL on failure - */ -LSB_UTIL_EXPORT char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, - size_t *r_len); - #ifdef __cplusplus } #endif diff --git a/sandboxes/heka/input/heka_kafka.lua b/sandboxes/heka/input/heka_kafka.lua index 305d26c..2349a99 100644 --- a/sandboxes/heka/input/heka_kafka.lua +++ b/sandboxes/heka/input/heka_kafka.lua @@ -2,7 +2,7 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "heka_kafka_consumer" +require "kafka" --[[ # Heka Kafka Consumer Input @@ -37,7 +37,7 @@ local topics = read_config("topics") or error("topics must be set") local consumer_conf = read_config("consumer_conf") local topic_conf = read_config("topic_conf") -local consumer = heka_kafka_consumer.new(brokerlist, topics, consumer_conf, topic_conf) +local consumer = kafka.consumer(brokerlist, topics, consumer_conf, topic_conf) local err_msg = { Logger = read_config("Logger"), diff --git a/sandboxes/heka/input/heka_stdin.lua b/sandboxes/heka/input/heka_stdin.lua index d788092..747a6d2 100644 --- a/sandboxes/heka/input/heka_stdin.lua +++ b/sandboxes/heka/input/heka_stdin.lua @@ -12,10 +12,9 @@ filename = "heka_stdin.lua" --]] local stdin = require "io".stdin -require "heka_stream_reader" require "string" -local hsr = heka_stream_reader.new(read_config("Logger")) +local hsr = create_stream_reader(read_config("Logger")) function process_message() local cnt = 0 diff --git a/sandboxes/heka/input/heka_tcp.lua b/sandboxes/heka/input/heka_tcp.lua index bf326ef..9947503 100644 --- a/sandboxes/heka/input/heka_tcp.lua +++ b/sandboxes/heka/input/heka_tcp.lua @@ -27,7 +27,6 @@ ssl_params = { require "coroutine" local socket = require "socket" -require "heka_stream_reader" require "string" require "table" @@ -46,8 +45,7 @@ local sockets = {server} local function handle_client(client, caddr, cport) local found, consumed, need = false, 0, 8192 * 4 - local hsr = heka_stream_reader.new( - string.format("%s:%d -> %s:%d", caddr, cport, address, port)) + local hsr = create_stream_reader(string.format("%s:%d -> %s:%d", caddr, cport, address, port)) client:settimeout(0) while client do local buf, err, partial = client:receive(need) diff --git a/sandboxes/heka/output/elasticsearch_bulk_api.lua b/sandboxes/heka/output/elasticsearch_bulk_api.lua index 8b21cb3..74a8fcc 100644 --- a/sandboxes/heka/output/elasticsearch_bulk_api.lua +++ b/sandboxes/heka/output/elasticsearch_bulk_api.lua @@ -31,7 +31,7 @@ encoder_cfg = { --]] require "table" -require "heka_json" +require "rjson" require "string" local ltn12 = require "ltn12" local time = require "os".time @@ -88,7 +88,7 @@ local function send_request() -- hand coded since socket.http doesn't support ke local sink = ltn12.sink.table(body) client:receivebody(headers, sink) local response = table.concat(body) - local ok, doc = pcall(heka_json.parse, response) + local ok, doc = pcall(rjson.parse, response) if ok then if doc:value(doc:find("errors")) then ret = -1 diff --git a/sandboxes/heka/output/heka_kafka.lua b/sandboxes/heka/output/heka_kafka.lua index 661dbd9..1bbdf68 100644 --- a/sandboxes/heka/output/heka_kafka.lua +++ b/sandboxes/heka/output/heka_kafka.lua @@ -2,7 +2,7 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "heka_kafka_producer" +require "kafka" --[[ # Heka Kafka Producer Output @@ -31,7 +31,7 @@ local topic_constant = read_config("topic_constant") local topic_variable = read_config("topic_variable") or "Logger" local producer_conf = read_config("producer_conf") -local producer = heka_kafka_producer.new(brokerlist, producer_conf) +local producer = kafka.producer(brokerlist, producer_conf) function process_message(sequence_id) local topic = topic_constant diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fa9bf4e..661552f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,7 +4,6 @@ set(LUA_SANDBOX_SRC luasandbox.c -luasandbox_modules.c luasandbox_output.c luasandbox_serialize.c ) @@ -27,10 +26,8 @@ if(LIBM_LIBRARY) endif() install(TARGETS luasandbox DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) -install(DIRECTORY "${EP_BASE}/lib/" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core PATTERN "lua" EXCLUDE) -install(DIRECTORY "${EP_BASE}/lib/lua/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/modules COMPONENT core) -install(DIRECTORY "${EP_BASE}/io/lib/lua/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/io_modules COMPONENT core) -install(DIRECTORY "${EP_BASE}/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core) +install(DIRECTORY "${EP_BASE}/inst/${CMAKE_INSTALL_LIBDIR}/" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) +install(DIRECTORY "${EP_BASE}/inst/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core) add_subdirectory(util) add_subdirectory(heka) diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index ed55c4c..ed5d09a 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -4,15 +4,10 @@ set(HEKA_SRC message.c -rapidjson.cpp sandbox.c stream_reader.c ) -if(LIBRTKAFKA_LIBRARY) - set(HEKA_SRC ${HEKA_SRC} kafka.c) -endif() - add_library(luasandboxheka SHARED ${HEKA_SRC}) set_target_properties(luasandboxheka PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) target_compile_definitions(luasandboxheka PRIVATE -Dluasandboxheka_EXPORTS) @@ -25,9 +20,5 @@ if(LIBM_LIBRARY) target_link_libraries(luasandboxheka ${LIBM_LIBRARY}) endif() -if(LIBRTKAFKA_LIBRARY) - target_link_libraries(luasandboxheka ${LIBRTKAFKA_LIBRARY}) -endif() - install(TARGETS luasandboxheka DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) add_subdirectory(test) diff --git a/src/heka/kafka.c b/src/heka/kafka.c deleted file mode 100644 index b242bf6..0000000 --- a/src/heka/kafka.c +++ /dev/null @@ -1,753 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua Kafka functions @file */ - -#include -#include -#include -#include -#include -#include -#include - -#include "luasandbox/heka/sandbox.h" -#include "luasandbox/lauxlib.h" -#include "luasandbox/lua.h" -#include "kafka_impl.h" -#include "sandbox_impl.h" - -const char *mozsvc_heka_kafka_producer = "mozsvc.heka_kafka_producer"; -const char *mozsvc_heka_kafka_producer_table = "heka_kafka_producer"; - -const char *mozsvc_heka_kafka_consumer = "mozsvc.heka_kafka_consumer"; -const char *mozsvc_heka_kafka_consumer_table = "heka_kafka_consumer"; - -typedef struct heka_kafka_producer { - rd_kafka_t *rk; - void *msg_opaque; - int failures; -} heka_kafka_producer; - - -typedef struct heka_kafka_consumer { - rd_kafka_t *rk; - rd_kafka_topic_partition_list_t *topics; -} heka_kafka_consumer; - - -typedef struct heka_kafka_topic { - rd_kafka_topic_t *rkt; -} heka_kafka_topic; - - -static heka_kafka_producer* check_producer(lua_State *lua, int min_args) -{ - heka_kafka_producer *kp = luaL_checkudata(lua, 1, mozsvc_heka_kafka_producer); - int n = lua_gettop(lua); - luaL_argcheck(lua, min_args <= n, n, "incorrect number of arguments"); - return kp; -} - - -static void msg_delivered(rd_kafka_t *rk, - void *payload, - size_t len, - int error_code, - void *opaque, - void *msg_opaque) -{ - (void)rk; - (void)payload; - (void)len; - heka_kafka_producer *kp = (heka_kafka_producer *)opaque; - kp->msg_opaque = msg_opaque; - if (error_code) ++kp->failures; -} - - -static bool load_conf(lua_State *lua, rd_kafka_conf_t *conf, int idx) -{ - if (!conf) { - lua_pushstring(lua, "rd_kafka_conf_new() failed"); - return false; - } - if (lua_isnil(lua, idx)) { - return true; - } - - char errstr[512]; - lua_pushnil(lua); - while (lua_next(lua, idx) != 0) { - int kt = lua_type(lua, -2); - int vt = lua_type(lua, -1); - switch (kt) { - case LUA_TSTRING: - switch (vt) { - case LUA_TSTRING: - { - const char *key = lua_tostring(lua, -2); - const char *value = lua_tostring(lua, -1); - if (value) { - rd_kafka_conf_res_t r; - r = rd_kafka_conf_set(conf, key, value, errstr, sizeof errstr); - if (r) { - lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, - errstr); - return false; - } - } - } - break; - case LUA_TNUMBER: - { - const char *key = lua_tostring(lua, -2); - int i = (int)lua_tointeger(lua, -1); - char value[12]; - snprintf(value, sizeof value, "%d", i); - rd_kafka_conf_res_t r; - r = rd_kafka_conf_set(conf, key, value, errstr, sizeof errstr); - if (r) { - lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, - errstr); - return false; - } - } - break; - case LUA_TBOOLEAN: - { - const char *key = lua_tostring(lua, -2); - const char *value = "false"; - if (lua_toboolean(lua, -1)) { - value = "true"; - } - rd_kafka_conf_res_t r; - r = rd_kafka_conf_set(conf, key, value, errstr, sizeof errstr); - if (r) { - lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, - errstr); - return false; - } - } - break; - default: - lua_pushfstring(lua, "invalid config value type: %s", - lua_typename(lua, vt)); - return false; - } - break; - default: - lua_pushfstring(lua, "invalid config key type: %s", - lua_typename(lua, kt)); - return false; - } - lua_pop(lua, 1); - } - return true; -} - - -static -bool load_topic_conf(lua_State *lua, rd_kafka_topic_conf_t *conf, int idx) -{ - if (!conf) { - lua_pushstring(lua, "rd_kafka_topic_conf_new() failed"); - return false; - } - if (lua_isnil(lua, idx)) { - return true; - } - - char errstr[512]; - lua_pushnil(lua); - while (lua_next(lua, idx) != 0) { - int kt = lua_type(lua, -2); - int vt = lua_type(lua, -1); - switch (kt) { - case LUA_TSTRING: - switch (vt) { - case LUA_TSTRING: - { - const char *key = lua_tostring(lua, -2); - const char *value = lua_tostring(lua, -1); - if (value) { - rd_kafka_conf_res_t r; - r = rd_kafka_topic_conf_set(conf, key, value, errstr, - sizeof errstr); - if (r) { - lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, - errstr); - return false; - } - } - } - break; - case LUA_TNUMBER: - { - const char *key = lua_tostring(lua, -2); - int i = (int)lua_tointeger(lua, -1); - char value[12]; - snprintf(value, sizeof value, "%d", i); - rd_kafka_conf_res_t r; - r = rd_kafka_topic_conf_set(conf, key, value, errstr, - sizeof errstr); - if (r) { - lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, - errstr); - return false; - } - } - break; - case LUA_TBOOLEAN: - { - const char *key = lua_tostring(lua, -2); - const char *value = "false"; - if (lua_toboolean(lua, -1)) { - value = "true"; - } - rd_kafka_conf_res_t r; - r = rd_kafka_topic_conf_set(conf, key, value, errstr, - sizeof errstr); - if (r) { - lua_pushfstring(lua, "Failed to set %s = %s : %s", key, value, - errstr); - return false; - } - } - break; - default: - lua_pushfstring(lua, "invalid config value type: %s", - lua_typename(lua, vt)); - return false; - } - break; - default: - lua_pushfstring(lua, "invalid config key type: %s", - lua_typename(lua, kt)); - return false; - } - lua_pop(lua, 1); - } - return true; -} - - -static int producer_new(lua_State *lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n >= 1 && n <= 2, n, "incorrect number of arguments"); - - const char *brokerlist = luaL_checkstring(lua, 1); - int t = lua_type(lua, 2); - switch (t) { - case LUA_TNONE: - lua_pushnil(lua); - break; - case LUA_TNIL: - break; - default: - luaL_checktype(lua, 2, LUA_TTABLE); // producer config - } - - heka_kafka_producer *kp = lua_newuserdata(lua, sizeof(heka_kafka_producer)); - kp->rk = NULL; - kp->msg_opaque = NULL; - kp->failures = 0; - lua_pushlightuserdata(lua, kp); // setup a topic table for this producer - lua_newtable(lua); - lua_rawset(lua, LUA_ENVIRONINDEX); - luaL_getmetatable(lua, mozsvc_heka_kafka_producer); - lua_setmetatable(lua, -2); - - rd_kafka_conf_t *conf = rd_kafka_conf_new(); - if (!load_conf(lua, conf, 2)) { - rd_kafka_conf_destroy(conf); - return lua_error(lua); - } - rd_kafka_conf_set_opaque(conf, kp); - rd_kafka_conf_set_dr_cb(conf, msg_delivered); - rd_kafka_conf_set_log_cb(conf, NULL); // disable logging - - char errstr[512]; - kp->rk = rd_kafka_new(RD_KAFKA_PRODUCER, conf, errstr, sizeof errstr); - if (!kp->rk) { - rd_kafka_conf_destroy(conf); // the producer has not taken ownership - return luaL_error(lua, "rd_kafka_new failed: %s", errstr); - } - - if (rd_kafka_brokers_add(kp->rk, brokerlist) == 0) { - return luaL_error(lua, "invalid broker list"); - } - return 1; -} - - -static heka_kafka_topic* get_topic(lua_State *lua, heka_kafka_producer *kp, - const char *topic) -{ - heka_kafka_topic *kt = NULL; - lua_pushlightuserdata(lua, kp); - lua_rawget(lua, LUA_ENVIRONINDEX); - lua_getfield(lua, -1, topic); - if (lua_type(lua, -1) == LUA_TLIGHTUSERDATA) { - kt = lua_touserdata(lua, -1);; - } - lua_pop(lua, 2); - return kt; -} - - -static int producer_create_topic(lua_State *lua) -{ - heka_kafka_producer *kp = check_producer(lua, 2); - const char *topic = NULL; - int n = lua_gettop(lua); - luaL_argcheck(lua, n >= 2 && n <= 3, n, "incorrect number of arguments"); - - topic = luaL_checkstring(lua, 2); - - int t = lua_type(lua, 3); - switch (t) { - case LUA_TNONE: - lua_pushnil(lua); - break; - case LUA_TNIL: - break; - default: - luaL_checktype(lua, 3, LUA_TTABLE); // topic config - } - - if (get_topic(lua, kp, topic)) { - return 0; - } - - heka_kafka_topic *kt = malloc(sizeof(heka_kafka_topic)); - if (!kt) { - return luaL_error(lua, "memory allocation failed"); - } - - rd_kafka_topic_conf_t *tconf = rd_kafka_topic_conf_new(); - if (!load_topic_conf(lua, tconf, 3)) { - rd_kafka_topic_conf_destroy(tconf); - free(kt); - return lua_error(lua); - } - - char errstr[512]; - kt->rkt = rd_kafka_topic_new(kp->rk, topic, tconf); - if (!kt->rkt) { - rd_kafka_topic_conf_destroy(tconf); - free(kt); - return luaL_error(lua, "rd_kafka_topic_new failed: %s", errstr); - } - lua_pushlightuserdata(lua, kp); - lua_rawget(lua, LUA_ENVIRONINDEX); - lua_pushstring(lua, topic); - lua_pushlightuserdata(lua, kt); - lua_rawset(lua, -3); // update the producer topic table - return 0; -} - - -static int producer_has_topic(lua_State *lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, 2 == n, n, "incorrect number of arguments"); - heka_kafka_producer *kp = luaL_checkudata(lua, 1, mozsvc_heka_kafka_producer); - const char *topic = luaL_checkstring(lua, 2); - if (get_topic(lua, kp, topic)) { - lua_pushboolean(lua, true); - } else { - lua_pushboolean(lua, false); - } - return 1; -} - - -static int producer_destroy_topic(lua_State *lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, 2 == n, n, "incorrect number of arguments"); - heka_kafka_producer *kp = luaL_checkudata(lua, 1, mozsvc_heka_kafka_producer); - const char *topic = luaL_checkstring(lua, 2); - heka_kafka_topic *kt = get_topic(lua, kp, topic); - if (kt) { - lua_pushlightuserdata(lua, kp); - lua_rawget(lua, LUA_ENVIRONINDEX); - lua_pushnil(lua); - lua_setfield(lua, -2, topic); - rd_kafka_topic_destroy(kt->rkt); - free(kt); - } - return 0; -} - - -static int producer_poll(lua_State *lua) -{ - heka_kafka_producer *kp = check_producer(lua, 1); - kp->failures = 0; - kp->msg_opaque = NULL; - rd_kafka_poll(kp->rk, 0); - if (kp->msg_opaque) { - lua_getfield(lua, LUA_GLOBALSINDEX, LSB_HEKA_UPDATE_CHECKPOINT); - if (lua_type(lua, -1) == LUA_TFUNCTION) { - lua_pushlightuserdata(lua, kp->msg_opaque); - lua_pushinteger(lua, kp->failures); - if (lua_pcall(lua, 2, 0, 0)) { - lua_error(lua); - } - } - lua_pop(lua, 1); - } - return 0; -} - - -static int producer_send(lua_State *lua) -{ - heka_kafka_producer *kp = check_producer(lua, 4); - const char *topic = luaL_checkstring(lua, 2); - heka_kafka_topic *kt = get_topic(lua, kp, topic); - if (!kt) return luaL_error(lua, "invalid topic"); - - int32_t partition = (int32_t)luaL_checkinteger(lua, 3); - luaL_checktype(lua, 4, LUA_TLIGHTUSERDATA); - void *sequence_id = lua_touserdata(lua, 4); - - size_t len = 0; - const char *msg = NULL; - if (lua_type(lua, 5) == LUA_TSTRING) { - msg = lua_tolstring(lua, 5, &len); - } else { - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __func__); - } - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); - if (hsb->msg) { - len = hsb->msg->raw.len; - msg = hsb->msg->raw.s; - } else { - return luaL_error(lua, "no active message", __func__); - } - } - - errno = 0; - int ret = rd_kafka_produce(kt->rkt, partition, - RD_KAFKA_MSG_F_COPY, - (void *)msg, len, - NULL, 0, // optional key/len - sequence_id // opaque pointer - ); - if (ret == -1) { - lua_pushinteger(lua, errno); - } else { - lua_pushinteger(lua, 0); - } - return 1; -} - - -static int producer_gc(lua_State *lua) -{ - heka_kafka_producer *kp = check_producer(lua, 1); - lua_pushlightuserdata(lua, kp); - lua_rawget(lua, LUA_ENVIRONINDEX); - assert(lua_type(lua, -1) == LUA_TTABLE); - lua_pushnil(lua); /* first key */ - while (lua_next(lua, -2) != 0) { - heka_kafka_topic *kt = lua_touserdata(lua, -1); - if (kt) { - rd_kafka_topic_destroy(kt->rkt); - free(kt); - } - lua_pop(lua, 1); - } - if (kp->rk) rd_kafka_destroy(kp->rk); - - lua_pushlightuserdata(lua, kp); - lua_pushnil(lua); - lua_rawset(lua, LUA_ENVIRONINDEX); // remove the producer topic table - - // This may timeout because it might not be the last plugin running. - rd_kafka_wait_destroyed(1000); - return 0; -} - - -static const struct luaL_reg producerlib_f[] = -{ - { "new", producer_new }, - { NULL, NULL } -}; - - -static const struct luaL_reg producerlib_m[] = -{ - { "create_topic", producer_create_topic }, - { "has_topic", producer_has_topic }, - { "destroy_topic", producer_destroy_topic }, - { "poll", producer_poll }, - { "__gc", producer_gc }, - { NULL, NULL } -}; - - -int luaopen_heka_kafka_producer(lua_State *lua) -{ - lua_newtable(lua); - lua_replace(lua, LUA_ENVIRONINDEX); - - luaL_newmetatable(lua, mozsvc_heka_kafka_producer); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, producerlib_m); - // special case send since it needs access to the sandbox - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - lua_pushlightuserdata(lua, lsb); - lua_pushcclosure(lua, producer_send, 1); - lua_setfield(lua, -2, "send"); - luaL_register(lua, mozsvc_heka_kafka_producer_table, producerlib_f); - return 1; -} - - -static heka_kafka_consumer* check_consumer(lua_State *lua, int args) -{ - heka_kafka_consumer *kc = luaL_checkudata(lua, 1, mozsvc_heka_kafka_consumer); - int n = lua_gettop(lua); - luaL_argcheck(lua, args == n, n, "incorrect number of arguments"); - return kc; -} - - -static bool add_consumer_topics(lua_State *lua, - heka_kafka_consumer *kc, - int cnt) -{ - bool is_subscription = true; - - kc->topics = rd_kafka_topic_partition_list_new(cnt); - if (!kc->topics) { - lua_pushstring(lua, "rd_kafka_topic_partition_list_new failed"); - return false; - } - - lua_pushnil(lua); - while (lua_next(lua, 2) != 0) { - int kt = lua_type(lua, -2); - int vt = lua_type(lua, -1); - if (kt != LUA_TNUMBER || vt != LUA_TSTRING) { - lua_pushstring(lua, "topics must be an array of strings"); - return false; - } - const char *topic = lua_tostring(lua, -1); - char *t; - long partition = -1; - - if ((t = strstr(topic, ":"))) { // Parse "topic[:partition] - char s[strlen(topic)]; - memcpy(s, topic, t - topic); - s[t - topic] = 0; - - partition = strtol(t + 1, NULL, 10); - if (partition > INT32_MAX) { - lua_pushstring(lua, "invalid topic partition > INT32_MAX"); - return false; - } else if (partition < 0) { - lua_pushstring(lua, "invalid topic partition < 0"); - return false; - } - is_subscription = false; - rd_kafka_topic_partition_list_add(kc->topics, s, (int32_t)partition); - } else { - rd_kafka_topic_partition_list_add(kc->topics, topic, (int32_t)partition); - } - lua_pop(lua, 1); - } - - rd_kafka_resp_err_t err; - if (is_subscription) { - if ((err = rd_kafka_subscribe(kc->rk, kc->topics))) { - lua_pushfstring(lua, "rd_kafka_subscribe failed: %s", - rd_kafka_err2str(err)); - return false; - } - - } else { - if ((err = rd_kafka_assign(kc->rk, kc->topics))) { - lua_pushfstring(lua, "rd_kafka_assign failed: %s", rd_kafka_err2str(err)); - return false; - } - } - return true; -} - - -static int consumer_new(lua_State *lua) -{ - static const char *group_id = "group.id"; - int n = lua_gettop(lua); - luaL_argcheck(lua, n >= 3 && n <= 4, n, "incorrect number of arguments"); - - const char *brokerlist = luaL_checkstring(lua, 1); - - luaL_checktype(lua, 2, LUA_TTABLE); // topics - int topic_cnt = (int)lua_objlen(lua, 2); - luaL_argcheck(lua, topic_cnt > 0, 2, "the topics array is empty"); - - luaL_checktype(lua, 3, LUA_TTABLE); // consumer config - - int t = lua_type(lua, 4); // topic config - switch (t) { - case LUA_TNONE: - lua_pushnil(lua); - break; - case LUA_TNIL: - break; - default: - luaL_checktype(lua, 3, LUA_TTABLE); - } - - heka_kafka_consumer *kc = lua_newuserdata(lua, sizeof(heka_kafka_consumer)); - kc->rk = NULL; - kc->topics = NULL; - luaL_getmetatable(lua, mozsvc_heka_kafka_consumer); - lua_setmetatable(lua, -2); - - rd_kafka_conf_t *conf = rd_kafka_conf_new(); - if (!load_conf(lua, conf, 3)) { - rd_kafka_conf_destroy(conf); - return lua_error(lua); - } - - rd_kafka_conf_set_log_cb(conf, NULL); // disable logging - - char errstr[512]; - size_t len; - if (rd_kafka_conf_get(conf, group_id, NULL, &len) != RD_KAFKA_CONF_OK) { - rd_kafka_conf_destroy(conf); - return luaL_error(lua, "%s must be set", group_id); - } - - rd_kafka_topic_conf_t *tconf = rd_kafka_topic_conf_new(); - if (!load_topic_conf(lua, tconf, 4)) { - rd_kafka_topic_conf_destroy(tconf); - rd_kafka_conf_destroy(conf); - return lua_error(lua); - } - if (rd_kafka_topic_conf_set(tconf, "offset.store.method", "broker", - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK - || rd_kafka_topic_conf_set(tconf, "auto.commit.enable", "true", - errstr, sizeof(errstr)) != RD_KAFKA_CONF_OK) { - rd_kafka_topic_conf_destroy(tconf); - rd_kafka_conf_destroy(conf); - return luaL_error(lua, "rd_kafka_topic_conf_set failed: %s", errstr); - } - rd_kafka_conf_set_default_topic_conf(conf, tconf); - - kc->rk = rd_kafka_new(RD_KAFKA_CONSUMER, conf, errstr, sizeof errstr); - if (!kc->rk) { - rd_kafka_conf_destroy(conf); - return luaL_error(lua, "rd_kafka_new failed: %s", errstr); - } - - if (rd_kafka_brokers_add(kc->rk, brokerlist) == 0) { - return luaL_error(lua, "invalid broker list"); - } - - rd_kafka_poll_set_consumer(kc->rk); - if (!add_consumer_topics(lua, kc, topic_cnt)) { - return lua_error(lua); - } - return 1; -} - - -static int consumer_receive(lua_State *lua) -{ - bool err = false; - heka_kafka_consumer *kc = check_consumer(lua, 1); - rd_kafka_message_t *rkmessage = rd_kafka_consumer_poll(kc->rk, 1000); - if (rkmessage) { - if (rkmessage->err) { - if (rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION || - rkmessage->err == RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC) { - if (rkmessage->rkt) { - err = true; - lua_pushfstring(lua, "topic: %s partition: %d offset: %g err: %s", - rd_kafka_topic_name(rkmessage->rkt), - (int)rkmessage->partition, - (double)rkmessage->offset, - rd_kafka_message_errstr(rkmessage)); - } else { - err = true; - lua_pushfstring(lua, "%s err: %s", rd_kafka_err2str(rkmessage->err), - rd_kafka_message_errstr(rkmessage)); - } - } else { - lua_pushnil(lua); - lua_pushnil(lua); - lua_pushnil(lua); - lua_pushnil(lua); - } - } else { - lua_pushlstring(lua, rkmessage->payload, rkmessage->len); - lua_pushstring(lua, rd_kafka_topic_name(rkmessage->rkt)); - lua_pushinteger(lua, (lua_Integer)rkmessage->partition); - if (rkmessage->key_len) { - lua_pushlstring(lua, rkmessage->key, rkmessage->key_len); - } else { - lua_pushnil(lua); - } - } - rd_kafka_message_destroy(rkmessage); - } else { - lua_pushnil(lua); - lua_pushnil(lua); - lua_pushnil(lua); - lua_pushnil(lua); - } - if (err) return lua_error(lua); - return 4; -} - - -static int consumer_gc(lua_State *lua) -{ - heka_kafka_consumer *kc = check_consumer(lua, 1); - if (kc->rk) rd_kafka_consumer_close(kc->rk); - if (kc->topics) rd_kafka_topic_partition_list_destroy(kc->topics); - if (kc->rk) rd_kafka_destroy(kc->rk); - rd_kafka_wait_destroyed(1000); - return 0; -} - -static const struct luaL_reg consumerlib_f[] = -{ - { "new", consumer_new }, - { NULL, NULL } -}; - - -static const struct luaL_reg consumerlib_m[] = -{ - { "receive", consumer_receive }, - { "__gc", consumer_gc }, - { NULL, NULL } -}; - - -int luaopen_heka_kafka_consumer(lua_State *lua) -{ - luaL_newmetatable(lua, mozsvc_heka_kafka_consumer); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, consumerlib_m); - luaL_register(lua, mozsvc_heka_kafka_consumer_table, consumerlib_f); - return 1; -} diff --git a/src/heka/kafka_impl.h b/src/heka/kafka_impl.h deleted file mode 100644 index be65866..0000000 --- a/src/heka/kafka_impl.h +++ /dev/null @@ -1,20 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Lua Kafka constants/types @file */ - -#ifndef luasandbox_heka_kafka_impl_h_ -#define luasandbox_heka_kafka_impl_h_ - -#include "luasandbox/lua.h" - -extern const char* mozsvc_heka_kafka_producer_table; -extern const char* mozsvc_heka_kafka_consumer_table; - -int luaopen_heka_kafka_producer(lua_State *lua); -int luaopen_heka_kafka_consumer(lua_State *lua); - -#endif diff --git a/src/heka/message.c b/src/heka/message.c index abd452c..d5e24b3 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -830,10 +830,10 @@ int heka_encode_message(lua_State *lua) return luaL_argerror(lua, n, "incorrect number of arguments"); } - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "encode_message() invalid upvalueindex"); - } + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "encode_message() invalid " LSB_THIS_PTR); lsb->output.pos = 0; lsb_err_value ret = heka_encode_message_table(lsb, 1); diff --git a/src/heka/rapidjson.cpp b/src/heka/rapidjson.cpp deleted file mode 100644 index b202a83..0000000 --- a/src/heka/rapidjson.cpp +++ /dev/null @@ -1,721 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua RapidJSON wrapper implementation @file */ - -#include "rapidjson_impl.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "luasandbox_output.h" -#include "sandbox_impl.h" -#include "stream_reader_impl.h" - -namespace rj = rapidjson; - -typedef struct heka_json -{ - rj::Document *doc; - rj::Value *val; - char *insitu; - std::set *refs; -} heka_json; - - -typedef struct heka_schema -{ - rj::SchemaDocument *doc; -} heka_schema; - - -typedef struct heka_object_iter -{ - rj::Value::MemberIterator *it; - rj::Value::MemberIterator *end; -} heka_object_iterator; - - -const char *mozsvc_heka_json = "mozsvc.heka_json"; -const char *mozsvc_heka_json_table = "heka_json"; - -static const char *mozsvc_heka_schema = "mozsvc.heka_schema"; -static const char *mozsvc_heka_object_iter = "mozsvc.heka_object_iter"; - - -static rj::Value* check_value(lua_State *lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n >= 1 && n <= 2, 0, "invalid number of arguments"); - heka_json *hj = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_json)); - rj::Value *v = static_cast(lua_touserdata(lua, 2)); - if (!v) { - int t = lua_type(lua, 2); - if (t == LUA_TNONE) { - v = hj->doc ? hj->doc : hj->val; - } else if (t == LUA_TNIL) { - v = NULL; - } else { - luaL_checktype(lua, 2, LUA_TLIGHTUSERDATA); - } - } else if (hj->refs->find(v) == hj->refs->end()) { - luaL_error(lua, "invalid value"); - } - return v; -} - - -static int schema_gc(lua_State *lua) -{ - heka_schema *hs = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_schema)); - delete(hs->doc); - return 0; -} - - -static int iter_gc(lua_State *lua) -{ - heka_object_iter *hoi = static_cast(lua_touserdata(lua, 1)); - delete(hoi->it); - delete(hoi->end); - return 0; -} - - -static int hj_gc(lua_State *lua) -{ - heka_json *hj = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_json)); - delete(hj->refs); - delete(hj->doc); - delete(hj->val); - free(hj->insitu); - return 0; -} - - -static int hj_parse_schema(lua_State *lua) -{ - const char *json = luaL_checkstring(lua, 1); - heka_schema *hs = static_cast(lua_newuserdata(lua, sizeof*hs)); - hs->doc = NULL; - luaL_getmetatable(lua, mozsvc_heka_schema); - lua_setmetatable(lua, -2); - - { // allows doc to be destroyed before the longjmp - rj::Document doc; - if (doc.Parse(json).HasParseError()) { - lua_pushfstring(lua, "failed to parse offset:%f %s", - (lua_Number)doc.GetErrorOffset(), - rj::GetParseError_En(doc.GetParseError())); - } else { - hs->doc = new rj::SchemaDocument(doc); - if (!hs->doc) { - lua_pushstring(lua, "memory allocation failed"); - } - } - } - if (!hs->doc) return lua_error(lua); - return 1; -} - - -static int hj_parse(lua_State *lua) -{ - const char *json = luaL_checkstring(lua, 1); - heka_json *hj = static_cast(lua_newuserdata(lua, sizeof*hj)); - hj->doc = new rj::Document; - hj->val = NULL; - hj->insitu = NULL; - hj->refs = new std::set; - luaL_getmetatable(lua, mozsvc_heka_json); - lua_setmetatable(lua, -2); - - if (!hj->doc || !hj->refs) { - lua_pushstring(lua, "memory allocation failed"); - return lua_error(lua); - } else { - if (hj->doc->Parse(json).HasParseError()) { - lua_pushfstring(lua, "failed to parse offset:%f %s", - (lua_Number)hj->doc->GetErrorOffset(), - rj::GetParseError_En(hj->doc->GetParseError())); - return lua_error(lua); - } - } - hj->refs->insert(hj->doc); - return 1; -} - - -static int hj_validate(lua_State *lua) -{ - heka_json *hj = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_json)); - heka_schema *hs = static_cast - (luaL_checkudata(lua, 2, mozsvc_heka_schema)); - - rj::SchemaValidator validator(*hs->doc); - rj::Value *v = hj->doc ? hj->doc : hj->val; - if (!v->Accept(validator)) { - lua_pushboolean(lua, false); - luaL_Buffer b; - luaL_buffinit(lua, &b); - rj::StringBuffer sb; - validator.GetInvalidSchemaPointer().StringifyUriFragment(sb); - luaL_addstring(&b, "SchemaURI: "); - luaL_addstring(&b, sb.GetString()); - luaL_addstring(&b, " Keyword: "); - luaL_addstring(&b, validator.GetInvalidSchemaKeyword()); - sb.Clear(); - validator.GetInvalidDocumentPointer().StringifyUriFragment(sb); - luaL_addstring(&b, " DocumentURI: "); - luaL_addstring(&b, sb.GetString()); - luaL_pushresult(&b); - } - return 2; // ok, err -} - - -static int hj_find(lua_State *lua) -{ - heka_json *hj = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_json)); - - int start = 3; - rj::Value *v = static_cast(lua_touserdata(lua, 2)); - if (!v) { - v = hj->doc ? hj->doc : hj->val; - start = 2; - } else if (hj->refs->find(v) == hj->refs->end()) { - return luaL_error(lua, "invalid value"); - } - - int n = lua_gettop(lua); - for (int i = start; i <= n; ++i) { - switch (lua_type(lua, i)) { - case LUA_TSTRING: - { - if (!v->IsObject()) { - lua_pushnil(lua); - return 1; - } - rj::Value::MemberIterator itr = v->FindMember(lua_tostring(lua, i)); - if (itr == v->MemberEnd()) { - lua_pushnil(lua); - return 1; - } - v = &itr->value; - } - break; - case LUA_TNUMBER: - { - if (!v->IsArray()) { - lua_pushnil(lua); - return 1; - } - rj::SizeType idx = static_cast(lua_tonumber(lua, i)); - if (idx >= v->Size()) { - lua_pushnil(lua); - return 1; - } - v = &(*v)[idx]; - } - break; - default: - lua_pushnil(lua); - return 1; - } - } - hj->refs->insert(v); - lua_pushlightuserdata(lua, v); - return 1; -} - - -static int hj_type(lua_State *lua) -{ - rj::Value *v = check_value(lua); - if (!v) { - lua_pushnil(lua); - return 1; - } - - switch (v->GetType()) { - case rj::kStringType: - lua_pushstring(lua, "string"); - break; - case rj::kNumberType: - lua_pushstring(lua, "number"); - break; - case rj::kFalseType: - case rj::kTrueType: - lua_pushstring(lua, "boolean"); - break; - case rj::kObjectType: - lua_pushstring(lua, "object"); - break; - case rj::kArrayType: - lua_pushstring(lua, "array"); - break; - case rj::kNullType: - lua_pushstring(lua, "null"); - break; - } - return 1; -} - - -static int hj_size(lua_State *lua) -{ - rj::Value *v = check_value(lua); - if (!v) { - lua_pushnil(lua); - return 1; - } - - switch (v->GetType()) { - case rj::kStringType: - lua_pushnumber(lua, (lua_Number)v->GetStringLength()); - break; - case rj::kNumberType: - return luaL_error(lua, "attempt to get length of a number"); - case rj::kFalseType: - case rj::kTrueType: - return luaL_error(lua, "attempt to get length of a boolean"); - case rj::kObjectType: - lua_pushnumber(lua, (lua_Number)v->MemberCount()); - break; - case rj::kArrayType: - lua_pushnumber(lua, (lua_Number)v->Size()); - break; - case rj::kNullType: - return luaL_error(lua, "attempt to get length of a NULL"); - } - return 1; -} - - -static int hj_make_field(lua_State *lua) -{ - rj::Value *v = check_value(lua); - if (!v) { - lua_pushnil(lua); - return 1; - } - - lua_createtable(lua, 0, 2); - lua_pushlightuserdata(lua, (void *)v); - lua_setfield(lua, -2, "value"); - lua_pushvalue(lua, 1); - lua_setfield(lua, -2, "userdata"); - return 1; -} - - -static int hj_object_iter(lua_State *lua) -{ - heka_object_iter *hoi = static_cast - (lua_touserdata(lua, lua_upvalueindex(1))); - rj::Value *v = (rj::Value *)lua_touserdata(lua, lua_upvalueindex(2)); - heka_json *hj = (heka_json *)lua_touserdata(lua, lua_upvalueindex(3)); - - if (hj->refs->find(v) == hj->refs->end()) { - return luaL_error(lua, "iterator has been invalidated"); - } - - if (*hoi->it != *hoi->end) { - rj::Value *next = &(*hoi->it)->value; - hj->refs->insert(next); - lua_pushlstring(lua, (*hoi->it)->name.GetString(), - (size_t)(*hoi->it)->name.GetStringLength()); - lua_pushlightuserdata(lua, next); - ++*hoi->it; - } else { - lua_pushnil(lua); - lua_pushnil(lua); - } - return 2; -} - - -static int hj_array_iter(lua_State *lua) -{ - rj::SizeType it = (rj::SizeType)lua_tonumber(lua, lua_upvalueindex(1)); - rj::SizeType end = (rj::SizeType)lua_tonumber(lua, lua_upvalueindex(2)); - rj::Value *v = (rj::Value *)lua_touserdata(lua, lua_upvalueindex(3)); - heka_json *hj = (heka_json *)lua_touserdata(lua, lua_upvalueindex(4)); - - if (hj->refs->find(v) == hj->refs->end()) { - return luaL_error(lua, "iterator has been invalidated"); - } - - if (it != end) { - rj::Value *next = &(*v)[it]; - hj->refs->insert(next); - lua_pushnumber(lua, (lua_Number)it); - lua_pushlightuserdata(lua, next); - - ++it; - lua_pushnumber(lua, (lua_Number)it); - lua_replace(lua, lua_upvalueindex(1)); - } else { - lua_pushnil(lua); - lua_pushnil(lua); - } - return 2; -} - - -static int hj_value(lua_State *lua) -{ - rj::Value *v = check_value(lua); - if (!v) { - lua_pushnil(lua); - return 1; - } - - switch (v->GetType()) { - case rj::kStringType: - lua_pushlstring(lua, v->GetString(), (size_t)v->GetStringLength()); - break; - case rj::kNumberType: - lua_pushnumber(lua, (lua_Number)v->GetDouble()); - break; - case rj::kFalseType: - case rj::kTrueType: - lua_pushboolean(lua, v->GetBool()); - break; - case rj::kObjectType: - return luaL_error(lua, "value() not allowed on an object"); - break; - case rj::kArrayType: - return luaL_error(lua, "value() not allowed on an array"); - break; - default: - lua_pushnil(lua); - break; - } - return 1; -} - - -static int hj_iter(lua_State *lua) -{ - rj::Value *v = check_value(lua); - if (!v) { - lua_pushnil(lua); - return 1; - } - - switch (v->GetType()) { - case rj::kObjectType: - { - heka_object_iter *hoi = static_cast - (lua_newuserdata(lua, sizeof*hoi)); - hoi->it = new rj::Value::MemberIterator; - hoi->end = new rj::Value::MemberIterator; - luaL_getmetatable(lua, mozsvc_heka_object_iter); - lua_setmetatable(lua, -2); - if (!hoi->it || !hoi->end) { - return luaL_error(lua, "memary allocation failure"); - } - *hoi->it = v->MemberBegin(); - *hoi->end = v->MemberEnd(); - lua_pushlightuserdata(lua, (void *)v); - lua_pushvalue(lua, 1); - lua_pushcclosure(lua, hj_object_iter, 3); - } - break; - case rj::kArrayType: - { - lua_pushnumber(lua, 0); - lua_pushnumber(lua, (lua_Number)v->Size()); - lua_pushlightuserdata(lua, (void *)v); - lua_pushvalue(lua, 1); - lua_pushcclosure(lua, hj_array_iter, 4); - } - break; - default: - return luaL_error(lua, "iter() not allowed on a primitive type"); - break; - } - return 1; -} - - -lsb_const_string read_message(lua_State *lua, int idx, lsb_heka_message *m) -{ - lsb_const_string ret = { NULL, 0 }; - size_t field_len; - const char *field = luaL_checklstring(lua, idx, &field_len); - int fi = (int)luaL_optinteger(lua, idx + 1, 0); - luaL_argcheck(lua, fi >= 0, idx + 1, "field index must be >= 0"); - int ai = (int)luaL_optinteger(lua, idx + 2, 0); - luaL_argcheck(lua, ai >= 0, idx + 2, "array index must be >= 0"); - - if (strcmp(field, LSB_PAYLOAD) == 0) { - if (m->payload.s) ret = m->payload; - } else { - if (field_len >= 8 - && memcmp(field, LSB_FIELDS "[", 7) == 0 - && field[field_len - 1] == ']') { - lsb_read_value v; - lsb_const_string f = { field + 7, field_len - 8 }; - lsb_read_heka_field(m, &f, fi, ai, &v); - if (v.type == LSB_READ_STRING) ret = v.u.s; - } - } - return ret; -} - - -static int hj_parse_message(lua_State *lua) -{ - lsb_lua_sandbox *lsb = static_cast - (lua_touserdata(lua, lua_upvalueindex(1))); - if (!lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __func__); - } - int n = lua_gettop(lua); - int idx = 1; - - lsb_heka_message *msg = NULL; - lsb_heka_sandbox *hsb = static_cast(lsb_get_parent(lsb)); - if (hsb->type == 'i') { - luaL_argcheck(lua, n >= 2 && n <= 4, 0, "invalid number of arguments"); - heka_stream_reader *hsr = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_stream_reader)); - msg = &hsr->msg; - idx = 2; - } else { - luaL_argcheck(lua, n >= 1 && n <= 3, 0, "invalid number of arguments"); - if (!hsb->msg || !hsb->msg->raw.s) { - return luaL_error(lua, "no active message"); - } - msg = hsb->msg; - } - - lsb_const_string json = read_message(lua, idx, msg); - if (!json.s) return luaL_error(lua, "field not found"); - - char *inflated = NULL; -#ifdef HAVE_ZLIB - // automatically handle gzipped strings (optimization for Mozilla telemetry - // messages) - if (json.len > 2) { - if (json.s[0] == 0x1f && (unsigned char)json.s[1] == 0x8b) { - size_t mms = (size_t)lua_tointeger(lua, lua_upvalueindex(2)); - inflated = lsb_ungzip(json.s, json.len, mms, NULL); - if (!inflated) return luaL_error(lua, "lsb_ungzip failed"); - } - } -#endif - - heka_json *hj = static_cast(lua_newuserdata(lua, sizeof*hj)); - hj->doc = new rj::Document; - hj->val = NULL; - hj->insitu = inflated; - hj->refs = new std::set; - luaL_getmetatable(lua, mozsvc_heka_json); - lua_setmetatable(lua, -2); - - if (!hj->doc || !hj->refs) { - lua_pushstring(lua, "memory allocation failed"); - return lua_error(lua); - } - - bool err = false; - if (hj->insitu) { - if (hj->doc->ParseInsitu(hj->insitu) - .HasParseError()) { - err = true; - lua_pushfstring(lua, "failed to parse offset:%f %s", - (lua_Number)hj->doc->GetErrorOffset(), - rj::GetParseError_En(hj->doc->GetParseError())); - } - } else { - rj::MemoryStream ms(json.s, json.len); - if (hj->doc->ParseStream<0, rj::UTF8<> >(ms).HasParseError()) { - err = true; - lua_pushfstring(lua, "failed to parse offset:%f %s", - (lua_Number)hj->doc->GetErrorOffset(), - rj::GetParseError_En(hj->doc->GetParseError())); - } - } - - if (err) return lua_error(lua); - hj->refs->insert(hj->doc); - return 1; -} - - -class OutputBufferWrapper { -public: - typedef char Ch; - OutputBufferWrapper(lsb_output_buffer *ob) : ob_(ob), err_(NULL) { } -#if _BullseyeCoverage -#pragma BullseyeCoverage off -#endif - Ch Peek() const { assert(false); return '\0'; } - Ch Take() { assert(false); return '\0'; } - size_t Tell() const { return 0; } - Ch* PutBegin() { assert(false); return 0; } - size_t PutEnd(Ch *) { assert(false); return 0; } -#if _BullseyeCoverage -#pragma BullseyeCoverage on -#endif - void Put(Ch c) - { - const char *err = lsb_outputc(ob_, c); - if (err) err_ = err; - } - void Flush() { return; } - const char* GetError() { return err_; } -private: - OutputBufferWrapper(const OutputBufferWrapper&); - OutputBufferWrapper& operator=(const OutputBufferWrapper&); - lsb_output_buffer *ob_; - const char *err_; -}; - - -static int output_heka_json(lua_State *lua) -{ - lsb_output_buffer *ob = static_cast - (lua_touserdata(lua, -1)); - heka_json *hj = static_cast(lua_touserdata(lua, -2)); - rj::Value *v = static_cast(lua_touserdata(lua, -3)); - if (!(ob && hj)) { - return 1; - } - if (!v) { - v = hj->doc ? hj->doc : hj->val; - } else { - if (hj->refs->find(v) == hj->refs->end()) { - return 1; - } - } - OutputBufferWrapper obw(ob); - rapidjson::Writer writer(obw); - v->Accept(writer); - return obw.GetError() == NULL ? 0 : 1; -} - - -static int hj_remove(lua_State *lua) -{ - hj_find(lua); - rj::Value *v = static_cast(lua_touserdata(lua, -1)); - if (!v) { - lua_pushnil(lua); - return 1; - } - - heka_json *hj = static_cast - (luaL_checkudata(lua, 1, mozsvc_heka_json)); - - heka_json *nv = static_cast(lua_newuserdata(lua, sizeof*nv)); - nv->doc = NULL; - nv->val = new rj::Value; - nv->insitu = NULL; - nv->refs = new std::set; - luaL_getmetatable(lua, mozsvc_heka_json); - lua_setmetatable(lua, -2); - - if (!nv->val || !nv->refs) { - lua_pushstring(lua, "memory allocation failed"); - return lua_error(lua); - } - - *nv->val = *v; // move the value out replacing the original with NULL - hj->refs->erase(v); - nv->refs->insert(nv->val); - return 1; -} - - -static const struct luaL_reg schemalib_m[] = -{ - { "__gc", schema_gc }, - { NULL, NULL } -}; - - -static const struct luaL_reg iterlib_m[] = -{ - { "__gc", iter_gc }, - { NULL, NULL } -}; - - -static const struct luaL_reg hjlib_f[] = -{ - { "parse_schema", hj_parse_schema }, - { "parse", hj_parse }, - { NULL, NULL } -}; - - -static const struct luaL_reg hjlib_m[] = -{ - { "validate", hj_validate }, - { "type", hj_type }, - { "find", hj_find }, - { "value", hj_value }, - { "iter", hj_iter }, - { "size", hj_size }, - { "make_field", hj_make_field }, - { "remove", hj_remove }, - { "__gc", hj_gc }, - { NULL, NULL } -}; - - -int luaopen_heka_json(lua_State *lua) -{ - lua_newtable(lua); - lsb_add_output_function(lua, output_heka_json); - lua_replace(lua, LUA_ENVIRONINDEX); - - luaL_newmetatable(lua, mozsvc_heka_schema); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, schemalib_m); - lua_pop(lua, 1); - - luaL_newmetatable(lua, mozsvc_heka_object_iter); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, iterlib_m); - lua_pop(lua, 1); - - luaL_newmetatable(lua, mozsvc_heka_json); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, hjlib_m); - luaL_register(lua, mozsvc_heka_json_table, hjlib_f); - // special case parse_message since it needs access to the sandbox - lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); - if (lua_type(lua, -1) != LUA_TTABLE) { - return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); - } - lsb_lua_sandbox *lsb = static_cast - (lua_touserdata(lua, lua_upvalueindex(1))); - lua_pushlightuserdata(lua, static_cast(lsb)); - lua_getfield(lua, -2, LSB_HEKA_MAX_MESSAGE_SIZE); - lua_pushcclosure(lua, hj_parse_message, 2); - lua_setfield(lua, -3, "parse_message"); - lua_pop(lua, 1); // remove LSB_CONFIG_TABLE - return 1; -} diff --git a/src/heka/rapidjson_impl.h b/src/heka/rapidjson_impl.h deleted file mode 100644 index 3390d3e..0000000 --- a/src/heka/rapidjson_impl.h +++ /dev/null @@ -1,36 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Hindsight Heka JSON structures @file */ - -#ifndef luasandbox_heka_rapidjson_impl_h_ -#define luasandbox_heka_rapidjson_impl_h_ - -#ifdef __cplusplus -extern "C" -{ -#endif - -#include "luasandbox/lauxlib.h" -#include "luasandbox/lua.h" -#include "luasandbox/util/heka_message.h" - -#ifdef _MSC_VER -#ifndef __func__ -#define __func__ __FUNCTION__ -#endif -#endif - -extern const char *mozsvc_heka_json; -extern const char *mozsvc_heka_json_table; - -int luaopen_heka_json(lua_State *lua); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 934325a..86112c8 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -13,18 +13,16 @@ #include #include "../luasandbox_defines.h" +#include "../luasandbox_impl.h" #include "luasandbox.h" +#include "luasandbox/lualib.h" +#include "luasandbox/heka/stream_reader.h" #include "luasandbox/util/heka_message_matcher.h" #include "luasandbox/util/protobuf.h" #include "luasandbox/util/running_stats.h" #include "luasandbox_output.h" #include "message_impl.h" -#include "rapidjson_impl.h" #include "sandbox_impl.h" -#include "stream_reader_impl.h" -#ifdef HAVE_KAFKA -#include "kafka_impl.h" -#endif #ifdef _WIN32 #include @@ -36,15 +34,18 @@ lsb_err_id LSB_ERR_HEKA_INPUT = "invalid input"; static const char *pm_func_name = "process_message"; static const char *im_func_name = "inject_message"; -static const char *mozsvc_heka_message_matcher = "mozsvc.heka_message_matcher"; +static const char *lsb_heka_message_matcher = "lsb.heka_message_matcher"; + +int heka_create_stream_reader(lua_State *lua); static int read_message(lua_State *lua) { - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __func__); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); } - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); if (!hsb->msg || !hsb->msg->raw.s) { lua_pushnil(lua); @@ -57,7 +58,7 @@ static int read_message(lua_State *lua) static lsb_message_matcher* mm_check(lua_State *lua) { lsb_message_matcher **ppmm = luaL_checkudata(lua, 1, - mozsvc_heka_message_matcher); + lsb_heka_message_matcher); return *ppmm; } @@ -73,11 +74,12 @@ static int mm_gc(lua_State *lua) static int mm_eval(lua_State *lua) { lsb_message_matcher *mm = mm_check(lua); - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", __func__); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); } - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); if (!hsb->msg || !hsb->msg->raw.s) { return luaL_error(lua, "no active message"); } @@ -89,19 +91,13 @@ static int mm_eval(lua_State *lua) static int mm_create(lua_State *lua) { const char *exp = luaL_checkstring(lua, 1); - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == lsb) { - return luaL_error(lua, "%() invalid lightuserdata", __func__); - } - lsb_message_matcher **ppmm = lua_newuserdata(lua, sizeof*ppmm); - if (luaL_newmetatable(lua, mozsvc_heka_message_matcher) == 1) { + if (luaL_newmetatable(lua, lsb_heka_message_matcher) == 1) { lua_pushvalue(lua, -1); lua_setfield(lua, -2, "__index"); - lua_pushcclosure(lua, mm_gc, 0); + lua_pushcfunction(lua, mm_gc); lua_setfield(lua, -2, "__gc"); - lua_pushlightuserdata(lua, lsb); - lua_pushcclosure(lua, mm_eval, 1); + lua_pushcfunction(lua, mm_eval); lua_setfield(lua, -2, "eval"); } lua_setmetatable(lua, -2); @@ -115,10 +111,10 @@ static int mm_create(lua_State *lua) static int inject_message_input(lua_State *lua) { - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", im_func_name); - } + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); const char *scp = NULL; double ncp = NAN; @@ -144,7 +140,7 @@ static int inject_message_input(lua_State *lua) case LUA_TUSERDATA: { heka_stream_reader *hsr = luaL_checkudata(lua, 1, - mozsvc_heka_stream_reader); + LSB_HEKA_STREAM_READER); if (hsr->msg.raw.s) { output.len = hsr->msg.raw.len; output.s = hsr->msg.raw.s; @@ -195,10 +191,10 @@ static int inject_message_input(lua_State *lua) static int inject_message_analysis(lua_State *lua) { luaL_checktype(lua, 1, LUA_TTABLE); - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == lsb) { - return luaL_error(lua, "%s() invalid lightuserdata", im_func_name); - } + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); if (heka_encode_message_table(lsb, 1)) { return luaL_error(lua, "%s() failed: %s", im_func_name, lsb_get_error(lsb)); @@ -221,10 +217,10 @@ static int inject_payload(lua_State *lua) { static const char *default_type = "txt"; - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "%s invalid lightuserdata", __func__); - } + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, __func__); int n = lua_gettop(lua); @@ -278,13 +274,13 @@ static int inject_payload(lua_State *lua) static int update_checkpoint(lua_State *lua) { - static const char *func_name = LSB_HEKA_UPDATE_CHECKPOINT; - - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "%s() invalid upvalueindex", func_name); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); } - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + int result = 0; int n = lua_gettop(lua); switch (n) { @@ -300,10 +296,10 @@ static int update_checkpoint(lua_State *lua) result = hsb->cb.ucp(hsb->parent, NULL); break; default: - return luaL_error(lua, "%s() invalid number of args: %d", func_name, n); + return luaL_error(lua, "%s() invalid number of args: %d", __func__, n); } if (result) { - return luaL_error(lua, "%s() failed: rejected by the callback", func_name); + return luaL_error(lua, "%s() failed: rejected by the callback", __func__); } return result; } @@ -328,23 +324,24 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) const char **list; size_t list_size; + lua_pushlightuserdata(lua, hsb); + lua_setfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); switch (hsb->type) { case 'i': case 'o': - list_size = sizeof io / sizeof*io; + list_size = sizeof(io) / sizeof(*io); list = io; break; default: - list_size = sizeof analysis / sizeof*analysis; + list_size = sizeof(analysis) / sizeof(*analysis); list = analysis; lua_newtable(lua); lua_pushboolean(lua, true); - lua_setfield(lua, -2, "io"); + lua_setfield(lua, -2, LUA_IOLIBNAME); lua_pushboolean(lua, true); - lua_setfield(lua, -2, "coroutine"); - lua_pushboolean(lua, true); - lua_setfield(lua, -2, "lsb_compression"); + lua_setfield(lua, -2, LUA_COLIBNAME); lua_setfield(lua, 1, "disable_modules"); break; } @@ -457,25 +454,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, lsb_add_function(hsb->lsb, inject_message_input, "inject_message"); // inject_payload is intentionally excluded from input plugins // you can construct whatever you need with inject_message - -// preload the Heka stream reader module - luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); - lua_pushstring(lua, mozsvc_heka_stream_reader_table); - lua_pushcfunction(lua, luaopen_heka_stream_reader); - lua_rawset(lua, -3); -// preload Heka JSON with a pointer back to the sandbox - lua_pushstring(lua, mozsvc_heka_json_table); - lua_pushlightuserdata(lua, (void *)hsb->lsb); - lua_pushcclosure(lua, luaopen_heka_json, 1); - lua_rawset(lua, -3); -#ifdef HAVE_KAFKA -// preload the Heka Kafka consumer - lua_pushstring(lua, mozsvc_heka_kafka_consumer_table); - lua_pushcfunction(lua, luaopen_heka_kafka_consumer); - lua_rawset(lua, -3); -#endif - - lua_pop(lua, 1); // remove the preloaded table + lsb_add_function(hsb->lsb, heka_create_stream_reader, "create_stream_reader"); if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { @@ -759,21 +738,6 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); lsb_add_function(hsb->lsb, update_checkpoint, LSB_HEKA_UPDATE_CHECKPOINT); lsb_add_function(hsb->lsb, mm_create, "create_message_matcher"); -// preload Heka JSON with a pointer back to the sandbox - luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); - lua_pushstring(lua, mozsvc_heka_json_table); - lua_pushlightuserdata(lua, (void *)hsb->lsb); - lua_pushcclosure(lua, luaopen_heka_json, 1); - lua_rawset(lua, -3); -#ifdef HAVE_KAFKA -// preload the Heka Kafka producer - lua_pushstring(lua, mozsvc_heka_kafka_producer_table); - lua_pushlightuserdata(lua, (void *)hsb->lsb); - lua_pushcclosure(lua, luaopen_heka_kafka_producer, 1); - lua_rawset(lua, -3); -#endif - - lua_pop(lua, 1); // remove the preloaded table if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { @@ -925,3 +889,17 @@ bool lsb_heka_is_running(lsb_heka_sandbox *hsb) if (lsb_get_state(hsb->lsb) == LSB_RUNNING) return true; return false; } + + +const lsb_heka_message* lsb_heka_get_message(lsb_heka_sandbox *hsb) +{ + if (!hsb) return NULL; + return hsb->msg; +} + + +char lsb_heka_get_type(lsb_heka_sandbox *hsb) +{ + if (!hsb) return '\0'; + return hsb->type; +} diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c index 4ed61ae..f6ee2a4 100644 --- a/src/heka/stream_reader.c +++ b/src/heka/stream_reader.c @@ -6,7 +6,7 @@ /** @brief Hindsight Heka stream reader implementation @file */ -#include "stream_reader_impl.h" +#include "luasandbox/heka/stream_reader.h" #include #include @@ -15,59 +15,9 @@ #include "message_impl.h" #include "luasandbox/lauxlib.h" -const char *mozsvc_heka_stream_reader = "mozsvc.heka_stream_reader"; -const char *mozsvc_heka_stream_reader_table = "heka_stream_reader"; - -static int hsr_new(lua_State *lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n == 1, 0, "incorrect number of arguments"); - size_t len; - const char *name = luaL_checklstring(lua, 1, &len); - luaL_argcheck(lua, len < 255, 1, "name is too long"); - - size_t nbytes = sizeof(heka_stream_reader); - heka_stream_reader *hsr = lua_newuserdata(lua, nbytes); - - size_t mms = 0; - lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); - if (lua_type(lua, -1) == LUA_TTABLE) { - lua_getfield(lua, -1, LSB_HEKA_MAX_MESSAGE_SIZE); - mms = (size_t)lua_tointeger(lua, -1); - lua_pop(lua, 1); // remove limit - } else { - free(hsr); - return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); - } - lua_pop(lua, 1); // remove config - - if (lsb_init_heka_message(&hsr->msg, 8)) { - free(hsr); - return luaL_error(lua, "failed to init the message struct"); - } - if (lsb_init_input_buffer(&hsr->buf, mms)) { - lsb_free_heka_message(&hsr->msg); - free(hsr); - return luaL_error(lua, "failed to init the input buffer"); - } - hsr->name = malloc(len + 1); - if (!hsr->name) { - lsb_free_input_buffer(&hsr->buf); - lsb_free_heka_message(&hsr->msg); - free(hsr); - return luaL_error(lua, "memory allocation failed"); - } - strcpy(hsr->name, name); - - luaL_getmetatable(lua, mozsvc_heka_stream_reader); - lua_setmetatable(lua, -2); - return 1; -} - - static heka_stream_reader* check_hsr(lua_State *lua, int args) { - heka_stream_reader *hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); + heka_stream_reader *hsr = luaL_checkudata(lua, 1, LSB_HEKA_STREAM_READER); luaL_argcheck(lua, args == lua_gettop(lua), 0, "incorrect number of arguments"); return hsr; @@ -107,7 +57,7 @@ static int hsr_find_message(lua_State *lua) { int n = lua_gettop(lua); luaL_argcheck(lua, n > 1 && n < 4, 0, "incorrect number of arguments"); - heka_stream_reader *hsr = luaL_checkudata(lua, 1, mozsvc_heka_stream_reader); + heka_stream_reader *hsr = luaL_checkudata(lua, 1, LSB_HEKA_STREAM_READER); lsb_input_buffer *b = &hsr->buf; FILE *fh = NULL; @@ -211,13 +161,6 @@ static int hsr_gc(lua_State *lua) } -static const struct luaL_reg heka_stream_readerlib_f[] = -{ - { "new", hsr_new }, - { NULL, NULL } -}; - - static const struct luaL_reg heka_stream_readerlib_m[] = { { "find_message", hsr_find_message }, @@ -228,12 +171,52 @@ static const struct luaL_reg heka_stream_readerlib_m[] = }; -int luaopen_heka_stream_reader(lua_State *lua) +int heka_create_stream_reader(lua_State *lua) { - luaL_newmetatable(lua, mozsvc_heka_stream_reader); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, heka_stream_readerlib_m); - luaL_register(lua, mozsvc_heka_stream_reader_table, heka_stream_readerlib_f); + int n = lua_gettop(lua); + luaL_argcheck(lua, n == 1, 0, "incorrect number of arguments"); + size_t len; + const char *name = luaL_checklstring(lua, 1, &len); + luaL_argcheck(lua, len < 255, 1, "name is too long"); + + size_t nbytes = sizeof(heka_stream_reader); + heka_stream_reader *hsr = lua_newuserdata(lua, nbytes); + + if (luaL_newmetatable(lua, LSB_HEKA_STREAM_READER) == 1) { + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, heka_stream_readerlib_m); + } + lua_setmetatable(lua, -2); + + size_t mms = 0; + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + lua_getfield(lua, -1, LSB_HEKA_MAX_MESSAGE_SIZE); + mms = (size_t)lua_tointeger(lua, -1); + lua_pop(lua, 1); // remove limit + } else { + free(hsr); + return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); + } + lua_pop(lua, 1); // remove config + + if (lsb_init_heka_message(&hsr->msg, 8)) { + free(hsr); + return luaL_error(lua, "failed to init the message struct"); + } + if (lsb_init_input_buffer(&hsr->buf, mms)) { + lsb_free_heka_message(&hsr->msg); + free(hsr); + return luaL_error(lua, "failed to init the input buffer"); + } + hsr->name = malloc(len + 1); + if (!hsr->name) { + lsb_free_input_buffer(&hsr->buf); + lsb_free_heka_message(&hsr->msg); + free(hsr); + return luaL_error(lua, "memory allocation failed"); + } + strcpy(hsr->name, name); return 1; } diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt index 3a2cd5c..f0afa42 100644 --- a/src/heka/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -8,16 +8,10 @@ set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src/util; add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -add_executable(test_heka_sandbox test_sandbox.c) -target_link_libraries(test_heka_sandbox luasandboxheka luasandbox) +add_executable(test_heka_sandbox test_heka_sandbox.c) +target_link_libraries(test_heka_sandbox luasandboxheka) add_test(NAME test_heka_sandbox COMMAND test_heka_sandbox) -if(LIBRTKAFKA_LIBRARY) - add_executable(test_kafka test_kafka.c) - target_link_libraries(test_kafka luasandboxheka luasandbox) - add_test(NAME test_kafka COMMAND test_kafka CONFIGURATIONS integration) -endif() - if(WIN32) STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua index 990e37c..2e4933c 100644 --- a/src/heka/test/lua/encode_message.lua +++ b/src/heka/test/lua/encode_message.lua @@ -5,10 +5,10 @@ require "string" require "table" require "circular_buffer" -require "heka_json" +require "rjson" local cb = circular_buffer.new(2,1,1) -local doc = heka_json.parse([[{"foo":"bar"}]]) +local doc = rjson.parse([[{"foo":"bar"}]]) assert(doc) local msgs = { @@ -114,5 +114,5 @@ ok, err = pcall(encode_message, {Fields = { {noname = "foo", value = 1}} }) assert(not ok) assert("encode_message() failed: field name must be a string" == err, string.format("received: %s", err)) -ok, err = pcall(heka_json.parse_message, "Payload") -assert("no active message" == err, err) +ok, err = pcall(rjson.parse_message, "Payload") +assert("parse_message() no active message" == err, err) diff --git a/src/heka/test/lua/heka_json.lua b/src/heka/test/lua/heka_json.lua deleted file mode 100644 index e50f8c2..0000000 --- a/src/heka/test/lua/heka_json.lua +++ /dev/null @@ -1,312 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "heka_json" -require "heka_stream_reader" - -schema_json = [[{ - "type":"object", - "required":["Timestamp"], - "properties":{ - "Timestamp":{ - "type":"integer", - "minimum":0 - }, - "Type":{ - "type":"string" - }, - "Logger":{ - "type":"string" - }, - "Hostname":{ - "type":"string", - "format":"hostname" - }, - "EnvVersion":{ - "type":"string", - "pattern":"^[0-9]+(\\.[0-9]+){0,2}" - }, - "Severity":{ - "type":"integer", - "minimum":0, - "maximum":7 - }, - "Pid":{ - "type":"integer", - "minimum":0 - }, - "Fields":{ - "type":"object", - "minProperties":1, - "additionalProperties":{ - "anyOf": [ - { "$ref": "#/definitions/field_value"}, - { "$ref": "#/definitions/field_array"}, - { "$ref": "#/definitions/field_object"} - ] - } - } - }, - "definitions":{ - "field_value":{ - "type":["string", "number", "boolean"] - }, - "field_array":{ - "type":"array", - "minItems": 1, - "oneOf": [ - {"items": {"type":"string"}}, - {"items": {"type":"number"}}, - {"items": {"type":"boolean"}} - ] - }, - "field_object":{ - "type":"object", - "required":["value"], - "properties":{ - "value":{ - "oneOf": [ - { "$ref": "#/definitions/field_value" }, - { "$ref": "#/definitions/field_array" } - ] - }, - "representation":{"type":"string"} - } - } - } -}]] - -json = [[{"Timestamp":0, "Type":"foo", "Logger":"bar"}]] - -schema = heka_json.parse_schema(schema_json) -assert(schema) - -ok, err = pcall(heka_json.parse_schema, "{") -assert(not ok) -assert(err == "failed to parse offset:1 Missing a name for object member.", err) -ok, err = pcall(heka_json.parse, "{") -assert(not ok) -assert(err == "failed to parse offset:1 Missing a name for object member.", err) - -doc = heka_json.parse(json) -assert(doc) -assert(doc:validate(schema)) -doc1 = heka_json.parse(json) - -json = [[{"Timestamp":-1, "Type":"foo", "Logger":"bar"}]] -doc = heka_json.parse(json) -assert(doc) -assert(not doc:validate(schema)) - -json = [[{"Timestamp":0, "EnvVersion":"unknown"}]] -doc = heka_json.parse(json) -assert(doc) -assert(not doc:validate(schema)) - -json = [[{"array":[1,"string",true,false,[3,4],{"foo":"bar"},null]}]] -doc = heka_json.parse(json) -assert(doc) -assert(doc:type() == "object") -a = doc:find("array"); -assert(doc:type(a) == "array") -str = doc:find(a, 1) -assert("string" == doc:type(str), doc:type(str)) -tmp = doc:find("array", 3); -assert("boolean" == doc:type(tmp), doc:type(tmp)) -ok, err = pcall(doc.iter, doc, tmp) -assert("iter() not allowed on a primitive type", err, err) -ok, err = pcall(doc.find, doc, doc1, "array") -assert("invalid value" == err, err) -tmp = doc:find("array", "key") -assert(not tmp) -tmp = doc:find("array", 5, 0) -assert(not tmp) -tmp = doc:find("array", 7) -assert(not tmp) -tmp = doc:find(true) -assert(not tmp) -tmp = doc:find("array", 5, "missing") -assert(not tmp) - -ra = { - {value = 1, type = "number"}, - {value = "string", type = "string", len = 6}, - {value = true, type = "boolean"}, - {value = false, type = "boolean"}, - {type = "array", len = 2}, - {type = "object", len = 1}, - {type = "null"}, -} - -for i,v in doc:iter(a) do - assert(ra[i+1].type == doc:type(v)) - if ra[i+1].value ~= nil then - assert(ra[i+1].value == doc:value(v)) - else - ok, err = pcall(doc.value, doc, v) - if ra[i+1].type == "null" then - assert(ok) - else - assert(not ok) - end - end - if ra[i+1].len then - assert(ra[i+1].len == doc:size(v)) - else - ok, err = pcall(doc.size, doc, v) - assert(not ok) - end -end - -a0 = doc:find("array", 0); -assert(a0) -assert(doc:value(a0) == 1) - -a5 = doc:find("array", 5); -assert(a5) -assert(doc:type(a5) == "object") -assert(doc:size(a5) == 1) -for k,v in doc:iter(a5) do - assert(doc:type(v) == "string") - assert(doc:value(v) == "bar") - assert(k == "foo", tostring(k)) -end - -a5x = doc:find("array", 5); -assert(a5 == a5x) - -ok, doc = pcall(heka_json.parse_message, 1) -assert("bad argument #0 to '?' (invalid number of arguments)" == doc, doc) - -ok, doc = pcall(heka_json.parse_message, "", "Fields[json]") -assert("bad argument #1 to '?' (mozsvc.heka_stream_reader expected, got string)" == doc, doc) - -hsr = heka_stream_reader.new("test") -hsr:decode_message("\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\4\82\23\10\4\106\115\111\110\16\1\42\13\123\34\102\111\111\34\58\34\98\97\114\34\125") -ok, doc = pcall(heka_json.parse_message, hsr, "Fields[missing]") -assert("field not found" == doc, doc) - -ok, doc = pcall(heka_json.parse_message, hsr, "Fields[missing]", "") -assert("bad argument #3 to '?' (number expected, got string)" == doc, doc) - -ok, doc = pcall(heka_json.parse_message, hsr, "Fields[missing]", 0, "") -assert("bad argument #4 to '?' (number expected, got string)" == doc, doc) - -ok, err = pcall(heka_json.parse_message, hsr, "Fields[json") -assert("field not found" == err, err) - -ok, err = pcall(heka_json.parse_message, hsr, "Fieldsjson]") -assert("field not found" == err, err) - -ok, err = pcall(heka_json.parse_message, hsr, "foo") -assert("field not found" == err, err) - -ok, doc = pcall(heka_json.parse_message, hsr, "Fields[json]") -assert(ok, doc) - -foo = doc:find("foo") -assert(foo) -assert("bar" == doc:value(foo), tostring(doc:value(foo))) -ok, err = pcall(doc.value, doc, (doc1:find())) -assert(not ok) -assert("invalid value" == err, err) - -assert("object" == doc:type(), doc:type()); -rdoc = doc:remove() -assert("null" == doc:type(), doc:type()); -foo = rdoc:find("foo") -assert("bar" == rdoc:value(foo), tostring(doc:value(foo))) - -assert("object" == rdoc:type(), rdoc:type()); -nrdoc = rdoc:remove("foo") -assert("object" == rdoc:type(), rdoc:type()); - -nested = '{"main":{"m1":1}, "payload":{"values":[1,2,3]}}' -json = heka_json.parse(nested) -assert(json) -assert(json:size() == 2, json:size()) -values = json:find("payload", "values") -assert(values) -vit = json:iter(values) -i,v = vit() -assert(i == 0, tostring(i)) -assert(json:value(v) == 1, tostring(json:value(v))) -i,v = vit() -json:remove("payload") -assert(i == 1, tostring(i)) -assert(json:value(v) == 2, tostring(json:value(v))) -i,v = vit() -assert(i == 2, tostring(i)) -assert(json:value(v) == 3, tostring(json:value(v))) -i,v = vit() -assert(i == nil, tostring(i)) - -json = heka_json.parse(nested) -assert(json) -object = json:find("main") -vit = json:iter(object) -json:remove("main") -ok, err = pcall(vit) -assert(err == "iterator has been invalidated") - -json = heka_json.parse(nested) -assert(json) -values = json:find("payload", "values") -vit = json:iter(values) -rvalues = json:remove("payload", "values") -ok, err = pcall(vit) -assert(err == "iterator has been invalidated") -schema_array = [[{ -"type":"array", -"minItems": 1, -"oneOf": [{"items": {"type":"number"}}] -}]] -sa = heka_json.parse_schema(schema_array) -assert(sa) -assert(rvalues:validate(sa)) - -if read_config("have_zlib") then - gz_nested = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139\8\0\0\0\0\0\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" - hsr:decode_message(gz_nested) - ok, json = pcall(heka_json.parse_message, hsr, "Payload") - assert(ok, json) - values = json:find("payload", "values") - assert(values) - - gz_too_large = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\51\31\139\8\0\0\0\0\0\0\3\237\193\49\13\0\0\8\3\176\31\25\147\129\178\201\71\7\73\219\52\155\2\0\0\0\0\0\0\0\0\255\100\14\116\172\116\102\0\36\0\0" - hsr:decode_message(gz_too_large) - ok, json = pcall(heka_json.parse_message, hsr, "Payload") - assert("lsb_ungzip failed" == json, json) - - gz_corrupt = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\99\50\66\31\139foobar\0\3\171\86\202\77\204\204\83\178\170\86\202\53\84\178\50\172\213\81\80\42\72\172\204\201\79\76\1\137\149\37\230\148\166\22\43\89\69\27\234\24\233\24\199\214\214\114\1\0\64\251\6\210\48\0\0\0" - hsr:decode_message(gz_corrupt) - ok, json = pcall(heka_json.parse_message, hsr, "Payload") - assert("lsb_ungzip failed" == json, json) -end - -minimal = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0" -hsr:decode_message(minimal) -ok, err = pcall(heka_json.parse_message, hsr, "Payload") -assert("field not found" == err, err) - -invalid_json = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\50\001{" -hsr:decode_message(invalid_json) -ok, err = pcall(heka_json.parse_message, hsr, "Payload") -assert("failed to parse offset:1 Missing a name for object member." == err, err) - -short_json = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\50\1\57" -hsr:decode_message(short_json) -ok, doc = pcall(heka_json.parse_message, hsr, "Payload") -assert(ok, doc) -assert("number" == doc:type(), doc:type()) -assert(9 == doc:value(), tostring(doc:value())) -rv = doc:remove("foo") -assert(not rv) - -assert(nil == doc:find(nil)) -assert(nil == doc:find(nil)) -assert(nil == doc:size(nil)) -assert(nil == doc:type(nil)) -assert(nil == doc:make_field(nil)) -assert(nil == doc:iter(nil)) diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index 6044361..55d5948 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -4,16 +4,16 @@ require "string" require "bloom_filter"; -require "heka_json" +require "rjson" local bf = bloom_filter.new(10, 0.01) local json = [[{"foo":"bar"}]] -local doc = heka_json.parse(json) +local doc = rjson.parse(json) assert(doc) -local doc1 = heka_json.parse(json) -local doc2 = heka_json.parse(json) +local doc1 = rjson.parse(json) +local doc2 = rjson.parse(json) local large_json = string.format('{"foo":"%s"}', string.rep("x", 66000)) -local large_doc = heka_json.parse(large_json) +local large_doc = rjson.parse(large_json) assert(large_doc) -- Table tests @@ -49,7 +49,8 @@ local err_msgs = { } for i, v in ipairs(msgs) do - inject_message(v[1], v[2]) + local ok, err = pcall(inject_message, v[1], v[2]) + assert(ok, string.format("test: %d err: %s", i, tostring(err))) end for i, v in ipairs(err_msgs) do @@ -60,8 +61,7 @@ end -- Stream Reader tests require "io" -require "heka_stream_reader" -local hsr = heka_stream_reader.new("test") +local hsr = create_stream_reader("test") ok, err = pcall(inject_message, hsr) local eerr = "inject_message() attempted to inject a nil message" assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) @@ -130,7 +130,7 @@ assert("read_message() incorrect number of arguments" == err, string.format("rec ok, err = pcall(hsr.read_message, 1, 2) assert(not ok) -assert("bad argument #1 to '?' (mozsvc.heka_stream_reader expected, got number)" == err, string.format("received: %s", err)) +assert("bad argument #1 to '?' (lsb.heka_stream_reader expected, got number)" == err, string.format("received: %s", err)) -- String tests diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index 0a974ee..a94f632 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -5,6 +5,7 @@ assert(read_config) assert(decode_message) assert(inject_message) +assert(create_stream_reader) assert(not read_message) assert(not encode_message) assert(not update_checkpoint) diff --git a/src/heka/test/lua/kafka_consumer.lua b/src/heka/test/lua/kafka_consumer.lua deleted file mode 100644 index 1afe996..0000000 --- a/src/heka/test/lua/kafka_consumer.lua +++ /dev/null @@ -1,79 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "heka_kafka_consumer" -require "string" - -local ok, err -ok, err = pcall(heka_kafka_consumer.new) -assert(err == "bad argument #0 to '?' (incorrect number of arguments)", err) - -ok, err = pcall(heka_kafka_consumer.new, true, nil, nil) -assert(err == "bad argument #1 to '?' (string expected, got boolean)", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", true, nil) -assert(err == "bad argument #2 to '?' (table expected, got boolean)", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {}, nil) -assert(err == "bad argument #2 to '?' (the topics array is empty)", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {}) -assert(err == "group.id must be set", err) - -ok, err = pcall(heka_kafka_consumer.new, "", {"test"}, {["group.id"] = "foo"}) -assert(err == "invalid broker list", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = "foobar"}) -assert(err == 'Failed to set auto.offset.reset = foobar : Invalid value for configuration property "auto.offset.reset"', err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = 0}) -assert(err == 'Failed to set auto.offset.reset = 0 : Invalid value for configuration property "auto.offset.reset"', err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = true}) -assert(err == 'Failed to set auto.offset.reset = true : Invalid value for configuration property "auto.offset.reset"', err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {["auto.offset.reset"] = assert}) -assert(err == "invalid config value type: function", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo"}, {[assert] = true}) -assert(err == "invalid config key type: function", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test:9000000000"}, {["group.id"] = "foo"}) -assert(err == "invalid topic partition > INT32_MAX", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test:-1"}, {["group.id"] = "foo"}) -assert(err == "invalid topic partition < 0", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {true}, {["group.id"] = "foo"}) -assert(err == "topics must be an array of strings", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"one", item = "test"}, {["group.id"] = "foo"}) -assert(err == "topics must be an array of strings", err) - -ok, err = pcall(heka_kafka_consumer.new, "test", {"test"}, {["group.id"] = "foo", ["message.max.bytes"] = true}) -assert(err:match("^Failed to set message.max.bytes = true"), err) - - -local consumer = heka_kafka_consumer.new("localhost:9092", {"test"}, {["group.id"] = "integration_testing"}, {["auto.offset.reset"] = "smallest"}) -local consumer1 = heka_kafka_consumer.new("localhost:9092", {"test:1"}, {["group.id"] = "other"}) -local pb, topic, partition, key = consumer1:receive() -assert(not pb) - -local payloads = {"one", "two", "three"} - -function process_message() - local cnt = 0 - for i=1, 10 do - pb, topic, partition, key = consumer:receive() - if pb then - cnt = cnt + 1 - local msg = decode_message(pb) - if msg.Payload ~= payloads[cnt] then - return -1, string.format("expected: %s received: %s", payloads[cnt], msg.Payload) - end - if cnt == 3 then return 0 end - end - end - return -1, string.format("received %d/3 messages", cnt) -end diff --git a/src/heka/test/lua/kafka_producer.lua b/src/heka/test/lua/kafka_producer.lua deleted file mode 100644 index d076a2c..0000000 --- a/src/heka/test/lua/kafka_producer.lua +++ /dev/null @@ -1,75 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "heka_kafka_producer" -require "string" - -local ok, err -ok, err = pcall(heka_kafka_producer.new) -assert(err == "bad argument #0 to '?' (incorrect number of arguments)", err) - -ok, err = pcall(heka_kafka_producer.new, "") -assert(err == "invalid broker list", err) - -ok, err = pcall(heka_kafka_producer.new, "brokerlist", true) -assert(err == "bad argument #2 to '?' (table expected, got boolean)", err) - -ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = "foo"}) -assert(err:match("^Failed to set message.max.bytes = foo"), err) - -ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = 1}) -assert(err == 'Failed to set message.max.bytes = 1 : Configuration property "message.max.bytes" value 1 is outside allowed range 1000..1000000000\n', err) - -ok, err = pcall(heka_kafka_producer.new, "local host", {["message.max.bytes"] = true}) -assert(err:match("^Failed to set message.max.bytes = true"), err) - -ok, err = pcall(heka_kafka_producer.new, "brokerlist", {[assert] = true}) -assert(err == "invalid config key type: function", err) - -ok, err = pcall(heka_kafka_producer.new, "brokerlist", {foo = assert}) -assert(err == "invalid config value type: function", err) - -local producer = heka_kafka_producer.new("localhost:9092", - { - ["topic.metadata.refresh.interval.ms"] = -1, - ["batch.num.messages"] = 1, - ["queue.buffering.max.ms"] = 1, - }) -local cnt = 0 -local topic = "test" -local sid -function process_message(sequence_id) - sid = sequence_id - if cnt == 0 then - local topic_tmp = "tmp" - producer:create_topic(topic) - assert(producer:has_topic(topic)) - assert(0 == producer:send(topic, -1, sequence_id)) -- send the original - - producer:create_topic(topic_tmp) - producer:create_topic(topic_tmp, nil) - assert(producer:has_topic(topic_tmp)) - producer:destroy_topic(topic_tmp) - assert(not producer:has_topic(topic_tmp)) - ok, err = pcall(producer.create_topic, producer, "topic", true) - assert(err == "bad argument #3 to '?' (table expected, got boolean)", err) - ok, err = pcall(producer.send, producer, "foobar", -1, sequence_id) - assert(err == "invalid topic", err) - elseif cnt == 1 then - assert(0 == producer:send(topic, -1, sequence_id, encode_message({Payload = "two"}))) - elseif cnt == 2 then - assert(0 == producer:send(topic, -1, sequence_id, encode_message({Payload = "three"}))) - end - cnt = cnt + 1 - return 0 -end - -function timer_event(ns) - if sid then - ok, err = pcall(producer.send, producer, topic, -1, sid) - assert(err == "no active message", err) - sid = nil - end - producer:poll() -end diff --git a/src/heka/test/lua/output.lua b/src/heka/test/lua/output.lua index 80f9337..deb8869 100644 --- a/src/heka/test/lua/output.lua +++ b/src/heka/test/lua/output.lua @@ -9,6 +9,7 @@ assert(read_message) assert(decode_message) assert(encode_message) assert(update_checkpoint) +assert(create_message_matcher) assert(not inject_message) assert(not add_to_payload) assert(not inject_payload) diff --git a/src/heka/test/test.h.in b/src/heka/test/test.h.in index fde124c..688064e 100644 --- a/src/heka/test/test.h.in +++ b/src/heka/test/test.h.in @@ -9,7 +9,7 @@ #ifndef test_h_ #define test_h_ -#include "${CMAKE_SOURCE_DIR}/src/test/mu_test.h" +#include "luasandbox/test/mu_test.h" #include #include @@ -17,11 +17,12 @@ #include #define TEST_LUA_PATH "${CMAKE_SOURCE_DIR}/modules/?.lua" +#define TEST_BASE_CPATH "${CMAKE_BINARY_DIR}/ep_base/inst/lib/luasandbox/" #ifdef _WIN32 -#define TEST_LUA_CPATH "${CMAKE_BINARY_DIR}/ep_base/lib/lua/?.dll" +#define TEST_LUA_CPATH TEST_BASE_CPATH "modules/?.dll;" TEST_BASE_CPATH "io_modules/?.dll" #else -#define TEST_LUA_CPATH "${CMAKE_BINARY_DIR}/ep_base/lib/lua/?.so" +#define TEST_LUA_CPATH TEST_BASE_CPATH "modules/?.so;" TEST_BASE_CPATH "io_modules/?.so" #endif #endif diff --git a/src/heka/test/test_sandbox.c b/src/heka/test/test_heka_sandbox.c similarity index 97% rename from src/heka/test/test_sandbox.c rename to src/heka/test/test_heka_sandbox.c index 82ce15a..7c2a9e8 100644 --- a/src/heka/test/test_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -419,10 +419,10 @@ static char* test_pm_error() }; struct pm_result results[] = { - { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:25: boom" }, + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:26: boom" }, { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, - { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:31: aaaaaaaaaaaaaaaaaaaaaaaaa" + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:32: aaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, // >max error message @@ -604,22 +604,6 @@ static char* test_read_message() } -static char* test_heka_json() -{ - lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, "lua/heka_json.lua", NULL, -#ifdef HAVE_ZLIB - "have_zlib = true\n" -#endif - "max_message_size = 8196\n" - "path = [[" TEST_LUA_PATH "]]\n" - "cpath = [[" TEST_LUA_CPATH "]]\n", &logger, iim); - mu_assert(hsb, "lsb_heka_create_input failed"); - e = lsb_heka_destroy_sandbox(hsb); - return NULL; -} - - static char* test_heka_util() { lsb_heka_message m; @@ -685,7 +669,6 @@ static char* all_tests() mu_run_test(test_encode_message); mu_run_test(test_decode_message); mu_run_test(test_read_message); - mu_run_test(test_heka_json); mu_run_test(test_heka_util); mu_run_test(benchmark_decode_message); diff --git a/src/heka/test/test_kafka.c b/src/heka/test/test_kafka.c deleted file mode 100644 index 4f58062..0000000 --- a/src/heka/test/test_kafka.c +++ /dev/null @@ -1,115 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Heka Kafka sandbox integration tests @file */ - -#include "test.h" - -#include -#include - -#include "luasandbox/heka/sandbox.h" - -char *e = NULL; - -void dlog(void *context, const char *component, int level, const char *fmt, ...) -{ - (void)context; - va_list args; - va_start(args, fmt); - fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, - component ? component : "unnamed"); - vfprintf(stderr, fmt, args); - fwrite("\n", 1, 1, stderr); - va_end(args); -} -static lsb_logger logger = { .context = NULL, .cb = dlog }; - -static volatile ptrdiff_t g_sequence = 0; -static int ucp(void *parent, void *sequence_id) -{ - (void)parent; - g_sequence = (ptrdiff_t)sequence_id; - return 0; -} - - -static char* test_producer() -{ - static const char pb[] = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x32\x03one"; - lsb_heka_message m; - mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); - mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, NULL), "failed"); - lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_output(NULL, "lua/kafka_producer.lua", NULL, NULL, &logger, - ucp); - mu_assert(hsb, "lsb_heka_create_output failed"); - mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)1, false), "err: %s", - lsb_heka_get_error(hsb)); - mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)2, false), "err: %s", - lsb_heka_get_error(hsb)); - mu_assert(0 == lsb_heka_pm_output(hsb, &m, (void*)3, false), "err: %s", - lsb_heka_get_error(hsb)); - while (g_sequence != 3) { - mu_assert(0 == lsb_heka_timer_event(hsb, 0, false), "err: %s", - lsb_heka_get_error(hsb)); - } - lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(stats.pm_failures == 0, "pm_failures: %llu", stats.pm_failures); - e = lsb_heka_destroy_sandbox(hsb); - mu_assert(!e, "%s", e); - lsb_free_heka_message(&m); - return NULL; -} - - -static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, - const char *cp_string) -{ - (void)parent; - (void)pb; - (void)pb_len; - (void)cp_numeric; - (void)cp_string; - return 0; -} - - -static char* test_consumer() -{ - lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_input(NULL, "lua/kafka_consumer.lua", NULL, NULL, - &logger, iim); - mu_assert(hsb, "lsb_heka_create_input failed"); - mu_assert(0 == lsb_heka_pm_input(hsb, 0, NULL, false), "err: %s", - lsb_heka_get_error(hsb)); - e = lsb_heka_destroy_sandbox(hsb); - mu_assert(!e, "%s", e); - return NULL; -} - - -static char* all_tests() -{ - mu_run_test(test_producer); - mu_run_test(test_consumer); - return NULL; -} - - -int main() -{ - char *result = all_tests(); - if (result) { - printf("%s\n", result); - } else { - printf("ALL TESTS PASSED\n"); - } - printf("Tests run: %d\n", mu_tests_run); - free(e); - - return result != 0; -} diff --git a/src/luasandbox.c b/src/luasandbox.c index 4b90f4c..1998bc2 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -44,8 +44,6 @@ static const luaL_Reg preload_module_list[] = { { LUA_OSLIBNAME, luaopen_os }, { LUA_STRLIBNAME, luaopen_string }, { LUA_MATHLIBNAME, luaopen_math }, - { LSB_HASH_MODULE, luaopen_lsb_hash }, - { LSB_COMPRESSION_MODULE, luaopen_lsb_compression }, { NULL, NULL } }; @@ -130,10 +128,10 @@ static void instruction_manager(lua_State *lua, lua_Debug *ar) static int output(lua_State *lua) { - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "output() invalid lightuserdata"); - } + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, __func__); int n = lua_gettop(lua); if (n == 0) { @@ -146,10 +144,11 @@ static int output(lua_State *lua) static int output_print(lua_State *lua) { - lsb_lua_sandbox *lsb = lua_touserdata(lua, lua_upvalueindex(1)); - if (!lsb) { - return luaL_error(lua, "print() invalid lightuserdata"); - } + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "print() invalid " LSB_THIS_PTR); + lsb->output.buf[0] = 0; lsb->output.pos = 0; // clear the buffer @@ -493,12 +492,14 @@ lsb_lua_sandbox* lsb_create(void *parent, size_t ol = get_int(lsb->lua, -1, "output_limit"); int log_level = get_int(lsb->lua, -1, "log_level"); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); - lua_pushcclosure(lsb->lua, &read_config, 0); + lua_pushlightuserdata(lsb->lua, lsb); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lua_pushcfunction(lsb->lua, &read_config); lua_setglobal(lsb->lua, "read_config"); - lua_pushlightuserdata(lsb->lua, (void *)lsb); - lua_pushcclosure(lsb->lua, &output, 1); + lua_pushcfunction(lsb->lua, &output); lua_setglobal(lsb->lua, "output"); + preload_modules(lsb->lua); lsb->parent = parent; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = ml; @@ -555,7 +556,6 @@ lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; #endif - preload_modules(lsb->lua); // load package module lua_pushcfunction(lsb->lua, luaopen_package); lua_pushstring(lsb->lua, LUA_LOADLIBNAME); @@ -741,8 +741,7 @@ void lsb_add_function(lsb_lua_sandbox *lsb, lua_CFunction func, if (!lsb || !func || !func_name) return; if (!lsb->lua) return; - lua_pushlightuserdata(lsb->lua, (void *)lsb); - lua_pushcclosure(lsb->lua, func, 1); + lua_pushcfunction(lsb->lua, func); lua_setglobal(lsb->lua, func_name); } diff --git a/src/luasandbox_defines.h b/src/luasandbox_defines.h index 6d491d7..d4315c9 100644 --- a/src/luasandbox_defines.h +++ b/src/luasandbox_defines.h @@ -15,4 +15,10 @@ #define CLOSE_ON_EXEC "e" #endif +#ifdef _MSC_VER +#ifndef __func__ +#define __func__ __FUNCTION__ +#endif +#endif + #endif diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h index 8a17474..0a5f8c5 100644 --- a/src/luasandbox_impl.h +++ b/src/luasandbox_impl.h @@ -13,9 +13,6 @@ #include "luasandbox/lua.h" #include "luasandbox/util/output_buffer.h" -#define LSB_HASH_MODULE "lsb_hash" -#define LSB_COMPRESSION_MODULE "lsb_compression" - struct lsb_lua_sandbox { lua_State *lua; void *parent; @@ -46,22 +43,4 @@ lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb); */ lsb_err_value restore_global_data(lsb_lua_sandbox *lsb); -/** - * Built in Lua hash module - * - * @param lua Lua state - * - * @return int number of return values - */ -int luaopen_lsb_hash(lua_State *lua); - -/** - * Built in Lua compression module - * - * @param lua Lua state - * - * @return int number of return values - */ -int luaopen_lsb_compression(lua_State *lua); - #endif diff --git a/src/luasandbox_modules.c b/src/luasandbox_modules.c deleted file mode 100644 index 1404640..0000000 --- a/src/luasandbox_modules.c +++ /dev/null @@ -1,94 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Additional built in Lua sandbox modules @file */ - -#include - -#include "luasandbox/lauxlib.h" -#include "luasandbox/lua.h" -#include "luasandbox/util/util.h" -#include "luasandbox_impl.h" - -#ifdef HAVE_ZLIB -#include - - -static int zlib_crc32(lua_State *lua) -{ - size_t len; - const char *buf; - - if (lua_type(lua, 1) == LUA_TSTRING) { - buf = lua_tolstring(lua, 1, &len); - } else { - return luaL_argerror(lua, 1, "must be a string"); - } - lua_pushnumber(lua, (lua_Number)lsb_crc32(buf, len)); - return 1; -} - - -static int zlib_ungzip(lua_State *lua) -{ - size_t len; - const char *buf; - - if (lua_type(lua, 1) == LUA_TSTRING) { - buf = lua_tolstring(lua, 1, &len); - } else { - return luaL_argerror(lua, 1, "must be a string"); - } - - lua_Number mss = lua_tonumber(lua, 2); // unlimited if max is not provided - int t = lua_type(lua, 2); - if (mss < 0 || (t != LUA_TNUMBER && t != LUA_TNONE && t != LUA_TNIL)) { - return luaL_argerror(lua, 2, "must be a positive number or nil"); - } - - char *gbuf = lsb_ungzip(buf, len, (size_t)mss, &len); - if (gbuf) { - lua_pushlstring(lua, gbuf, len); - free(gbuf); - } else { - lua_pushnil(lua); - } - return 1; -} -#endif - - -static const struct luaL_reg hashlib_f[] = -{ -#ifdef HAVE_ZLIB - { "crc32", zlib_crc32 }, -#endif - { NULL, NULL } -}; - - -int luaopen_lsb_hash(lua_State *lua) -{ - luaL_register(lua, LSB_HASH_MODULE, hashlib_f); - return 1; -} - - -static const struct luaL_reg compressionlib_f[] = -{ -#ifdef HAVE_ZLIB - { "ungzip", zlib_ungzip }, -#endif - { NULL, NULL } -}; - - -int luaopen_lsb_compression(lua_State *lua) -{ - luaL_register(lua, LSB_COMPRESSION_MODULE, compressionlib_f); - return 1; -} - diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index d251dc3..b9176cb 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -2,20 +2,26 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -add_executable(test_luasandbox test_luasandbox.c) -target_link_libraries(test_luasandbox luasandbox luasandboxutil) -set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util") +add_library(luasandboxtest SHARED sandbox.c) +set_target_properties(luasandboxtest PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_compile_definitions(luasandboxtest PRIVATE -Dluasandboxtest_EXPORTS) +target_link_libraries(luasandboxtest luasandbox) +install(TARGETS luasandboxtest DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) -add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/lib/lua ${CMAKE_CURRENT_BINARY_DIR}/modules) +add_executable(test_generic_sandbox test_generic_sandbox.c) +target_link_libraries(test_generic_sandbox luasandboxtest) +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/test") + +add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/inst/lib/luasandbox/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -add_test(NAME test_luasandbox COMMAND test_luasandbox) +add_test(NAME test_generic_sandbox COMMAND test_generic_sandbox) if(WIN32) STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") - set_tests_properties(test_luasandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_generic_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) elseif(APPLE) STRING(REPLACE ";" ":" LIBRARY_PATHS "${LIBRARY_PATHS}") - set_tests_properties(test_luasandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) + set_tests_properties(test_generic_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() diff --git a/src/test/lua/bloom_filter.lua b/src/test/lua/bloom_filter.lua deleted file mode 100644 index a187e82..0000000 --- a/src/test/lua/bloom_filter.lua +++ /dev/null @@ -1,26 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "bloom_filter" - -bf = bloom_filter.new(20, 0.01) - -function process(ts) - if not bf:query(ts) then - if not bf:add(ts) then - error("key existed") - end - end - - return 0 -end - -function report(tc) - if tc == 99 then - bf:clear() - else - write_output(bf:count()) - end -end - diff --git a/src/test/lua/bloom_filter_benchmark.lua b/src/test/lua/bloom_filter_benchmark.lua deleted file mode 100644 index 8b81a3e..0000000 --- a/src/test/lua/bloom_filter_benchmark.lua +++ /dev/null @@ -1,17 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "bloom_filter" - -bf = bloom_filter.new(2e6, 0.01) - -function process(ts) - bf:add(ts) - return 0 -end - -function report(tc) - write_output(bf:count()) -end - diff --git a/src/test/lua/circular_buffer.lua b/src/test/lua/circular_buffer.lua deleted file mode 100644 index 1a9fc45..0000000 --- a/src/test/lua/circular_buffer.lua +++ /dev/null @@ -1,25 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "string" -require "math" -require "circular_buffer" - -data = circular_buffer.new(3, 3, 1) -local ADD_COL = data:set_header(1, "Add column") -local SET_COL = data:set_header(2, "Set column", "count") -local GET_COL = data:set_header(3, "Get column", "count", "sum") - - -function process(ts) - if data:add(ts, ADD_COL, 1) then - data:set(ts, GET_COL, data:get(ts, ADD_COL)) - end - data:set(ts, SET_COL, 1) - return 0 -end - -function report(tc) - write_output(data) -end diff --git a/src/test/lua/circular_buffer_add.lua b/src/test/lua/circular_buffer_add.lua deleted file mode 100644 index 14def70..0000000 --- a/src/test/lua/circular_buffer_add.lua +++ /dev/null @@ -1,12 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "circular_buffer" - -data = circular_buffer.new(1440, 10, 1) - -function process(ts) - data:add(ts, 4, 1) - return 0 -end diff --git a/src/test/lua/circular_buffer_delta.lua b/src/test/lua/circular_buffer_delta.lua deleted file mode 100644 index c9c1433..0000000 --- a/src/test/lua/circular_buffer_delta.lua +++ /dev/null @@ -1,54 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "circular_buffer" - -data = circular_buffer.new(3, 3, 1) -local ADD_COL = data:set_header(1, "Add column") -local SET_COL = data:set_header(2, "Set column", "count") -local GET_COL = data:set_header(3, "Get column", "count", "sum") - -local cb = circular_buffer.new(2, 2, 1) -local SUM_COL = cb:set_header(1, "Sum column") -local MIN_COL = cb:set_header(2, "Min", "count", "min") - -function process(ts) - if data:add(ts, ADD_COL, 1) then - data:set(ts, GET_COL, data:get(ts, ADD_COL)) - end - data:set(ts, SET_COL, 1) - return 0 -end - -function report(tc) - if tc == 0 then - write_output(data:format("cbuf")) - elseif tc == 1 then - write_output(data:format("cbufd")) - elseif tc == 2 then - -- the sum delta should reflect the difference - -- the min delta should reflect the current value - cb:set(0, SUM_COL, 3) - cb:set(0, MIN_COL, 6) - write_output(cb:format("cbufd")) - cb:set(0, SUM_COL, 5) - cb:set(0, MIN_COL, 5) - write_output(cb:format("cbufd")) - elseif tc == 3 then - cb:fromstring("1 1 1 2 3 4 0 3 4") - write_output(cb:format("cbufd")) - elseif tc == 4 then - data:set(0, ADD_COL, 1/0) - data:set(0, ADD_COL, 1/0) - data:set(0, SET_COL, -1/0) - data:add(0, SET_COL, 1) - data:set(0, GET_COL, data:get(0, ADD_COL)) - write_output(data:format("cbufd")) - elseif tc == 5 then - cb:annotate(1e9, 1, "info", "delta anno") - write_output(cb:format("cbufd")) -- annotation delta only - elseif tc == 6 then - write_output(cb:format("cbufd")) -- no delta - end -end diff --git a/src/test/lua/circular_buffer_errors.lua b/src/test/lua/circular_buffer_errors.lua deleted file mode 100644 index 0151999..0000000 --- a/src/test/lua/circular_buffer_errors.lua +++ /dev/null @@ -1,12 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "circular_buffer" - -function process(tc) - if tc == 0 then - local cb = circular_buffer.new(1000, 10, 1) -- new() too much memory - end - return 0 -end diff --git a/src/test/lua/cuckoo_filter.lua b/src/test/lua/cuckoo_filter.lua deleted file mode 100644 index ecd48ed..0000000 --- a/src/test/lua/cuckoo_filter.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "cuckoo_filter" - -cf = cuckoo_filter.new(16) - -function process(ts) - if not cf:query(ts) then - if not cf:add(ts) then - error("key existed") - end - end - - return 0 -end - -function report(tc) - if tc == 98 then - cf:delete(1); - write_output(cf:count()) - elseif tc == 99 then - cf:clear() - write_output(cf:count()) - else - write_output(cf:count()) - end -end - diff --git a/src/test/lua/cuckoo_filter_benchmark.lua b/src/test/lua/cuckoo_filter_benchmark.lua deleted file mode 100644 index eaa580c..0000000 --- a/src/test/lua/cuckoo_filter_benchmark.lua +++ /dev/null @@ -1,17 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "cuckoo_filter" - -cf = cuckoo_filter.new(2e6) - -function process(ts) - cf:add(ts) - return 0 -end - -function report(tc) - write_output(cf:count()) -end - diff --git a/src/test/lua/hyperloglog.lua b/src/test/lua/hyperloglog.lua deleted file mode 100644 index 9aae7cd..0000000 --- a/src/test/lua/hyperloglog.lua +++ /dev/null @@ -1,21 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "hyperloglog" - -hll = hyperloglog.new() - -function process(ts) - hll:add(ts) - return 0 -end - -function report(tc) - if tc == 99 then - hll:clear() - else - write_output(hll:count()) - end -end - diff --git a/src/test/lua/lsb_compression.lua b/src/test/lua/lsb_compression.lua deleted file mode 100644 index 6e0796b..0000000 --- a/src/test/lua/lsb_compression.lua +++ /dev/null @@ -1,29 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local comp = require "lsb_compression" - -local tests = { - {comp.ungzip, nil, nil, false, "bad argument #1 to '?' (must be a string)"}, - {comp.ungzip, "string", true, false, "bad argument #2 to '?' (must be a positive number or nil)"}, - {comp.ungzip, "string", -100, false, "bad argument #2 to '?' (must be a positive number or nil)"}, - {comp.ungzip, "\031\139\008\000\000\000\000\000\000\003\075\203\207\079\074\044\226\002\000\071\151\044\178\007\000\000\000", nil, true, "foobar\n"}, - {comp.ungzip, "\031\139\008\000\000\000\000\000\000\003\075\203\207\079\074\044\226\002\000\071\151\044\178\007\000\000\000", 4, true, nil}, - {comp.ungzip, "not gzip", nil, true, nil}, -} - -function process () - for i,v in ipairs(tests) do - if v[1] then - if v[4] then - local result = v[1](v[2], v[3]) - assert(result == v[5], result) - else - local ok, err = pcall(v[1], v[2], v[3]) - assert(not ok and err == v[5], err) - end - end - end - return 0 -end diff --git a/src/test/lua/lsb_hash.lua b/src/test/lua/lsb_hash.lua deleted file mode 100644 index 002aa19..0000000 --- a/src/test/lua/lsb_hash.lua +++ /dev/null @@ -1,25 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local hash = require "lsb_hash" - -local tests = { - {hash.crc32 , nil , "bad argument #1 to '?' (must be a string)"}, - {hash.crc32 , "foobar", 2666930069}, -} - -function process () - for i,v in ipairs(tests) do - if v[1] then - if type(v[2]) == "string" then - local result = v[1](v[2]) - assert(result == v[3], result) - else - local ok, err = pcall(v[1], v[2]) - assert(not ok and err == v[3], err) - end - end - end - return 0 -end diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 6d751c7..6af9162 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -2,24 +2,7 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "circular_buffer" - -local cbuf = circular_buffer.new(1440, 3, 60) -local test = circular_buffer.new(2, 2, 60) - function process(tc) - if tc == 0 then -- lua types - write_output(1.2, " string ", nil, " ", true, " ", false) - elseif tc == 1 then -- cbuf - write_output(cbuf) - elseif tc == 2 then - test:annotate(0, 1, "info", "annotation\"\t\b\r\n\240 end") - test:annotate(60e9, 2, "alert", "alert") - test:annotate(120e9, 2, "info", "out of range, should be ignored") - write_output(test) - elseif tc == 3 then - test:set(120e9, 1, 0/0) -- advance the buffer (pruning the annotation) - write_output(test) - end + write_output(1.2, " string ", nil, " ", true, " ", false) return 0 end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index c48dab5..4186638 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -3,7 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "cjson" -require "circular_buffer"; require "bloom_filter"; require "cuckoo_filter" +require "circular_buffer" require "lpeg" function process(tc) if tc == 0 then -- error internal reference diff --git a/src/test/output/serialize.data b/src/test/output/serialize.data deleted file mode 100644 index 1a221df..0000000 --- a/src/test/output/serialize.data +++ /dev/null @@ -1,72 +0,0 @@ -if _PRESERVATION_VERSION and _PRESERVATION_VERSION ~= 0 then return end -_G["rate"] = 0.12345678 -_G["nested"] = {} -_G["nested"]["nested"] = {} -_G["nested"]["nested"]["n2"] = "two" -_G["nested"]["nested"]["n1"] = "one" -_G["nested"]["arg2"] = 2 -if _G["nested"]["cb"] == nil then _G["nested"]["cb"] = circular_buffer.new(2, 6, 1) end -_G["nested"]["cb"]:set_header(1, "Column_1", "count", "sum") -_G["nested"]["cb"]:set_header(2, "Column_2", "count", "sum") -_G["nested"]["cb"]:set_header(3, "Column_3", "count", "sum") -_G["nested"]["cb"]:set_header(4, "Column_4", "count", "sum") -_G["nested"]["cb"]:set_header(5, "Column_5", "count", "sum") -_G["nested"]["cb"]:set_header(6, "Column_6", "count", "sum") -_G["nested"]["cb"]:fromstring("1 1 nan nan nan nan nan nan nan nan nan nan nan nan") -_G["nested"]["arg1"] = 1 -if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1, true) end -_G["delta"]:set_header(1, "Column_1", "count", "sum") -_G["delta"]:fromstring("1 1 2 nan 0 2") -_G["_VERSION"] = "Lua 5.1" -if _G["data"] == nil then _G["data"] = circular_buffer.new(3, 3, 1) end -_G["data"]:set_header(1, "Column_1", "count", "sum") -_G["data"]:set_header(2, "Column_2", "count", "sum") -_G["data"]:set_header(3, "Column_3", "count", "sum") -_G["data"]:fromstring("2 2 nan nan nan nan nan nan nan nan nan") -_G["large_key"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"]["BD48B609-8922-4E59-A358-C242075CE088"] = 8 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"]["iiiiiiiiiiiiiiiiiii"] = {} -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["hhhhhhhhhhhhhhhhhhh"]["iiiiiiiiiiiiiiiiiii"]["BD48B609-8922-4E59-A358-C242075CE089"] = 9 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["ggggggggggggggggggg"]["BD48B609-8922-4E59-A358-C242075CE087"] = 7 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["BD48B609-8922-4E59-A358-C242075CE086"] = 6 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["BD48B609-8922-4E59-A358-C242075CE085"] = 5 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["BD48B609-8922-4E59-A358-C242075CE084"] = 4 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["BD48B609-8922-4E59-A358-C242075CE083"] = 3 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["BD48B609-8922-4E59-A358-C242075CE082"] = 2 -_G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["BD48B609-8922-4E59-A358-C242075CE081"] = 1 -_G["boolean"] = true -_G["key with spaces"] = "kws" -_G["uuids"] = {} -_G["uuids"][1] = {} -_G["uuids"][1]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE088" -_G["uuids"][1]["type"] = "test" -_G["uuids"][2] = {} -_G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" -_G["uuids"][2]["type"] = "test1" -_G["cyclea"] = {} -_G["cyclea"]["type"] = "cycle a" -_G["cyclea"]["b"] = {} -_G["cyclea"]["b"]["type"] = "cycle b" -_G["cyclea"]["b"]["a"] = _G["cyclea"] -_G["rates"] = {} -_G["rates"][1] = 99.1 -_G["rates"][2] = 98 -_G["rates"][3] = 97 -_G["rates"][4] = 92.002 -_G["rates"][5] = 91.10001 -_G["rates"]["key"] = "val" -_G["count"] = 0 -_G["dataRef"] = _G["data"] -_G["kvp"] = {} -_G["kvp"]["b"] = "bar" -_G["kvp"]["a"] = "foo" -_G["kvp"]["r"] = _G["rates"] -_G["cycleb"] = _G["cyclea"]["b"] diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index 1535688..0f9a1f7 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -7,9 +7,9 @@ _G["dataRef"]:set_header(3, "Column_3", "count", "sum") _G["dataRef"]:fromstring("2 2 nan nan nan nan nan nan nan nan nan") if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1) end _G["delta"]:set_header(1, "Column_1", "count", "sum") +_G["delta"]:fromstring("1 1 2 nan 0 2") _G["delta"]:annotate(1e+09, 1, "alert", "boom", true) _G["delta"]:annotate(0, 1, "info", "annotation\"\t\b\r\n end", true) -_G["delta"]:fromstring("1 1 2 nan 0 2") _G["data"] = _G["dataRef"] _G["rate"] = 0.12345678 _G["kvp"] = {} diff --git a/src/test/sandbox.c b/src/test/sandbox.c new file mode 100644 index 0000000..82975bf --- /dev/null +++ b/src/test/sandbox.c @@ -0,0 +1,142 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Implementation of the test interface for the generic lua sandbox @file */ + +#include "luasandbox/test/sandbox.h" + +#include +#include +#include + +#include "luasandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox_output.h" + +const char *lsb_test_output = NULL; +size_t lsb_test_output_len = 0; + +static void logger(void *context, + const char *component, + int level, + const char *fmt, + ...) +{ + (void)context; + va_list args; + fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, + component ? component : "unnamed"); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fwrite("\n", 1, 1, stderr); +} +lsb_logger lsb_test_logger = { .context = NULL, .cb = logger }; + + +int lsb_test_write_output(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, __func__); + + int n = lua_gettop(lua); + lsb_output(lsb, 1, n, 1); + lsb_test_output = lsb_get_output(lsb, &lsb_test_output_len); + return 0; +} + + +int lsb_test_process(lsb_lua_sandbox *lsb, double ts) +{ + static const char *func_name = "process"; + lua_State *lua = lsb_get_lua(lsb); + if (!lua) return 1; + + if (lsb_pcall_setup(lsb, func_name)) return 1; + + lua_pushnumber(lua, ts); + if (lua_pcall(lua, 1, 2, 0) != 0) { + char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); + int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, + em ? em : LSB_NIL_ERROR); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + + if (!lua_isnumber(lua, 1)) { + char err[LSB_ERROR_SIZE]; + int len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a numeric error code", func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + + int status = (int)lua_tointeger(lua, 1); + switch (lua_type(lua, 2)) { + case LUA_TNIL: + lsb_set_error(lsb, NULL); + break; + case LUA_TSTRING: + lsb_set_error(lsb, lua_tostring(lua, 2)); + break; + default: + { + char err[LSB_ERROR_SIZE]; + int len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a nil or string error message", + func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + break; + } + lua_pop(lua, 2); + + lsb_pcall_teardown(lsb); + + return status; +} + + +int lsb_test_report(lsb_lua_sandbox *lsb, double tc) +{ + static const char *func_name = "report"; + lua_State *lua = lsb_get_lua(lsb); + if (!lua) return 1; + + if (lsb_pcall_setup(lsb, func_name)) return 1; + + lua_pushnumber(lua, tc); + if (lua_pcall(lua, 1, 0, 0) != 0) { + char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); + int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, + em ? em : LSB_NIL_ERROR); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + + lsb_pcall_teardown(lsb); + lua_gc(lua, LUA_GCCOLLECT, 0); + + return 0; +} diff --git a/src/test/test_luasandbox.c b/src/test/test_generic_sandbox.c similarity index 50% rename from src/test/test_luasandbox.c rename to src/test/test_generic_sandbox.c index 47eb4e8..eb1841a 100644 --- a/src/test/test_luasandbox.c +++ b/src/test/test_generic_sandbox.c @@ -12,18 +12,14 @@ #include #include -#include "luasandbox.h" +#include "../luasandbox_impl.h" #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" -#include "luasandbox_output.h" +#include "luasandbox/test/mu_test.h" +#include "luasandbox/test/sandbox.h" #include "luasandbox/util/util.h" -#include "../luasandbox_impl.h" - -#include "mu_test.h" char *e = NULL; -const char *written_data = NULL; -size_t written_data_len = 0; static char print_out[2048] = { 0 }; @@ -61,19 +57,6 @@ int file_exists(const char *fn) return 0; } -void dlog(void *context, const char *component, int level, const char *fmt, ...) -{ - (void)context; - va_list args; - fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, - component ? component : "unnamed"); - va_start(args, fmt); - vfprintf(stderr, fmt, args); - va_end(args); - fwrite("\n", 1, 1, stderr); -} -static lsb_logger logger = { .context = NULL, .cb = dlog }; - void print(void *context, const char *component, int level, const char *fmt, ...) { @@ -88,111 +71,6 @@ void print(void *context, const char *component, int level, const char *fmt, ... static lsb_logger printer = { .context = NULL, .cb = print }; -int process(lsb_lua_sandbox *lsb, double ts) -{ - static const char *func_name = "process"; - lua_State *lua = lsb_get_lua(lsb); - if (!lua) return 1; - - if (lsb_pcall_setup(lsb, func_name)) return 1; - - lua_pushnumber(lua, ts); - if (lua_pcall(lua, 1, 2, 0) != 0) { - char err[LSB_ERROR_SIZE]; - const char *em = lua_tostring(lua, -1); - int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - em ? em : LSB_NIL_ERROR); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - - if (!lua_isnumber(lua, 1)) { - char err[LSB_ERROR_SIZE]; - int len = snprintf(err, LSB_ERROR_SIZE, - "%s() must return a numeric error code", func_name); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - - int status = (int)lua_tointeger(lua, 1); - switch (lua_type(lua, 2)) { - case LUA_TNIL: - lsb_set_error(lsb, NULL); - break; - case LUA_TSTRING: - lsb_set_error(lsb, lua_tostring(lua, 2)); - break; - default: - { - char err[LSB_ERROR_SIZE]; - int len = snprintf(err, LSB_ERROR_SIZE, - "%s() must return a nil or string error message", - func_name); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - break; - } - lua_pop(lua, 2); - - lsb_pcall_teardown(lsb); - - return status; -} - - -int report(lsb_lua_sandbox *lsb, double tc) -{ - static const char *func_name = "report"; - lua_State *lua = lsb_get_lua(lsb); - if (!lua) return 1; - - if (lsb_pcall_setup(lsb, func_name)) return 1; - - lua_pushnumber(lua, tc); - if (lua_pcall(lua, 1, 0, 0) != 0) { - char err[LSB_ERROR_SIZE]; - const char *em = lua_tostring(lua, -1); - int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - em ? em : LSB_NIL_ERROR); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - - lsb_pcall_teardown(lsb); - lua_gc(lua, LUA_GCCOLLECT, 0); - - return 0; -} - - -int write_output(lua_State *lua) -{ - void *luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - luaL_error(lua, "write_output() invalid lightuserdata"); - } - lsb_lua_sandbox *lsb = (lsb_lua_sandbox *)luserdata; - - int n = lua_gettop(lua); - lsb_output(lsb, 1, n, 1); - written_data = lsb_get_output(lsb, &written_data_len); - return 0; -} - - static char* test_api_assertion() { lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", "", NULL); @@ -210,9 +88,9 @@ static char* test_api_assertion() mu_assert(lsb_get_lua_file(NULL) == NULL, "not null"); mu_assert(lsb_get_parent(NULL) == NULL, "not null"); mu_assert(lsb_get_state(NULL) == LSB_UNKNOWN, "not unknown"); - lsb_add_function(NULL, write_output, "foo"); + lsb_add_function(NULL, lsb_test_write_output, "foo"); lsb_add_function(sb, NULL, "foo"); - lsb_add_function(sb, write_output, NULL); + lsb_add_function(sb, lsb_test_write_output, NULL); mu_assert(lsb_pcall_setup(NULL, "foo") == LSB_ERR_UTIL_NULL, "not null"); mu_assert(lsb_pcall_setup(sb, NULL) == LSB_ERR_UTIL_NULL, "not null"); lsb_add_function(NULL, NULL, NULL); @@ -228,7 +106,7 @@ static char* test_api_assertion() static char* test_create() { static char *cfg = "function foo() return 0 end\nt = {[true] = 1}\n"; - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", cfg, &logger); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", cfg, &lsb_test_logger); mu_assert(sb, "lsb_create() failed"); lsb_destroy(sb); sb = lsb_create(NULL, "lua/counter.lua", cfg, NULL); @@ -258,7 +136,7 @@ static char* test_create_error() sb = lsb_create(NULL, "lua/counter.lua", "cpath = 1", NULL); mu_assert(!sb, "lsb_create() invalid config"); - sb = lsb_create(NULL, "lua/counter.lua", "test = {", &logger); + sb = lsb_create(NULL, "lua/counter.lua", "test = {", &lsb_test_logger); mu_assert(!sb, "lsb_create() invalid config"); return NULL; @@ -382,10 +260,10 @@ static char* test_stop() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); lsb_stop_sandbox(sb); - lua_getglobal(sb->lua, "process"); - lua_pushnumber(sb->lua, 0); - mu_assert_rv(2, lua_pcall(sb->lua, 1, 2, 0)); - const char *msg = lua_tostring(sb->lua, -1); + lua_getglobal(lsb_get_lua(sb), "process"); + lua_pushnumber(lsb_get_lua(sb), 0); + mu_assert_rv(2, lua_pcall(lsb_get_lua(sb), 1, 2, 0)); + const char *msg = lua_tostring(lsb_get_lua(sb), -1); mu_assert(strcmp(LSB_SHUTTING_DOWN, msg) == 0, "received: %s", msg); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -457,21 +335,21 @@ static char* test_simple_error() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - int result = process(sb, 1); + int result = lsb_test_process(sb, 1); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); mu_assert(strcmp("ok", lsb_get_error(sb)) == 0, "process() received: %d %s", result, lsb_get_error(sb)); - result = process(sb, 0); + result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); mu_assert(strcmp("", lsb_get_error(sb)) == 0, "process() received: %d %s", result, lsb_get_error(sb)); - result = process(sb, 2); + result = lsb_test_process(sb, 2); mu_assert(result == 1, "process() received: %d %s", result, lsb_get_error(sb)); @@ -485,11 +363,8 @@ static char* test_simple_error() static char* test_output() { const char *outputs[] = { - "1.2 string nil true false" - , "" // this cbuf output is not verified, it is used for benchmarking - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":60,\"column_info\":[{\"name\":\"Column_1\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Column_2\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[{\"x\":0,\"col\":1,\"shortText\":\"i\",\"text\":\"annotation\\\"\\t\\b\\r\\n end\"},{\"x\":60000,\"col\":2,\"shortText\":\"a\",\"text\":\"alert\"}]}\nnan\tnan\nnan\tnan\n" - , "{\"time\":60,\"rows\":2,\"columns\":2,\"seconds_per_row\":60,\"column_info\":[{\"name\":\"Column_1\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Column_2\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[{\"x\":60000,\"col\":2,\"shortText\":\"a\",\"text\":\"alert\"}]}\nnan\tnan\nnan\tnan\n" - , NULL + "1.2 string nil true false", + NULL }; lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); @@ -497,15 +372,15 @@ static char* test_output() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); for (int x = 0; outputs[x]; ++x) { - int result = process(sb, x); + int result = lsb_test_process(sb, x); mu_assert(!result, "process() test: %d failed: %d %s", x, result, lsb_get_error(sb)); if (outputs[x][0]) { - mu_assert(strcmp(outputs[x], written_data) == 0, - "test: %d received: %s", x, written_data); + mu_assert(strcmp(outputs[x], lsb_test_output) == 0, + "test: %d received: %s", x, lsb_test_output); } } @@ -535,9 +410,9 @@ static char* test_output_errors() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); - int result = process(sb, i); + int result = lsb_test_process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); const char *le = lsb_get_error(sb); @@ -552,166 +427,6 @@ static char* test_output_errors() } -static char* test_cbuf_errors() -{ - const char *tests[] = - { - "process() not enough memory" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer_errors.lua", - MODULE_PATH "memory_limit = 32767", NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s %s", ret, - lsb_get_error(sb)); - - int result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char *le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - -static char* test_cbuf_core() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, - "../../ep_base/Source/lua_circular_buffer/test.lua", - test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, ""); - mu_assert(!ret, "lsb_init() received: %s", ret); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_cbuf() -{ - const char *output_file = "circular_buffer.preserve"; - const char *outputs[] = { - "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\nnan\tnan\tnan\nnan\tnan\tnan\nnan\tnan\tnan\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" - , "{\"time\":2,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n3\t1\t3\nnan\tnan\tnan\n1\t1\t1\n" - , "{\"time\":8,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\nnan\tnan\tnan\nnan\tnan\tnan\n1\t1\t1\n" - , NULL - }; - - remove(output_file); - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - int result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, "error %s", - lsb_get_error(sb)); - mu_assert(strcmp(outputs[0], written_data) == 0, "received: %s", - written_data); - - process(sb, 0); - process(sb, 1e9); - process(sb, 1e9); - process(sb, 2e9); - process(sb, 2e9); - process(sb, 2e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[1], written_data) == 0, "received: %s", - written_data); - - process(sb, 4e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[2], written_data) == 0, "received: %s", - written_data); - - process(sb, 10e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[3], written_data) == 0, "received: %s", - written_data); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_cbuf_delta() -{ - const char *output_file = "circular_buffer_delta.preserve"; - const char *outputs[] = { - "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n0\t1\t1\t1\n1\t2\t1\t2\n2\t3\t1\t3\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" - , "" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}],\"annotations\":[]}\n0\t2\t5\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}],\"annotations\":[]}\n0\t3\t4\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}],\"annotations\":[]}\n0\tinf\t-inf\tinf\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}],\"annotations\":[{\"x\":1000,\"col\":1,\"shortText\":\"i\",\"text\":\"delta anno\"}]}\n" - , "" - , NULL - }; - - remove(output_file); - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/circular_buffer_delta.lua", - test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - process(sb, 0); - process(sb, 1e9); - process(sb, 1e9); - process(sb, 2e9); - process(sb, 2e9); - process(sb, 2e9); - int result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[0], written_data) == 0, "received: %s", - written_data); - - result = report(sb, 1); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[1], written_data) == 0, "received: %s", - written_data); - - for (int i = 2; outputs[i] != NULL; ++i) { - result = report(sb, i - 2); - mu_assert(result == 0, "report() received: %d error: %s", result, - lsb_get_error(sb)); - mu_assert(strcmp(outputs[i], written_data) == 0, "test: %d received: %s", i, - written_data); - } - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_cjson() { lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cjson.lua", @@ -721,7 +436,7 @@ static char* test_cjson() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - int result = process(sb, 0); + int result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -739,13 +454,13 @@ static char* test_cjson_unlimited() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); - int result = process(sb, 0); + int result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - mu_assert(written_data_len == 103001, "received %d bytes", (int)written_data_len); + mu_assert(lsb_test_output_len == 103001, "received %d bytes", (int)lsb_test_output_len); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -802,7 +517,7 @@ static char* test_errors() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - int result = process(sb, i); + int result = lsb_test_process(sb, i); mu_assert(result == 1, "test: %d received: %d", i, result); const char *le = lsb_get_error(sb); @@ -839,7 +554,7 @@ static char* test_lpeg() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - int result = process(sb, 0); + int result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -860,7 +575,7 @@ static char* test_util() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - int result = process(sb, 0); + int result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); @@ -907,12 +622,12 @@ static char* test_restore() mu_assert(sb, "lsb_create() received: NULL"); lsb_err_value ret = lsb_init(sb, output_file); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - int result = process(sb, 0); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + int result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - mu_assert(strcmp("101", written_data) == 0, "test: initial load received: %s", - written_data); + mu_assert(strcmp("101", lsb_test_output) == 0, "test: initial load received: %s", + lsb_test_output); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -921,13 +636,13 @@ static char* test_restore() mu_assert(sb, "lsb_create() received: NULL"); ret = lsb_init(sb, output_file); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - mu_assert(strcmp("102", written_data) == 0, "test: reload received: %s", - written_data); - result = report(sb, 2); // change the preservation version + mu_assert(strcmp("102", lsb_test_output) == 0, "test: reload received: %s", + lsb_test_output); + result = lsb_test_report(sb, 2); // change the preservation version mu_assert(result == 0, "report() received: %d", result); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -937,12 +652,12 @@ static char* test_restore() mu_assert(sb, "lsb_create() received: NULL"); ret = lsb_init(sb, output_file); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - mu_assert(strcmp("101", written_data) == 0, - "test: reload with version change received: %s", written_data); + mu_assert(strcmp("101", lsb_test_output) == 0, + "test: reload with version change received: %s", lsb_test_output); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -973,182 +688,6 @@ static char* test_serialize_failure() } -static char* test_bloom_filter_core() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, - "../../ep_base/Source/lua_bloom_filter/test.lua", - test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_bloom_filter() -{ - const char *output_file = "bloom_filter.preserve"; - const char *tests[] = { - "1" - , "2" - , "3" - , NULL - }; - - remove(output_file); - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/bloom_filter.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - int i = 0; - for (; tests[i]; ++i) { - int result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i], written_data) == 0, "test: %d received: %s", i, - written_data); - } - - int result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, - written_data); // count should remain the same - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/bloom_filter.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - report(sb, 0); - mu_assert(strcmp("3", written_data) == 0, "test: count received: %s", - written_data); - - for (int i = 0; tests[i]; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, - written_data); // count should remain the same - - // test clear - report(sb, 99); - process(sb, 0); - report(sb, 0); - mu_assert(strcmp("1", written_data) == 0, "test: clear received: %s", - written_data); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_hyperloglog_core() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, - "../../ep_base/Source/lua_hyperloglog/test.lua", - test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - -static char* test_hyperloglog() -{ - const char *output_file = "hyperloglog.preserve"; - - remove(output_file); - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - - for (int i = 0; i < 100000; ++i) { - int result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - - int result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, "test: initial received: %s", - written_data); // count should remain the same - - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, "test: cache received: %s", - written_data); // count should remain the same - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, "test: reload received: %s", - written_data); // count should remain the same - - for (int i = 0; i < 100000; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, - "test: data replay received: %s", written_data); - // count should remain the same - - // test clear - report(sb, 99); - report(sb, 0); - mu_assert(strcmp("0", written_data) == 0, "test: clear received: %s", - written_data); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_struct() { lsb_lua_sandbox *sb = lsb_create(NULL, "lua/struct.lua", test_cfg, NULL); @@ -1157,7 +696,7 @@ static char* test_struct() lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - int result = process(sb, 0); + int result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); e = lsb_destroy(sb); @@ -1182,84 +721,6 @@ static char* test_sandbox_config() } -static char* test_cuckoo_filter() -{ - const char *output_file = "cuckoo_filter.preserve"; - const char *tests[] = { - "1" - , "2" - , "3" - , NULL - }; - - remove(output_file); - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cuckoo_filter.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - int i = 0; - for (; tests[i]; ++i) { - int result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i], written_data) == 0, "test: %d received: %s", i, - written_data); - } - - int result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, - written_data); // count should remain the same - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/cuckoo_filter.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - report(sb, 0); - mu_assert(strcmp("3", written_data) == 0, "test: count received: %s", - written_data); - - for (int i = 0; tests[i]; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, - written_data); // count should remain the same - - // test deletion - report(sb, 98); - mu_assert(strcmp("2", written_data) == 0, "test: delete received: %s", - written_data); - // tst clear - report(sb, 99); - mu_assert(strcmp("0", written_data) == 0, "test: clear received: %s", - written_data); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_sax() { const char *output_file = "sax.preserve"; @@ -1271,18 +732,18 @@ static char* test_sax() lsb_err_value ret = lsb_init(sb, output_file); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); - int result = report(sb, 0); + int result = lsb_test_report(sb, 0); mu_assert(result == 0, "report() received: %d %s", result, lsb_get_error(sb)); - mu_assert(strcmp("### CDC ##### #####", written_data) == 0, "received: %s", written_data); + mu_assert(strcmp("### CDC ##### #####", lsb_test_output) == 0, "received: %s", lsb_test_output); - result = process(sb, 0); + result = lsb_test_process(sb, 0); mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - result = report(sb, 0); + result = lsb_test_report(sb, 0); mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(word, written_data) == 0, "received: %s", written_data); + mu_assert(strcmp(word, lsb_test_output) == 0, "received: %s", lsb_test_output); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -1292,10 +753,10 @@ static char* test_sax() ret = lsb_init(sb, output_file); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); - report(sb, 0); - mu_assert(strcmp(word, written_data) == 0, "received: %s", written_data); + lsb_test_report(sb, 0); + mu_assert(strcmp(word, lsb_test_output) == 0, "received: %s", lsb_test_output); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -1322,7 +783,7 @@ static char* test_print() for (int i = 0; tests[i]; ++i) { print_out[0] = 0; - int result = process(sb, i); + int result = lsb_test_process(sb, i); mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); } @@ -1349,7 +810,7 @@ static char* test_print_disabled() for (int i = 0; tests[i]; ++i) { print_out[0] = 0; - int result = process(sb, i); + int result = lsb_test_process(sb, i); mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); } @@ -1360,7 +821,7 @@ static char* test_print_disabled() } -static char* test_print_logger() +static char* test_print_lsb_test_logger() { const char *tests[] = { @@ -1376,7 +837,7 @@ static char* test_print_logger() for (int i = 0; tests[i]; ++i) { print_out[0] = 0; - int result = process(sb, i); + int result = lsb_test_process(sb, i); mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); } @@ -1387,32 +848,6 @@ static char* test_print_logger() } -static char* test_builtin_modules() -{ - const char *tests[] = { - "lua/lsb_hash.lua", - "lua/lsb_compression.lua", - NULL - }; - - for (int i = 0; tests[i]; ++i) { - lsb_lua_sandbox *sb = lsb_create(NULL, tests[i], test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - - int result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - return NULL; -} - - static char* benchmark_counter() { int iter = 10000000; @@ -1423,7 +858,7 @@ static char* benchmark_counter() mu_assert(!ret, "lsb_init() received: %s", ret); clock_t t = clock(); for (int x = 0; x < iter; ++x) { - process(sb, 0); + lsb_test_process(sb, 0); } t = clock() - t; e = lsb_destroy(sb); @@ -1483,7 +918,6 @@ static char* benchmark_deserialize() } - static char* benchmark_lua_types_output() { int iter = 1000000; @@ -1492,11 +926,11 @@ static char* benchmark_lua_types_output() mu_assert(sb, "lsb_create() received: NULL"); lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); clock_t t = clock(); for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); + mu_assert(0 == lsb_test_process(sb, 0), "%s", lsb_get_error(sb)); } t = clock() - t; e = lsb_destroy(sb); @@ -1508,145 +942,6 @@ static char* benchmark_lua_types_output() } -static char* benchmark_cbuf_output() -{ - int iter = 10000; - - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 1), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cbuf_output() %g seconds\n", ((double)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_cbuf_add() -{ - int iter = 1000000; - - lsb_lua_sandbox *sb = lsb_create(NULL, - "lua/circular_buffer_add.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - - double ts = 0; - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); - ts += 1e9; - } - t = clock() - t; - mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_cbuf_add() failed %s", - lsb_get_error(sb)); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cbuf_add() %g seconds\n", ((double)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_bloom_filter_add() -{ - int iter = 1000000; - - lsb_lua_sandbox *sb = lsb_create(NULL, - "lua/bloom_filter_benchmark.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - report(sb, 0); - mu_assert(strcmp("999970", written_data) == 0, "received: %s", written_data); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_bloom_filter_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_bloom_filter_add() %g seconds\n", ((double)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - -static char* benchmark_hyperloglog_add() -{ - int iter = 1000000; - - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/hyperloglog.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - report(sb, 0); - mu_assert(strcmp("1006268", written_data) == 0, "received: %s", written_data); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_hyperloglog_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_hyperloglog_add() %g seconds\n", ((double)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - -static char* benchmark_cuckoo_filter_add() -{ - int iter = 1000000; - - lsb_lua_sandbox *sb = lsb_create(NULL, - "lua/cuckoo_filter_benchmark.lua", test_cfg, - NULL); - mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - report(sb, 0); - mu_assert(strcmp("999985", written_data) == 0, "received: %s", written_data); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_cuckoo_filter_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cuckoo_filter_add() %g seconds\n", ((double)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - static char* benchmark_sax_add() { int iter = 1000000; @@ -1655,15 +950,15 @@ static char* benchmark_sax_add() mu_assert(sb, "lsb_create() received: NULL"); lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &write_output, "write_output"); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); clock_t t = clock(); for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed + mu_assert(0 == lsb_test_process(sb, x), "%s", lsb_get_error(sb)); // test add speed } t = clock() - t; - report(sb, 0); - mu_assert(strcmp("ABEGH", written_data) == 0, "received: %s", written_data); + lsb_test_report(sb, 0); + mu_assert(strcmp("ABEGH", lsb_test_output) == 0, "received: %s", lsb_test_output); mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_sax_add() failed %s", lsb_get_error(sb)); e = lsb_destroy(sb); @@ -1689,10 +984,6 @@ static char* all_tests() mu_run_test(test_simple_error); mu_run_test(test_output); mu_run_test(test_output_errors); - mu_run_test(test_cbuf_errors); - mu_run_test(test_cbuf_core); - mu_run_test(test_cbuf); - mu_run_test(test_cbuf_delta); mu_run_test(test_cjson); mu_run_test(test_cjson_unlimited); mu_run_test(test_errors); @@ -1701,28 +992,17 @@ static char* all_tests() mu_run_test(test_serialize); mu_run_test(test_restore); mu_run_test(test_serialize_failure); - mu_run_test(test_bloom_filter_core); - mu_run_test(test_bloom_filter); - mu_run_test(test_hyperloglog_core); - mu_run_test(test_hyperloglog); mu_run_test(test_struct); mu_run_test(test_sandbox_config); - mu_run_test(test_cuckoo_filter); mu_run_test(test_sax); mu_run_test(test_print); mu_run_test(test_print_disabled); - mu_run_test(test_print_logger); - mu_run_test(test_builtin_modules); + mu_run_test(test_print_lsb_test_logger); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); mu_run_test(benchmark_deserialize); mu_run_test(benchmark_lua_types_output); - mu_run_test(benchmark_cbuf_output); - mu_run_test(benchmark_cbuf_add); - mu_run_test(benchmark_bloom_filter_add); - mu_run_test(benchmark_hyperloglog_add); - mu_run_test(benchmark_cuckoo_filter_add); mu_run_test(benchmark_sax_add); return NULL; diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 26b63d5..939cf87 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -23,9 +23,5 @@ if(LIBM_LIBRARY) target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) endif() -if(ZLIB_FOUND) - target_link_libraries(luasandboxutil ${ZLIB_LIBRARIES}) -endif() - install(TARGETS luasandboxutil DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) add_subdirectory(test) diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 56d4890..8fd6afe 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -471,7 +471,7 @@ void lsb_free_heka_message(lsb_heka_message *m) } -bool lsb_read_heka_field(lsb_heka_message *m, +bool lsb_read_heka_field(const lsb_heka_message *m, lsb_const_string *name, int fi, int ai, diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 2a8895b..02ff853 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -12,8 +12,8 @@ #include -#include "../../test/mu_test.h" #include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/heka_message.h" #define TEST_UUID "\x0a\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 926a8db..cfbda04 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -12,7 +12,7 @@ #include #include -#include "../../test/mu_test.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/heka_message.h" #include "luasandbox/util/heka_message_matcher.h" diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c index 6dc6cec..3ecaabc 100644 --- a/src/util/test/test_input_buffer.c +++ b/src/util/test/test_input_buffer.c @@ -9,10 +9,10 @@ #include #include -#include "../../test/mu_test.h" #include "luasandbox/error.h" -#include "luasandbox/util/input_buffer.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/heka_message.h" +#include "luasandbox/util/input_buffer.h" static char* test_stub() { diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index 27f7225..c6b4069 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -11,7 +11,7 @@ #include #include -#include "../../test/mu_test.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/heka_message.h" diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c index f90bbca..38b2eb6 100644 --- a/src/util/test/test_protobuf.c +++ b/src/util/test/test_protobuf.c @@ -10,11 +10,10 @@ #include #include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/heka_message.h" #include "luasandbox/util/protobuf.h" -#include "../../test/mu_test.h" - static char* test_stub() { return NULL; diff --git a/src/util/test/test_running_stats.c b/src/util/test/test_running_stats.c index 6ce53e3..6a1b4e5 100644 --- a/src/util/test/test_running_stats.c +++ b/src/util/test/test_running_stats.c @@ -8,7 +8,7 @@ #include -#include "../../test/mu_test.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/running_stats.h" #ifdef _MSC_VER diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c index 2b528cc..78d58b2 100644 --- a/src/util/test/test_string_matcher.c +++ b/src/util/test/test_string_matcher.c @@ -8,7 +8,7 @@ #include -#include "../../test/mu_test.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/string_matcher.h" static char* test_stub() diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index ec1e205..01f96e5 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -11,8 +11,8 @@ #include #include -#include "../../test/mu_test.h" #include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" #include "luasandbox/util/util.h" static char* test_stub() @@ -64,69 +64,6 @@ static char* test_lsb_set_tz() } -static char* test_lsb_crc32() -{ - uint32_t checksum = lsb_crc32(NULL, 0); - mu_assert(checksum == 0U, "received: %" PRIu32, checksum); - - checksum = lsb_crc32("", 0); - mu_assert(checksum == 0U, "received: %" PRIu32, checksum); - - checksum = lsb_crc32("foobar", 6); - mu_assert(checksum == 2666930069U, "received: %" PRIu32, checksum); - return NULL; -} - - -static char* test_lsb_ungzip() -{ - size_t alen = 200; - char gz[] = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\x03\x4b\x4c\x1c\x1e\x00\x00\x58\xf0\x9a\x59\xc8\x00\x00\x00"; - - size_t len = 0; - char *ugz = lsb_ungzip(gz, sizeof(gz)-1, 0, &len); - mu_assert(ugz, "lsb_ungzip failed"); - mu_assert(len == alen, "received: %" PRIuSIZE, len); - for (size_t i = 0; i < alen; ++i) { - mu_assert(ugz[i] == 'a', "pos: %" PRIuSIZE " char: %c", i, ugz[i]); - } - free(ugz); - - ugz = lsb_ungzip(gz, sizeof(gz)-1, alen, &len); - mu_assert(ugz, "lsb_ungzip failed"); - mu_assert(len == alen, "received: %" PRIuSIZE, len); - for (size_t i = 0; i < alen; ++i) { - mu_assert(ugz[i] == 'a', "pos: %" PRIuSIZE " char: %c", i, ugz[i]); - } - free(ugz); - - ugz = lsb_ungzip(gz, sizeof(gz)-1, 199, &len); - if (ugz) { - free(ugz); - mu_assert(false, "output larger than max lsb_ungzip succeeded"); - } - - ugz = lsb_ungzip(NULL, 0, alen, &len); - if (ugz) { - free(ugz); - mu_assert(false, "NULL lsb_ungzip succeeded"); - } - - ugz = lsb_ungzip(gz, sizeof(gz)-1, sizeof(gz)-2, &len); - if (ugz) { - free(ugz); - mu_assert(false, "max smaller than input lsb_ungzip succeeded"); - } - - ugz = lsb_ungzip(gz, sizeof(gz)-1, (sizeof(gz)-1) * 2 - 1, &len); - if (ugz) { - free(ugz); - mu_assert(false, "max smaller than 2x input lsb_ungzip succeeded"); - } - return NULL; -} - - static char* benchmark_lsb_get_time() { int iter = 1000000; @@ -177,8 +114,6 @@ static char* all_tests() mu_run_test(test_lsb_lp2); mu_run_test(test_lsb_read_file); mu_run_test(test_lsb_set_tz); - mu_run_test(test_lsb_crc32); - mu_run_test(test_lsb_ungzip); mu_run_test(benchmark_lsb_get_time); mu_run_test(benchmark_lsb_get_timestamp); diff --git a/src/util/util.c b/src/util/util.c index 287dd0b..4599ee4 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -14,10 +14,6 @@ #include #include -#ifdef HAVE_ZLIB -#include -#endif - #ifdef _WIN32 #include #include @@ -162,75 +158,3 @@ bool lsb_set_tz(const char *tz) #endif return true; } - - -#ifdef HAVE_ZLIB -uint32_t lsb_crc32(const char *buf, size_t buf_len) -{ - uint32_t crc = (uint32_t)crc32(0L, Z_NULL, 0); - crc = (uint32_t)crc32(crc, (const Bytef *)buf, buf_len); - return crc; -} - - -char* lsb_ungzip(const char *s, size_t s_len, size_t max_len, size_t *r_len) -{ - if (!s || (max_len && s_len > max_len)) { - return NULL; - } - size_t buf_len = 2 * s_len; - if (max_len && buf_len > max_len) { - buf_len = max_len; - } - unsigned char *buf = malloc(buf_len); - if (!buf) { - return NULL; - } - - z_stream strm; - strm.zalloc = Z_NULL; - strm.zfree = Z_NULL; - strm.opaque = Z_NULL; - strm.avail_in = s_len; - strm.next_in = (unsigned char *)s; - strm.avail_out = buf_len; - strm.next_out = buf; - - int ret = inflateInit2(&strm, 16 + MAX_WBITS); - if (ret != Z_OK) { - free(buf); - return NULL; - } - - do { - if (ret == Z_BUF_ERROR) { - if (max_len && buf_len == max_len) { - ret = Z_MEM_ERROR; - break; - } - buf_len *= 2; - if (max_len && buf_len > max_len) { - buf_len = max_len; - } - unsigned char *tmp = realloc(buf, buf_len); - if (!tmp) { - ret = Z_MEM_ERROR; - break; - } else { - buf = tmp; - strm.avail_out = buf_len - strm.total_out; - strm.next_out = buf + strm.total_out; - } - } - ret = inflate(&strm, Z_FINISH); - } while (ret == Z_BUF_ERROR && strm.avail_in > 0); - - inflateEnd(&strm); - if (ret != Z_STREAM_END) { - free(buf); - return NULL; - } - if (r_len) *r_len = strm.total_out; - return (char *)buf; -} -#endif From 856593a47aae610fab5a6023b88f558a58becfb6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 11 Jul 2016 20:47:52 -0700 Subject: [PATCH 169/235] Move all extensions (sandboxes and modules) to a separate project - the lua_sandbox_extensions project will allow for easier package management, creation and deployment - move the modified Lua code into this project to simply builds and roll everything into libluasandbox (eliminating libluasb) --- CMakeLists.txt | 63 +- README.md | 3 +- cmake/CMakeLists.txt.cjson | 45 - cmake/CMakeLists.txt.lpeg | 39 - cmake/CMakeLists.txt.lua_geoip | 59 - cmake/CMakeLists.txt.lua_openssl | 73 - cmake/CMakeLists.txt.lua_postgres | 37 - cmake/CMakeLists.txt.lua_sec | 52 - cmake/CMakeLists.txt.lua_snappy | 38 - cmake/CMakeLists.txt.sax | 35 - cmake/CMakeLists.txt.struct | 30 - cmake/FindLua.cmake | 4 - cmake/externals.cmake | 244 --- cmake/luasandboxConfig.cmake.in | 3 +- covfn.txt | 358 ++--- docs/heka/analysis.md | 2 - docs/heka/input.md | 4 +- docs/heka/output.md | 4 +- docs/sandbox.md | 14 +- gen_gh_pages.lua | 152 -- include/luasandbox/lauxlib.h | 174 +++ include/luasandbox/lua.h | 389 +++++ include/luasandbox/luaconf.h | 765 ++++++++++ include/luasandbox/lualib.h | 55 + include/luasandbox/test/sandbox.h | 45 +- modules/heka/elasticsearch.lua | 120 -- modules/heka/elasticsearch/payload.lua | 55 - modules/heka/msg_interpolate.lua | 99 -- modules/heka/util.lua | 64 - modules/lpeg/cbufd.lua | 70 - modules/lpeg/common_log_format.lua | 483 ------ modules/lpeg/date_time.lua | 326 ---- modules/lpeg/ip_address.lua | 59 - modules/lpeg/mysql.lua | 129 -- modules/lpeg/postfix.lua | 540 ------- modules/lpeg/syslog.lua | 281 ---- modules/lpeg/syslog_message.lua | 640 -------- modules/lsb/util.lua | 77 - sandboxes/heka/analysis/throughput.lua | 48 - sandboxes/heka/input/heka_kafka.lua | 60 - sandboxes/heka/input/heka_stdin.lua | 32 - sandboxes/heka/input/heka_tcp.lua | 107 -- sandboxes/heka/input/syslog_udp.lua | 99 -- sandboxes/heka/output/debug.lua | 57 - .../heka/output/elasticsearch_bulk_api.lua | 178 --- sandboxes/heka/output/heka_kafka.lua | 63 - sandboxes/heka/output/heka_log_rolling.lua | 53 - sandboxes/heka/output/heka_s3_partition.lua | 281 ---- sandboxes/heka/output/heka_tcp.lua | 84 -- sandboxes/heka/output/heka_tcp_matcher.lua | 117 -- sandboxes/heka/output/inject_payload.lua | 55 - src/CMakeLists.txt | 69 +- src/cli/CMakeLists.txt | 2 +- src/heka/CMakeLists.txt | 2 +- src/heka/test/CMakeLists.txt | 5 +- src/heka/test/lua/encode_message.lua | 16 - src/heka/test/lua/heka_util.lua | 28 - src/heka/test/lua/iim.lua | 21 +- src/heka/test/test_heka_sandbox.c | 42 +- src/lua/lapi.c | 1087 +++++++++++++ src/lua/lapi.h | 16 + src/lua/lauxlib.c | 653 ++++++++ src/lua/lbaselib.c | 657 ++++++++ src/lua/lcode.c | 831 ++++++++++ src/lua/lcode.h | 76 + src/lua/ldblib.c | 398 +++++ src/lua/ldebug.c | 642 ++++++++ src/lua/ldebug.h | 33 + src/lua/ldo.c | 519 +++++++ src/lua/ldo.h | 57 + src/lua/ldump.c | 164 ++ src/lua/lfunc.c | 174 +++ src/lua/lfunc.h | 34 + src/lua/lgc.c | 710 +++++++++ src/lua/lgc.h | 110 ++ src/lua/linit.c | 38 + src/lua/liolib.c | 556 +++++++ src/lua/llex.c | 463 ++++++ src/lua/llex.h | 81 + src/lua/llimits.h | 128 ++ src/lua/lmathlib.c | 263 ++++ src/lua/lmem.c | 86 ++ src/lua/lmem.h | 49 + src/lua/loadlib.c | 686 +++++++++ src/lua/lobject.c | 214 +++ src/lua/lobject.h | 381 +++++ src/lua/lopcodes.c | 102 ++ src/lua/lopcodes.h | 268 ++++ src/lua/loslib.c | 322 ++++ src/lua/lparser.c | 1339 +++++++++++++++++ src/lua/lparser.h | 82 + src/lua/lstate.c | 214 +++ src/lua/lstate.h | 169 +++ src/lua/lstring.c | 111 ++ src/lua/lstring.h | 31 + src/lua/lstrlib.c | 871 +++++++++++ src/lua/ltable.c | 588 ++++++++ src/lua/ltable.h | 40 + src/lua/ltablib.c | 287 ++++ src/lua/ltm.c | 75 + src/lua/ltm.h | 54 + src/lua/lundump.c | 227 +++ src/lua/lundump.h | 36 + src/lua/lvm.c | 767 ++++++++++ src/lua/lvm.h | 36 + src/lua/lzio.c | 82 + src/lua/lzio.h | 67 + src/luasandbox.c | 28 - src/luasandbox_serialize.c | 18 - src/test/CMakeLists.txt | 8 +- src/test/lua/cjson.lua | 31 - src/test/lua/cjson_unlimited.lua | 13 - src/test/lua/decoder.lua | 42 - src/test/lua/lpeg.lua | 18 - src/test/lua/lpeg_cbufd.lua | 30 - src/test/lua/lpeg_clf.lua | 509 ------- src/test/lua/lpeg_date_time.lua | 149 -- src/test/lua/lpeg_ip_address.lua | 170 --- src/test/lua/lpeg_mysql.lua | 289 ---- src/test/lua/lpeg_postfix.lua | 1100 -------------- src/test/lua/lpeg_syslog.lua | 143 -- src/test/lua/lpeg_syslog_message.lua | 354 ----- src/test/lua/no_external_modules.lua | 8 + src/test/lua/output.lua | 9 +- src/test/lua/output_errors.lua | 17 +- src/test/lua/sax.lua | 53 - src/test/lua/sax_benchmark.lua | 24 - src/test/lua/serialize.lua | 13 +- src/test/lua/struct.lua | 11 - src/test/lua/util_test.lua | 59 - src/test/output/serialize.lua51.data | 34 +- src/test/sandbox.c | 4 +- src/test/test_generic_sandbox.c | 302 ++-- src/util/CMakeLists.txt | 2 +- src/util/test/test_output_buffer.c | 1 + 135 files changed, 16700 insertions(+), 8662 deletions(-) delete mode 100644 cmake/CMakeLists.txt.cjson delete mode 100644 cmake/CMakeLists.txt.lpeg delete mode 100644 cmake/CMakeLists.txt.lua_geoip delete mode 100644 cmake/CMakeLists.txt.lua_openssl delete mode 100644 cmake/CMakeLists.txt.lua_postgres delete mode 100644 cmake/CMakeLists.txt.lua_sec delete mode 100644 cmake/CMakeLists.txt.lua_snappy delete mode 100644 cmake/CMakeLists.txt.sax delete mode 100644 cmake/CMakeLists.txt.struct delete mode 100644 cmake/FindLua.cmake delete mode 100644 cmake/externals.cmake create mode 100644 include/luasandbox/lauxlib.h create mode 100644 include/luasandbox/lua.h create mode 100644 include/luasandbox/luaconf.h create mode 100644 include/luasandbox/lualib.h delete mode 100644 modules/heka/elasticsearch.lua delete mode 100644 modules/heka/elasticsearch/payload.lua delete mode 100644 modules/heka/msg_interpolate.lua delete mode 100644 modules/heka/util.lua delete mode 100644 modules/lpeg/cbufd.lua delete mode 100644 modules/lpeg/common_log_format.lua delete mode 100644 modules/lpeg/date_time.lua delete mode 100644 modules/lpeg/ip_address.lua delete mode 100644 modules/lpeg/mysql.lua delete mode 100644 modules/lpeg/postfix.lua delete mode 100644 modules/lpeg/syslog.lua delete mode 100644 modules/lpeg/syslog_message.lua delete mode 100644 modules/lsb/util.lua delete mode 100644 sandboxes/heka/analysis/throughput.lua delete mode 100644 sandboxes/heka/input/heka_kafka.lua delete mode 100644 sandboxes/heka/input/heka_stdin.lua delete mode 100644 sandboxes/heka/input/heka_tcp.lua delete mode 100644 sandboxes/heka/input/syslog_udp.lua delete mode 100644 sandboxes/heka/output/debug.lua delete mode 100644 sandboxes/heka/output/elasticsearch_bulk_api.lua delete mode 100644 sandboxes/heka/output/heka_kafka.lua delete mode 100644 sandboxes/heka/output/heka_log_rolling.lua delete mode 100644 sandboxes/heka/output/heka_s3_partition.lua delete mode 100644 sandboxes/heka/output/heka_tcp.lua delete mode 100644 sandboxes/heka/output/heka_tcp_matcher.lua delete mode 100644 sandboxes/heka/output/inject_payload.lua delete mode 100644 src/heka/test/lua/heka_util.lua create mode 100644 src/lua/lapi.c create mode 100644 src/lua/lapi.h create mode 100644 src/lua/lauxlib.c create mode 100644 src/lua/lbaselib.c create mode 100644 src/lua/lcode.c create mode 100644 src/lua/lcode.h create mode 100644 src/lua/ldblib.c create mode 100644 src/lua/ldebug.c create mode 100644 src/lua/ldebug.h create mode 100644 src/lua/ldo.c create mode 100644 src/lua/ldo.h create mode 100644 src/lua/ldump.c create mode 100644 src/lua/lfunc.c create mode 100644 src/lua/lfunc.h create mode 100644 src/lua/lgc.c create mode 100644 src/lua/lgc.h create mode 100644 src/lua/linit.c create mode 100644 src/lua/liolib.c create mode 100644 src/lua/llex.c create mode 100644 src/lua/llex.h create mode 100644 src/lua/llimits.h create mode 100644 src/lua/lmathlib.c create mode 100644 src/lua/lmem.c create mode 100644 src/lua/lmem.h create mode 100644 src/lua/loadlib.c create mode 100644 src/lua/lobject.c create mode 100644 src/lua/lobject.h create mode 100644 src/lua/lopcodes.c create mode 100644 src/lua/lopcodes.h create mode 100644 src/lua/loslib.c create mode 100644 src/lua/lparser.c create mode 100644 src/lua/lparser.h create mode 100644 src/lua/lstate.c create mode 100644 src/lua/lstate.h create mode 100644 src/lua/lstring.c create mode 100644 src/lua/lstring.h create mode 100644 src/lua/lstrlib.c create mode 100644 src/lua/ltable.c create mode 100644 src/lua/ltable.h create mode 100644 src/lua/ltablib.c create mode 100644 src/lua/ltm.c create mode 100644 src/lua/ltm.h create mode 100644 src/lua/lundump.c create mode 100644 src/lua/lundump.h create mode 100644 src/lua/lvm.c create mode 100644 src/lua/lvm.h create mode 100644 src/lua/lzio.c create mode 100644 src/lua/lzio.h delete mode 100644 src/test/lua/cjson.lua delete mode 100644 src/test/lua/cjson_unlimited.lua delete mode 100644 src/test/lua/decoder.lua delete mode 100644 src/test/lua/lpeg.lua delete mode 100644 src/test/lua/lpeg_cbufd.lua delete mode 100644 src/test/lua/lpeg_clf.lua delete mode 100644 src/test/lua/lpeg_date_time.lua delete mode 100644 src/test/lua/lpeg_ip_address.lua delete mode 100644 src/test/lua/lpeg_mysql.lua delete mode 100644 src/test/lua/lpeg_postfix.lua delete mode 100644 src/test/lua/lpeg_syslog.lua delete mode 100644 src/test/lua/lpeg_syslog_message.lua create mode 100644 src/test/lua/no_external_modules.lua delete mode 100644 src/test/lua/sax.lua delete mode 100644 src/test/lua/sax_benchmark.lua delete mode 100644 src/test/lua/struct.lua delete mode 100644 src/test/lua/util_test.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index 7418805..eab13fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,47 +3,25 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox C) +project(luasandbox VERSION 0.24.0 LANGUAGES C) + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") -set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 24) -set(CPACK_PACKAGE_VERSION_PATCH 0) +set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) +set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) +set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") +set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") include(GNUInstallDirs) if(WIN32) set(INSTALL_CMAKE_DIR cmake) set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) else() - set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/${CMAKE_PROJECT_NAME}/cmake") -endif() - -set(CPACK_DEB_COMPONENT_INSTALL ON) - -set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") -set(CPACK_RPM_COMPONENT_INSTALL ON) -if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") - set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") -else() - set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0") -endif() - -set(CPACK_COMPONENTS_ALL core) - -option(LUA_JIT "Enable LuaJIT" off) -if(LUA_JIT) - add_definitions(-DLUA_JIT) + set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/cmake") endif() -find_library(LIBM_LIBRARY m) - -find_package(Git REQUIRED) -set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") -set(CPACK_RPM_PACKAGE_PROVIDES "libluasb.so.0()(64bit)") - set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(mozsvc) -include(externals) include(CheckFunctionExists) check_function_exists(clock_gettime HAVE_CLOCK_GETTIME) @@ -51,26 +29,25 @@ if(HAVE_CLOCK_GETTIME) add_definitions(-DHAVE_CLOCK_GETTIME) endif() -include_directories("${EP_BASE}/inst/include") -install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION ${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/modules COMPONENT core) -install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR} COMPONENT core) -install(DIRECTORY "${CMAKE_SOURCE_DIR}/sandboxes/" DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/sandboxes COMPONENT core) - include(CMakePackageConfigHelpers) -configure_package_config_file(cmake/luasandboxConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake +configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake INSTALL_DESTINATION ${INSTALL_CMAKE_DIR} PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR) -write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfigVersion.cmake - VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake COMPATIBILITY AnyNewerVersion ) # todo change to SameMajorVersion after 1.0 -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/luasandboxConfigVersion.cmake - DESTINATION ${INSTALL_CMAKE_DIR} COMPONENT core) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + DESTINATION ${INSTALL_CMAKE_DIR}) + +find_library(LIBM_LIBRARY m) +find_package(Git REQUIRED) + +include_directories("${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}/include/luasandbox" ) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR}) add_subdirectory(src) -add_custom_target(copy_includes -COMMAND ${CMAKE_COMMAND} -E copy_directory ${CMAKE_SOURCE_DIR}/include ${EP_BASE}/inst/include) -add_dependencies(${LUA_PROJECT} copy_includes) -add_dependencies(luasandboxutil ${LUA_PROJECT}) add_dependencies(luasandbox luasandboxutil) add_dependencies(luasandboxtest luasandbox) add_dependencies(luasandboxheka luasandboxtest) diff --git a/README.md b/README.md index 1d8aa8d..2eed7a0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -Lua Sandbox Library -------------------- +# Lua Sandbox Library ## Overview diff --git a/cmake/CMakeLists.txt.cjson b/cmake/CMakeLists.txt.cjson deleted file mode 100644 index a0909ad..0000000 --- a/cmake/CMakeLists.txt.cjson +++ /dev/null @@ -1,45 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(cjson C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-Wall -fPIC") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) -set(FPCONV_SOURCES fpconv.c) - -# Handle platforms missing isinf() macro (Eg, some Solaris systems). -include(CheckSymbolExists) -CHECK_SYMBOL_EXISTS(isinf math.h HAVE_ISINF) -if(NOT HAVE_ISINF) - add_definitions(-DUSE_INTERNAL_ISINF) -endif() - -if(WIN32) - # Windows sprintf()/strtod() handle NaN/inf differently. Not supported. - add_definitions(-DDISABLE_INVALID_NUMBERS) -endif() - -add_definitions(-DLUA_SANDBOX -DDIST_VERSION="2.1.0") -add_library(cjson SHARED lua_cjson.c strbuf.c fpconv.c lua_cjson.def) -target_link_libraries(cjson ${LUASANDBOX_LUASB_LIBRARY}) -if(LIBM_LIBRARY) - target_link_libraries(cjson ${LIBM_LIBRARY}) -endif() -install(TARGETS cjson DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lpeg b/cmake/CMakeLists.txt.lpeg deleted file mode 100644 index 2caf5c2..0000000 --- a/cmake/CMakeLists.txt.lpeg +++ /dev/null @@ -1,39 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(lpeg C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) - -file(WRITE lpeg.def "EXPORTS\nluaopen_lpeg\n") -set(LPEG_SRC -lpcap.c -lpcode.c -lpprint.c -lptree.c -lpvm.c -lpeg.def -) - -add_library(lpeg SHARED ${LPEG_SRC}) -target_link_libraries(lpeg ${LUASANDBOX_LUASB_LIBRARY}) -install(TARGETS lpeg DESTINATION ${INSTALL_PATH}) -install(FILES re.lua DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lua_geoip b/cmake/CMakeLists.txt.lua_geoip deleted file mode 100644 index 8f5ab27..0000000 --- a/cmake/CMakeLists.txt.lua_geoip +++ /dev/null @@ -1,59 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(geoip C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC -fms-extensions") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -find_library(GEOIP_LIBRARY GeoIP REQUIRED) - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) - -file(WRITE geoip.def "EXPORTS\nluaopen_geoip") -set(GEOIP_SRC -src/lua-geoip.c -src/database.c -geoip.def -) - -file(WRITE city.def "EXPORTS\nluaopen_city\n") -set(CITY_SRC -src/city.c -src/database.c -city.def -) - -file(WRITE country.def "EXPORTS\nluaopen_country\n") -set(COUNTRY_SRC -src/country.c -src/database.c -country.def -) - -add_library(geoip SHARED ${GEOIP_SRC}) -target_link_libraries(geoip ${LUASANDBOX_LUASB_LIBRARY} ${GEOIP_LIBRARY}) -install(TARGETS geoip DESTINATION ${INSTALL_PATH}) - -add_library(city SHARED ${CITY_SRC}) -target_link_libraries(city ${LUASANDBOX_LUASB_LIBRARY} ${GEOIP_LIBRARY}) -install(TARGETS city DESTINATION ${INSTALL_PATH}/geoip) - -add_library(country SHARED ${COUNTRY_SRC}) -target_link_libraries(country ${LUASANDBOX_LUASB_LIBRARY} ${GEOIP_LIBRARY}) -install(TARGETS country DESTINATION ${INSTALL_PATH}/geoip) diff --git a/cmake/CMakeLists.txt.lua_openssl b/cmake/CMakeLists.txt.lua_openssl deleted file mode 100644 index ca7f74c..0000000 --- a/cmake/CMakeLists.txt.lua_openssl +++ /dev/null @@ -1,73 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(openssl C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-fPIC -DPTHREADS") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -find_package(OpenSSL REQUIRED) - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/deps) - -if (APPLE) - set(OPENSSL_ROOT_DIR /opt/local) -endif() - -file(WRITE luaopenssl.def "EXPORTS\nluaopen_openssl\n") -set(LUAOPENSSL_SRC -src/asn1.c -src/auxiliar.c -src/bio.c -src/callback.c -src/cipher.c -src/cms.c -src/compat.c -src/crl.c -src/csr.c -src/dh.c -src/digest.c -src/dsa.c -src/ec.c -src/engine.c -src/hmac.c -src/lbn.c -src/lhash.c -src/misc.c -src/ocsp.c -src/openssl.c -src/ots.c -src/pkcs12.c -src/pkcs7.c -src/pkey.c -src/rsa.c -src/ssl.c -src/th-lock.c -src/util.c -src/x509.c -src/xalgor.c -src/xattrs.c -src/xexts.c -src/xname.c -src/xstore.c -luaopenssl.def -) - -add_library(openssl SHARED ${LUAOPENSSL_SRC}) -target_link_libraries(openssl ${LUASANDBOX_LUASB_LIBRARY} ${OPENSSL_LIBRARIES}) -install(TARGETS openssl DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.lua_postgres b/cmake/CMakeLists.txt.lua_postgres deleted file mode 100644 index ba8a6ba..0000000 --- a/cmake/CMakeLists.txt.lua_postgres +++ /dev/null @@ -1,37 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(postgres C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -find_package(PostgreSQL REQUIRED) - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox ${PostgreSQL_INCLUDE_DIRS}) - -file(WRITE postgres.def "EXPORTS\nluaopen_postgres") -set(POSTGRES_SRC -src/ls_postgres.c -src/luasql.c -postgres.def -) - -add_library(postgres SHARED ${POSTGRES_SRC}) -target_link_libraries(postgres ${LUASANDBOX_LUASB_LIBRARY} ${PostgreSQL_LIBRARIES}) -install(TARGETS postgres DESTINATION ${INSTALL_PATH}/luasql) diff --git a/cmake/CMakeLists.txt.lua_sec b/cmake/CMakeLists.txt.lua_sec deleted file mode 100644 index 0f0c84f..0000000 --- a/cmake/CMakeLists.txt.lua_sec +++ /dev/null @@ -1,52 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(ssl C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-std=gnu99 -Wall -pedantic -fPIC -D_POSIX_C_SOURCE=199309L") - set(CMAKE_C_FLAGS_RELEASE "-O2") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -find_package(OpenSSL REQUIRED) - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox ${OPENSSL_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/src) - -if (APPLE) - set(OPENSSL_ROOT_DIR /opt/local) -endif() - -set(LUASOCKET_SRC -src/luasocket/buffer.c -src/luasocket/io.c -src/luasocket/timeout.c -src/luasocket/usocket.c -) -add_library(luasocket STATIC ${LUASOCKET_SRC}) -target_compile_definitions(luasocket PRIVATE LUASOCKET_DEBUG) - -file(WRITE luasec.def "EXPORTS\nluaopen_ssl_core\nlua_open_ssl_x509\nluaopen_ssl_context\n") -set(LUASEC_SRC -src/context.c -src/ssl.c -src/x509.c -luasec.def -) - -add_library(ssl SHARED ${LUASEC_SRC}) -target_link_libraries(ssl luasocket ${LUASANDBOX_LUASB_LIBRARY} ${OPENSSL_LIBRARIES}) -install(TARGETS ssl DESTINATION ${INSTALL_PATH}) -install(FILES src/ssl.lua DESTINATION ${INSTALL_PATH}) -install(FILES src/https.lua DESTINATION ${INSTALL_PATH}/ssl) diff --git a/cmake/CMakeLists.txt.lua_snappy b/cmake/CMakeLists.txt.lua_snappy deleted file mode 100644 index 6cd2414..0000000 --- a/cmake/CMakeLists.txt.lua_snappy +++ /dev/null @@ -1,38 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(snappy C CXX) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-fPIC -Wall") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/io_modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) - -file(WRITE snappy.def "EXPORTS\nluaopen_snappy") -set(SNAPPY_SRC -lua-snappy.cc -snappy/snappy.cc -snappy/snappy-c.cc -snappy/snappy-sinksource.cc -snappy/snappy-stubs-internal.cc -snappy.def -) - -add_library(snappy SHARED ${SNAPPY_SRC}) -target_link_libraries(snappy ${LUASANDBOX_LUASB_LIBRARY}) -install(TARGETS snappy DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.sax b/cmake/CMakeLists.txt.sax deleted file mode 100644 index 6839ce6..0000000 --- a/cmake/CMakeLists.txt.sax +++ /dev/null @@ -1,35 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(sax C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3 /WX") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-std=c99 -pedantic -Werror -Wall -Wextra -fPIC") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) -include_directories(${CMAKE_SOURCE_DIR}/include) - -find_library(LIBM_LIBRARY m) - -add_definitions(-DLUA_SANDBOX -DDIST_VERSION="0.5.0") -add_library(sax SHARED src/symtseries.c lua/lua_sax.c lua/lua_sax.def) -target_link_libraries(sax ${LUASANDBOX_LUASB_LIBRARY}) -if(LIBM_LIBRARY) - target_link_libraries(sax ${LIBM_LIBRARY}) -endif() -install(TARGETS sax DESTINATION ${INSTALL_PATH}) diff --git a/cmake/CMakeLists.txt.struct b/cmake/CMakeLists.txt.struct deleted file mode 100644 index 6900d18..0000000 --- a/cmake/CMakeLists.txt.struct +++ /dev/null @@ -1,30 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(struct C) - -if(MSVC) - set(CMAKE_C_FLAGS "/W3") - set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2") -else() - set(CMAKE_C_FLAGS "-D_POSIX_SOURCE -fPIC") - set(CMAKE_C_FLAGS_RELEASE "-O2") - set(CMAKE_SHARED_LIBRARY_SUFFIX ".so") -endif() - -include(GNUInstallDirs) -if(WIN32) - set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) -endif() -set(INSTALL_PATH ${CMAKE_INSTALL_LIBDIR}/luasandbox/modules) -set(CMAKE_SHARED_LIBRARY_PREFIX "") - -include_directories(${LUASANDBOX_INCLUDE_DIR}/luasandbox) - -file(WRITE struct.def "EXPORTS\nluaopen_struct") - -add_library(struct SHARED struct.c struct.def) -target_link_libraries(struct ${LUASANDBOX_LUASB_LIBRARY}) -install(TARGETS struct DESTINATION ${INSTALL_PATH}) diff --git a/cmake/FindLua.cmake b/cmake/FindLua.cmake deleted file mode 100644 index 72a77cb..0000000 --- a/cmake/FindLua.cmake +++ /dev/null @@ -1,4 +0,0 @@ -set(LUA_FOUND TRUE) -set(LUA_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY} ${LUASANDBOX_LUASB_LIBRARY}) -set(LUA_INCLUDE_DIR ${LUASANDBOX_INCLUDE_DIR}/luasandbox) -set(LUA_VERSION_STRING "5.1.5") diff --git a/cmake/externals.cmake b/cmake/externals.cmake deleted file mode 100644 index 5b98a0b..0000000 --- a/cmake/externals.cmake +++ /dev/null @@ -1,244 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -include(ExternalProject) -string(REPLACE ")$" "|INSTALL_ARGS)$" _ep_keywords_ExternalProject_Add ${_ep_keywords_ExternalProject_Add}) -set(EP_BASE ${CMAKE_BINARY_DIR}/ep_base) - -get_filename_component(GIT_PATH ${GIT_EXECUTABLE} PATH) - -set(LUASANDBOX_LUASB_LIBRARY ${EP_BASE}/inst/${CMAKE_INSTALL_LIBDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}luasb${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(LUASANDBOX_UTIL_LIBRARY ${EP_BASE}/../src/util/${CMAKE_SHARED_LIBRARY_PREFIX}luasandboxutil${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(LUASANDBOX_LIBRARY ${EP_BASE}/../src/${CMAKE_SHARED_LIBRARY_PREFIX}luasandbox${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(LUASANDBOX_TEST_LIBRARY ${EP_BASE}/../src/test/${CMAKE_SHARED_LIBRARY_PREFIX}luasandboxtest${CMAKE_SHARED_LIBRARY_SUFFIX}) -set(LUASANDBOX_HEKA_LIBRARY ${EP_BASE}/../src/heka/${CMAKE_SHARED_LIBRARY_PREFIX}luasandboxheka${CMAKE_SHARED_LIBRARY_SUFFIX}) - -set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) -set(SANDBOX_CMAKE_ARGS --DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} --DCMAKE_INSTALL_PREFIX=${EP_BASE}/inst --DLUASANDBOX_INCLUDE_DIR=${EP_BASE}/inst/include --DLUASANDBOX_LUASB_LIBRARY=${LUASANDBOX_LUASB_LIBRARY} --DLUASANDBOX_UTIL_LIBRARY=${LUASANDBOX_UTIL_LIBRARY} --DLUASANDBOX_LIBRARY=${LUASANDBOX_LIBRARY} --DLUASANDBOX_TEST_LIBRARY=${LUASANDBOX_TEST_LIBRARY} --DLUASANDBOX_HEKA_LIBRARY=${LUASANDBOX_HEKA_LIBRARY} --DUSE_RPATH=false # todo remove after the LUA_SOCKET build is updated to https://github.com/diegonehab/luasocket ---no-warn-unused-cli) - -set(INST_ARGS "install/strip") -if(MSVC) - set(INST_ARGS "install") -endif() - -if(LUA_JIT) - message(FATAL_ERROR, "LuaJIT support has not been added back in yet, issue #66") -else() - set(LUA_PROJECT "lua-5_1_5") - externalproject_add( - ${LUA_PROJECT} - GIT_REPOSITORY https://github.com/trink/lua.git - GIT_TAG 308080049143009486221653a490c9e16f711f78 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_library(luasb SHARED IMPORTED) - set_target_properties(luasb PROPERTIES IMPORTED_LOCATION ${LUASANDBOX_LUASB_LIBRARY}) - #install(TARGETS luasb DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core OPTIONAL) - #http://public.kitware.com/Bug/view.php?id=14311&nbn=6 -endif() - -externalproject_add( - lua_lpeg - URL http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-1.0.0.tar.gz - URL_MD5 0aec64ccd13996202ad0c099e2877ece - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lpeg /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_lpeg luasandboxheka) - -## sandbox enhanced modules - -externalproject_add( - lua_bloom_filter - GIT_REPOSITORY https://github.com/mozilla-services/lua_bloom_filter.git - GIT_TAG dbfd90e313be86fe1864fe0e18f6e9d5cb0e6f16 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - TEST_AFTER_INSTALL 1 -) -add_dependencies(lua_bloom_filter luasandboxheka) - -externalproject_add( - lua_circular_buffer - GIT_REPOSITORY https://github.com/mozilla-services/lua_circular_buffer.git - GIT_TAG f86e296bb0fafbc8529d74336cc17ff18d8a9e75 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - TEST_AFTER_INSTALL 1 -) -add_dependencies(lua_circular_buffer luasandboxheka) - -externalproject_add( - lua_hyperloglog - GIT_REPOSITORY https://github.com/mozilla-services/lua_hyperloglog.git - GIT_TAG eb1e27c00c206ceffaab6787b4f54c59d4451a5c - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - TEST_AFTER_INSTALL 1 -) -add_dependencies(lua_hyperloglog luasandboxheka) - -externalproject_add( - lua_cuckoo_filter - GIT_REPOSITORY https://github.com/mozilla-services/lua_cuckoo_filter.git - GIT_TAG a5c031fe70008d0f3d347371a7d7ae68acaba575 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - TEST_AFTER_INSTALL 1 -) -add_dependencies(lua_cuckoo_filter luasandboxheka) - -externalproject_add( - lua_sax - GIT_REPOSITORY https://github.com/trink/symtseries.git - GIT_TAG bd2426c93d0d55977ca7b21354105463b0470a49 - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.sax /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_sax luasandboxheka) - -externalproject_add( - lua_cjson - GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG b24f724a4982531e392a88679b3783fdc5361c5d - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.cjson /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} -) -add_dependencies(lua_cjson luasandboxheka) - -set(OPTIONAL_MODULES LUA_RJSON LUA_STRUCT LUA_KAFKA LUA_SOCKET LUA_SEC LUA_OPENSSL LUA_POSTGRES LUA_GEOIP LUA_SNAPPY) -foreach(module_name IN LISTS OPTIONAL_MODULES) - if(ALL_OPTIONAL_MODULES) - option(${module_name} "Include the ${module_name} module" on) - else() - option(${module_name} "Include the ${module_name} module" off) - endif() -endforeach() - -# Optionally Built modules - -if(LUA_RJSON) - externalproject_add( - lua_rjson - GIT_REPOSITORY https://github.com/mozilla-services/lua_rjson.git - GIT_TAG 664351d4b19011375cef41365000201b2728be20 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - TEST_AFTER_INSTALL 1 - ) - add_dependencies(lua_rjson luasandboxheka) -endif() - -if(LUA_KAFKA) - externalproject_add( - lua_kafka - GIT_REPOSITORY https://github.com/mozilla-services/lua_kafka.git - GIT_TAG 27b63f1e3f56ff641661e983e70e79062511d091 - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - # TEST_AFTER_INSTALL 1 # requires a running kafka installation - ) - add_dependencies(lua_kafka luasandboxheka) -endif() - -if(LUA_STRUCT) - externalproject_add( - lua_struct - URL http://www.inf.puc-rio.br/~roberto/struct/struct-0.2.tar.gz - URL_MD5 99384bf1f54457ec9f796ad0b539d19c - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.struct /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_dependencies(lua_struct luasandboxheka) -endif() - -if(LUA_SOCKET) # todo move off the LuaDist version to https://github.com/diegonehab/luasocket - externalproject_add( - lua_socket - GIT_REPOSITORY https://github.com/LuaDist/luasocket.git - GIT_TAG b97ed47e7a01e0d2523809efe11676333a85dcab - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/FindLua.cmake /cmake - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} -DCMAKE_INSTALL_PREFIX=${EP_BASE}/lua_socket - INSTALL_ARGS ${INST_ARGS} - ) - externalproject_add_step(lua_socket copy_install - COMMAND ${CMAKE_COMMAND} -E copy_directory ${EP_BASE}/lua_socket/lib/lua ${EP_BASE}/inst/lib/luasandbox/io_modules - DEPENDEES install) - add_dependencies(lua_socket luasandboxheka) -endif() - -if(LUA_SEC) - externalproject_add( - lua_sec - GIT_REPOSITORY https://github.com/brunoos/luasec.git - GIT_TAG 20443861ebc3f6498ee7d9c70fbdaa059bec15e1 - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_sec /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_dependencies(lua_sec luasandboxheka) -endif() - -if(LUA_OPENSSL) - externalproject_add( - lua_openssl - GIT_REPOSITORY https://github.com/zhaozg/lua-openssl.git - GIT_TAG 2e8930c5e13b52705ba9c390110e84b1f4748ba9 - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_openssl /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_dependencies(lua_openssl luasandboxheka) -endif() - -if(LUA_POSTGRES) - externalproject_add( - lua_postgres - GIT_REPOSITORY https://github.com/LuaDist/luasql-postgresql.git - GIT_TAG 29a3aa1964aeac93323ec5d1446ac7d32ec700df - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_postgres /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_dependencies(lua_postgres luasandboxheka) -endif() - -if(LUA_GEOIP) - externalproject_add( - lua_geoip - GIT_REPOSITORY https://github.com/agladysh/lua-geoip.git - GIT_TAG a07d261d8a2c7ff854fe6cd72cb8c2e16ec638ff - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_geoip /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_dependencies(lua_geoip luasandboxheka) -endif() - -if(LUA_SNAPPY) - externalproject_add( - lua_snappy - GIT_REPOSITORY https://github.com/forhappy/lua-snappy.git - GIT_TAG 6b4f3f6736857d5c72aa6ed0ec566af6e222278d - UPDATE_COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_LIST_DIR}/CMakeLists.txt.lua_snappy /CMakeLists.txt - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_ARGS ${INST_ARGS} - ) - add_dependencies(lua_snappy luasandboxheka) -endif() diff --git a/cmake/luasandboxConfig.cmake.in b/cmake/luasandboxConfig.cmake.in index 601ba86..f002aa1 100644 --- a/cmake/luasandboxConfig.cmake.in +++ b/cmake/luasandboxConfig.cmake.in @@ -5,10 +5,9 @@ set(LUASANDBOX_VERSION x.y.z) set_and_check(LUASANDBOX_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@") set_and_check(LUASANDBOX_LIB_DIR "@PACKAGE_CMAKE_INSTALL_LIBDIR@") -FIND_LIBRARY(LUASANDBOX_LUASB_LIBRARY NAMES luasb PATHS ${LUASANDBOX_LIB_DIR}) FIND_LIBRARY(LUASANDBOX_UTIL_LIBRARY NAMES luasandboxutil PATHS ${LUASANDBOX_LIB_DIR}) FIND_LIBRARY(LUASANDBOX_LIBRARY NAMES luasandbox PATHS ${LUASANDBOX_LIB_DIR}) FIND_LIBRARY(LUASANDBOX_HEKA_LIBRARY NAMES luasandboxheka PATHS ${LUASANDBOX_LIB_DIR}) FIND_LIBRARY(LUASANDBOX_TEST_LIBRARY NAMES luasandboxtest PATHS ${LUASANDBOX_LIB_DIR}) -set(LUASANDBOX_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY} ${LUASANDBOX_LUASB_LIBRARY}) +set(LUASANDBOX_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY}) diff --git a/covfn.txt b/covfn.txt index 24713b8..5f7c3a2 100644 --- a/covfn.txt +++ b/covfn.txt @@ -1,200 +1,158 @@ -Function Source Line FnCov C/D Coverage --------------------------------------------------------------------------------------------------- ------------------------------ ----- --------------------- -add_table_ref(table_ref_array*,const void*,size_t) ...uasandbox_serialize.c 170 1 / 1 1 / 4 = 25% -consumer_receive(lua_State*) ../src/heka/kafka.c 671 1 / 1 5 / 16 = 31% -restore_global_data(lsb_lua_sandbox*) ...uasandbox_serialize.c 495 1 / 1 7 / 18 = 38% -lsb_create(void*,const char*,const char*,lsb_logger) ../src/luasandbox.c 378 1 / 1 11 / 26 = 42% -serialize_data(lsb_lua_sandbox*,int,lsb_output_buffer*) ...uasandbox_serialize.c 229 1 / 1 8 / 18 = 44% -hsr_new(lua_State*) .../heka/stream_reader.c 21 1 / 1 4 / 8 = 50% -lsb_create_message_match_builder(const char*,const char*) ...eka/message_matcher.c 443 1 / 1 3 / 6 = 50% -msg_delivered(rd_kafka_t*,void*,size_t,int,void*,void*) ../src/heka/kafka.c 56 1 / 1 1 / 2 = 50% -producer_destroy_topic(lua_State*) ../src/heka/kafka.c 370 1 / 1 1 / 2 = 50% -luaopen_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 685 1 / 1 1 / 2 = 50% -instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 114 1 / 1 1 / 2 = 50% -read_config(lua_State*) ../src/luasandbox.c 138 1 / 1 1 / 2 = 50% -set_tz() ../src/luasandbox.c 335 1 / 1 1 / 2 = 50% -heka_encode_message_table(lsb_lua_sandbox*,int) ../src/heka/message.c 881 1 / 1 14 / 24 = 58% -set_random_seed() ../src/luasandbox.c 350 1 / 1 7 / 12 = 58% -lsb_read_file(const char*) ../src/util/util.c 51 1 / 1 6 / 10 = 60% -lsb_heka_create_analysis(void*,const char*,const...r*,const char*,lsb_logger,lsb_heka_im_analysis) ../src/heka/sandbox.c 547 1 / 1 11 / 18 = 61% -lsb_heka_create_output(void*,const char*,const c...st char*,lsb_logger,lsb_heka_update_checkpoint) ../src/heka/sandbox.c 630 1 / 1 11 / 18 = 61% -hj_parse(lua_State*) ...rc/heka/rapidjson.cpp 134 1 / 1 5 / 8 = 62% -hj_remove(lua_State*) ...rc/heka/rapidjson.cpp 616 1 / 1 5 / 8 = 62% -min_expand(MatchState*,const char*,const char*,const char*) ...util/string_matcher.c 171 1 / 1 5 / 8 = 62% -preserve_global_data(lsb_lua_sandbox*) ...uasandbox_serialize.c 366 1 / 1 27 / 42 = 64% -lsb_heka_create_input(void*,const char*,const char*,const char*,lsb_logger,lsb_heka_im_input) ../src/heka/sandbox.c 346 1 / 1 12 / 18 = 66% -producer_poll(lua_State*) ../src/heka/kafka.c 389 1 / 1 4 / 6 = 66% -lsb_init_heka_message(lsb_heka_message*,int) ...c/util/heka_message.c 417 1 / 1 4 / 6 = 66% -process_fields(lua_State*,const char*,const char*) ../src/heka/message.c 93 1 / 1 55 / 82 = 67% -encode_field_array(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 330 1 / 1 19 / 28 = 67% -lsb_ungzip(const char*,size_t,size_t,size_t*) ../src/util/util.c 113 1 / 1 30 / 44 = 68% -load_topic_conf(lua_State*,rd_kafka_topic_conf_t*,int) ../src/heka/kafka.c 154 1 / 1 15 / 22 = 68% -lsb_heka_pm_input(lsb_heka_sandbox*,double,const char*,_Bool) ../src/heka/sandbox.c 513 1 / 1 11 / 16 = 68% -read_string(lua_State*,int,const char*,const char*) ../src/heka/message.c 52 1 / 1 7 / 10 = 70% -check_int(lua_State*,int,const char*,int) ../src/luasandbox.c 191 1 / 1 5 / 7 = 71% -hj_parse_message(lua_State*) ...rc/heka/rapidjson.cpp 483 1 / 1 26 / 36 = 72% -producer_create_topic(lua_State*) ../src/heka/kafka.c 303 1 / 1 8 / 11 = 72% -hj_iter(lua_State*) ...rc/heka/rapidjson.cpp 413 1 / 1 8 / 11 = 72% -encode_fields(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 618 1 / 1 27 / 36 = 75% -serialize_kvp(lsb_lua_sandbox*,serialization_data*,size_t) ...uasandbox_serialize.c 286 1 / 1 18 / 24 = 75% -decode_header(char*,size_t,size_t) ...c/util/heka_message.c 19 1 / 1 12 / 16 = 75% -read_message(lua_State*) ../src/heka/sandbox.c 40 1 / 1 6 / 8 = 75% -hsr_decode_message(lua_State*) .../heka/stream_reader.c 77 1 / 1 6 / 8 = 75% -process_varint(lua_State*,const char*,int,int,const char*,const char*) ../src/heka/message.c 72 1 / 1 3 / 4 = 75% -encode_int(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 287 1 / 1 3 / 4 = 75% -load_op_node(lua_State*,match_node*) ...eka/message_matcher.c 321 1 / 1 3 / 4 = 75% -lsb_destroy_message_match_builder(lsb_message_match_builder*) ...eka/message_matcher.c 480 1 / 1 3 / 4 = 75% -output(lua_State*) ../src/luasandbox.c 122 1 / 1 3 / 4 = 75% -lsb_stop_sandbox(lsb_lua_sandbox*) ../src/luasandbox.c 560 1 / 1 3 / 4 = 75% -lsb_get_output(lsb_lua_sandbox*,size_t*) ...c/luasandbox_output.c 110 1 / 1 3 / 4 = 75% -lsb_clear_heka_message(lsb_heka_message*) ...c/util/heka_message.c 431 1 / 1 3 / 4 = 75% -lsb_outputc(lsb_output_buffer*,char) .../util/output_buffer.c 78 1 / 1 3 / 4 = 75% -lsb_outputs(lsb_output_buffer*,const char*,size_t) .../util/output_buffer.c 146 1 / 1 3 / 4 = 75% -consumer_new(lua_State*) ../src/heka/kafka.c 591 1 / 1 16 / 21 = 76% -encode_field_value(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 403 1 / 1 84 / 110 = 76% -process_message(lsb_heka_sandbox*,lsb_heka_message*,lua_State*,int,_Bool) ../src/heka/sandbox.c 428 1 / 1 30 / 39 = 76% -lsb_outputfd(lsb_output_buffer*,double) .../util/output_buffer.c 177 1 / 1 31 / 40 = 77% -lsb_init(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 460 1 / 1 28 / 36 = 77% -lsb_pcall_setup(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 675 1 / 1 14 / 18 = 77% -producer_new(lua_State*) ../src/heka/kafka.c 238 1 / 1 7 / 9 = 77% -output_heka_json(lua_State*) ...rc/heka/rapidjson.cpp 593 1 / 1 11 / 14 = 78% -producer_send(lua_State*) ../src/heka/kafka.c 410 1 / 1 8 / 10 = 80% -read_string(int,const char*,const char*,lsb_const_string*) ...c/util/heka_message.c 39 1 / 1 8 / 10 = 80% -read_string_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 57 1 / 1 8 / 10 = 80% -read_integer_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 78 1 / 1 8 / 10 = 80% -check_string(lua_State*,int,const char*,const char*) ../src/luasandbox.c 169 1 / 1 4 / 5 = 80% -matchbalance(MatchState*,const char*,const char*) ...util/string_matcher.c 137 1 / 1 13 / 16 = 81% -set_restrictions(lua_State*,lsb_heka_sandbox*) ../src/heka/sandbox.c 256 1 / 1 22 / 27 = 81% -load_conf(lua_State*,rd_kafka_conf_t*,int) ../src/heka/kafka.c 72 1 / 1 18 / 22 = 81% -lsb_serialize_binary(lsb_output_buffer*,const void*,size_t) ...uasandbox_serialize.c 547 1 / 1 9 / 11 = 81% -lsb_outputf(lsb_output_buffer*,const char*,...) .../util/output_buffer.c 92 1 / 1 23 / 28 = 82% -lsb_create_message_matcher(const lsb_message_match_builder*,const char*) ...eka/message_matcher.c 493 1 / 1 24 / 29 = 82% -heka_decode_message(lua_State*) ../src/heka/message.c 693 1 / 1 54 / 65 = 83% -heka_read_message(lua_State*,lsb_heka_message*) ../src/heka/message.c 922 1 / 1 45 / 54 = 83% -hsr_find_message(lua_State*) .../heka/stream_reader.c 106 1 / 1 25 / 30 = 83% -lsb_heka_pm_analysis(lsb_heka_sandbox*,lsb_heka_message*,_Bool) ../src/heka/sandbox.c 611 1 / 1 10 / 12 = 83% -ignore_value_type(lsb_lua_sandbox*,serialization_data*,int,lua_CFunction*) ...uasandbox_serialize.c 114 1 / 1 10 / 12 = 83% -producer_gc(lua_State*) ../src/heka/kafka.c 455 1 / 1 5 / 6 = 83% -hj_parse_schema(lua_State*) ...rc/heka/rapidjson.cpp 108 1 / 1 5 / 6 = 83% -inject_message_analysis(lua_State*) ../src/heka/sandbox.c 135 1 / 1 5 / 6 = 83% -lsb_destroy(lsb_lua_sandbox*) ../src/luasandbox.c 573 1 / 1 5 / 6 = 83% -lsb_pb_output_varint(char*,unsigned long long) ../src/util/protobuf.c 56 1 / 1 5 / 6 = 83% -lsb_output(lsb_lua_sandbox*,int,int,int) ...c/luasandbox_output.c 42 1 / 1 27 / 32 = 84% -heka_encode_message(lua_State*) ../src/heka/message.c 820 1 / 1 11 / 13 = 84% -match_class(int,int) ...util/string_matcher.c 68 1 / 1 11 / 13 = 84% -match(MatchState*,const char*,const char*) ...util/string_matcher.c 183 1 / 1 39 / 46 = 84% -lsb_pb_read_varint(const char*,const char*,long long*) ../src/util/protobuf.c 36 1 / 1 17 / 20 = 85% -inject_message_input(lua_State*) ../src/heka/sandbox.c 56 1 / 1 18 / 21 = 85% -lsb_heka_pm_output(lsb_heka_sandbox*,lsb_heka_message*,void*,_Bool) ../src/heka/sandbox.c 733 1 / 1 12 / 14 = 85% -lsb_expand_input_buffer(lsb_input_buffer*,size_t) ...c/util/input_buffer.c 49 1 / 1 12 / 14 = 85% -numeric_test(match_node*,double) ...eka/message_matcher.c 213 1 / 1 6 / 7 = 85% -add_consumer_topics(lua_State*,heka_kafka_consumer*,int) ../src/heka/kafka.c 528 1 / 1 19 / 22 = 86% -load_sandbox_config(const char*,lsb_logger) ../src/luasandbox.c 222 1 / 1 19 / 22 = 86% -matchbracketclass(int,const char*,const char*) ...util/string_matcher.c 99 1 / 1 19 / 22 = 86% -eval_tree(match_node*,lsb_heka_message*) ...eka/message_matcher.c 297 1 / 1 14 / 16 = 87% -lsb_heka_timer_event(lsb_heka_sandbox*,time_t,_Bool) ../src/heka/sandbox.c 759 1 / 1 14 / 16 = 87% -update_checkpoint(lua_State*) ../src/heka/sandbox.c 223 1 / 1 7 / 8 = 87% -lsb_init_input_buffer(lsb_input_buffer*,size_t) ...c/util/input_buffer.c 20 1 / 1 7 / 8 = 87% -lsb_outputd(lsb_output_buffer*,double) .../util/output_buffer.c 160 1 / 1 7 / 8 = 87% -process_fields(lsb_heka_field*,const char*,const char*) ...c/util/heka_message.c 121 1 / 1 50 / 57 = 87% -copy_table(lua_State*,lua_State*,lsb_logger) ../src/luasandbox.c 262 1 / 1 22 / 25 = 88% -hj_value(lua_State*) ...rc/heka/rapidjson.cpp 380 1 / 1 8 / 9 = 88% -load_expression_node(lua_State*,match_node*) ...eka/message_matcher.c 335 1 / 1 42 / 47 = 89% -lsb_decode_heka_message(lsb_heka_message*,const char*,size_t,lsb_logger) ...c/util/heka_message.c 201 1 / 1 51 / 57 = 89% -inject_payload(lua_State*) ../src/heka/sandbox.c 165 1 / 1 18 / 20 = 90% -check_value(lua_State*) ...rc/heka/rapidjson.cpp 55 1 / 1 9 / 10 = 90% -hj_type(lua_State*) ...rc/heka/rapidjson.cpp 246 1 / 1 9 / 10 = 90% -hj_size(lua_State*) ...rc/heka/rapidjson.cpp 279 1 / 1 9 / 10 = 90% -lsb_add_function(lsb_lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 663 1 / 1 9 / 10 = 90% -lsb_init_output_buffer(lsb_output_buffer*,size_t) .../util/output_buffer.c 27 1 / 1 9 / 10 = 90% -eval_node(match_node*,lsb_heka_message*) ...eka/message_matcher.c 235 1 / 1 31 / 34 = 91% -set_missing_headers(lua_State*,int,lsb_heka_sandbox*) ../src/heka/message.c 32 1 / 1 11 / 12 = 91% -memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 77 1 / 1 11 / 12 = 91% -lsb_read_heka_field(lsb_heka_message*,lsb_const_string*,int,int,lsb_read_value*) ...c/util/heka_message.c 462 1 / 1 24 / 26 = 92% -lsb_find_heka_message(lsb_heka_message*,lsb_input_buffer*,_Bool,size_t*,lsb_logger) ...c/util/heka_message.c 319 1 / 1 30 / 32 = 93% -classend(const char*) ...util/string_matcher.c 43 1 / 1 16 / 17 = 94% -lsb_expand_output_buffer(lsb_output_buffer*,size_t) .../util/output_buffer.c 52 1 / 1 17 / 18 = 94% -string_test(match_node*,lsb_const_string*) ...eka/message_matcher.c 167 1 / 1 18 / 19 = 94% -check_producer(lua_State*,int) ../src/heka/kafka.c 47 1 / 1 0 / 0 -get_topic(lua_State*,heka_kafka_producer*,const char*) ../src/heka/kafka.c 288 1 / 1 2 / 2 = 100% -producer_has_topic(lua_State*) ../src/heka/kafka.c 355 1 / 1 2 / 2 = 100% -luaopen_heka_kafka_producer(lua_State*) ../src/heka/kafka.c 500 1 / 1 0 / 0 -check_consumer(lua_State*,int) ../src/heka/kafka.c 519 1 / 1 0 / 0 -consumer_gc(lua_State*) ../src/heka/kafka.c 720 1 / 1 6 / 6 = 100% -luaopen_heka_kafka_consumer(lua_State*) ../src/heka/kafka.c 745 1 / 1 0 / 0 -encode_string(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 258 1 / 1 2 / 2 = 100% -encode_field_object(lsb_lua_sandbox*,lsb_output_buffer*) ../src/heka/message.c 381 1 / 1 4 / 4 = 100% -lsb_destroy_message_matcher(lsb_message_matcher*) ...eka/message_matcher.c 559 1 / 1 6 / 6 = 100% -lsb_eval_message_matcher(lsb_message_matcher*,lsb_heka_message*) ...eka/message_matcher.c 578 1 / 1 0 / 0 -schema_gc(lua_State*) ...rc/heka/rapidjson.cpp 78 1 / 1 0 / 0 -iter_gc(lua_State*) ...rc/heka/rapidjson.cpp 87 1 / 1 0 / 0 -hj_gc(lua_State*) ...rc/heka/rapidjson.cpp 96 1 / 1 0 / 0 -hj_validate(lua_State*) ...rc/heka/rapidjson.cpp 161 1 / 1 4 / 4 = 100% -hj_find(lua_State*) ...rc/heka/rapidjson.cpp 190 1 / 1 19 / 19 = 100% -hj_make_field(lua_State*) ...rc/heka/rapidjson.cpp 309 1 / 1 2 / 2 = 100% -hj_object_iter(lua_State*) ...rc/heka/rapidjson.cpp 326 1 / 1 4 / 4 = 100% -hj_array_iter(lua_State*) ...rc/heka/rapidjson.cpp 352 1 / 1 4 / 4 = 100% -read_message(lua_State*,int,lsb_heka_message*) ...rc/heka/rapidjson.cpp 457 1 / 1 14 / 14 = 100% -OutputBufferWrapper::OutputBufferWrapper(lsb_output_buffer*) ...rc/heka/rapidjson.cpp 566 1 / 1 0 / 0 -OutputBufferWrapper::Put(Ch) ...rc/heka/rapidjson.cpp 578 1 / 1 2 / 2 = 100% -OutputBufferWrapper::Flush() ...rc/heka/rapidjson.cpp 583 1 / 1 0 / 0 -OutputBufferWrapper::GetError() ...rc/heka/rapidjson.cpp 584 1 / 1 0 / 0 -lsb_heka_stop_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 709 1 / 1 0 / 0 -lsb_heka_terminate_sandbox(lsb_heka_sandbox*,const char*) ../src/heka/sandbox.c 715 1 / 1 0 / 0 -lsb_heka_destroy_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 721 1 / 1 2 / 2 = 100% -lsb_heka_get_error(lsb_heka_sandbox*) ../src/heka/sandbox.c 800 1 / 1 2 / 2 = 100% -lsb_heka_get_lua_file(lsb_heka_sandbox*) ../src/heka/sandbox.c 806 1 / 1 2 / 2 = 100% -lsb_heka_get_stats(lsb_heka_sandbox*) ../src/heka/sandbox.c 812 1 / 1 2 / 2 = 100% -lsb_heka_is_running(lsb_heka_sandbox*) ../src/heka/sandbox.c 833 1 / 1 4 / 4 = 100% -check_hsr(lua_State*,int) .../heka/stream_reader.c 68 1 / 1 0 / 0 -hsr_read_message(lua_State*) .../heka/stream_reader.c 192 1 / 1 6 / 6 = 100% -hsr_gc(lua_State*) .../heka/stream_reader.c 204 1 / 1 0 / 0 -luaopen_heka_stream_reader(lua_State*) .../heka/stream_reader.c 231 1 / 1 0 / 0 -libsize(const luaL_Reg*) ../src/luasandbox.c 43 1 / 1 2 / 2 = 100% -preload_modules(lua_State*) ../src/luasandbox.c 50 1 / 1 2 / 2 = 100% -instruction_usage(lsb_lua_sandbox*) ../src/luasandbox.c 108 1 / 1 0 / 0 -unprotected_panic(lua_State*) ../src/luasandbox.c 152 1 / 1 0 / 0 -get_usage_config(lua_State*,int,const char*) ../src/luasandbox.c 160 1 / 1 0 / 0 -stop_hook(lua_State*,lua_Debug*) ../src/luasandbox.c 552 1 / 1 0 / 0 -lsb_usage(lsb_lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 596 1 / 1 10 / 10 = 100% -lsb_get_error(lsb_lua_sandbox*) ../src/luasandbox.c 620 1 / 1 2 / 2 = 100% -lsb_set_error(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 626 1 / 1 4 / 4 = 100% -lsb_get_lua(lsb_lua_sandbox*) ../src/luasandbox.c 639 1 / 1 2 / 2 = 100% -lsb_get_lua_file(lsb_lua_sandbox*) ../src/luasandbox.c 645 1 / 1 2 / 2 = 100% -lsb_get_parent(lsb_lua_sandbox*) ../src/luasandbox.c 651 1 / 1 2 / 2 = 100% -lsb_get_state(lsb_lua_sandbox*) ../src/luasandbox.c 657 1 / 1 2 / 2 = 100% -lsb_pcall_teardown(lsb_lua_sandbox*) ../src/luasandbox.c 699 1 / 1 4 / 4 = 100% -lsb_terminate(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 713 1 / 1 6 / 6 = 100% -lsb_add_output_function(lua_State*,lua_CFunction) ...c/luasandbox_output.c 22 1 / 1 0 / 0 -lsb_get_output_function(lua_State*,int) ...c/luasandbox_output.c 30 1 / 1 0 / 0 -get_serialize_function(lua_State*,int) ...uasandbox_serialize.c 92 1 / 1 0 / 0 -find_table_ref(table_ref_array*,const void*) ...uasandbox_serialize.c 149 1 / 1 4 / 4 = 100% -get_preservation_version(lua_State*) ...uasandbox_serialize.c 195 1 / 1 4 / 4 = 100% -serialize_table(lsb_lua_sandbox*,serialization_data*,size_t) ...uasandbox_serialize.c 214 1 / 1 6 / 6 = 100% -file_exists(const char*) ...uasandbox_serialize.c 484 1 / 1 2 / 2 = 100% -lsb_add_serialize_function(lua_State*,lua_CFunction) ...uasandbox_serialize.c 539 1 / 1 0 / 0 -lsb_serialize_double(lsb_output_buffer*,double) ...uasandbox_serialize.c 574 1 / 1 6 / 6 = 100% -read_double_value(const char*,const char*,int,lsb_read_value*) ...c/util/heka_message.c 97 1 / 1 2 / 2 = 100% -process_varint(int,const char*,const char*,long long*) ...c/util/heka_message.c 110 1 / 1 4 / 4 = 100% -lsb_free_heka_message(lsb_heka_message*) ...c/util/heka_message.c 451 1 / 1 2 / 2 = 100% -lsb_write_heka_uuid(lsb_output_buffer*,const char*,size_t) ...c/util/heka_message.c 507 1 / 1 22 / 22 = 100% -lsb_free_input_buffer(lsb_input_buffer*) ...c/util/input_buffer.c 36 1 / 1 2 / 2 = 100% -lsb_free_output_buffer(lsb_output_buffer*) .../util/output_buffer.c 42 1 / 1 2 / 2 = 100% -lsb_pb_read_key(const char*,int*,int*) ../src/util/protobuf.c 15 1 / 1 8 / 8 = 100% -lsb_pb_write_key(lsb_output_buffer*,unsigned char,unsigned char) ../src/util/protobuf.c 25 1 / 1 2 / 2 = 100% -lsb_pb_write_varint(lsb_output_buffer*,unsigned long long) ../src/util/protobuf.c 75 1 / 1 2 / 2 = 100% -lsb_pb_write_bool(lsb_output_buffer*,int) ../src/util/protobuf.c 85 1 / 1 4 / 4 = 100% -lsb_pb_write_double(lsb_output_buffer*,double) ../src/util/protobuf.c 99 1 / 1 2 / 2 = 100% -lsb_pb_write_string(lsb_output_buffer*,char,const char*,size_t) ../src/util/protobuf.c 114 1 / 1 6 / 6 = 100% -lsb_pb_update_field_length(lsb_output_buffer*,size_t) ../src/util/protobuf.c 127 1 / 1 8 / 8 = 100% -lsb_init_running_stats(lsb_running_stats*) .../util/running_stats.c 13 1 / 1 0 / 0 -lsb_update_running_stats(lsb_running_stats*,double) .../util/running_stats.c 21 1 / 1 4 / 4 = 100% -lsb_sd_running_stats(lsb_running_stats*) .../util/running_stats.c 37 1 / 1 2 / 2 = 100% -lsb_init_const_string(lsb_const_string*) ../src/util/string.c 11 1 / 1 0 / 0 -singlematch(int,const char*,const char*) ...util/string_matcher.c 119 1 / 1 4 / 4 = 100% -max_expand(MatchState*,const char*,const char*,const char*) ...util/string_matcher.c 156 1 / 1 10 / 10 = 100% -lsb_string_match(const char*,size_t,const char*) ...util/string_matcher.c 261 1 / 1 10 / 10 = 100% -lsb_lp2(unsigned long long) ../src/util/util.c 37 1 / 1 2 / 2 = 100% -lsb_get_time() ../src/util/util.c 77 1 / 1 0 / 0 --------------------------------------------------------------------------------------------------- ------------------------------ ----- --------------------- -Total 100% 1874 / 2337 = 80% +Function Source Line FnCov C/D Coverage +------------------------------------------------------------------------------------------------ -------------------------------- ----- --------------------- +add_table_ref(table_ref_array*,const void*,size_t) .../luasandbox_serialize.c 171 1 / 1 1 / 4 = 25% +lsb_create(void*,const char*,const char*,lsb_logger*) ../src/luasandbox.c 428 1 / 1 16 / 42 = 38% +restore_global_data(lsb_lua_sandbox*) .../luasandbox_serialize.c 488 1 / 1 7 / 18 = 38% +lsb_heka_create_input(void*,const char*,const char*,const char*,lsb_logger*,lsb_heka_im_input) ../src/heka/sandbox.c 407 1 / 1 14 / 34 = 41% +lsb_heka_create_analysis(void*,const char*,cons...,const char*,lsb_logger*,lsb_heka_im_analysis) ../src/heka/sandbox.c 598 1 / 1 14 / 34 = 41% +lsb_heka_create_output(void*,const char*,const ... char*,lsb_logger*,lsb_heka_update_checkpoint) ../src/heka/sandbox.c 689 1 / 1 14 / 34 = 41% +serialize_data(lsb_lua_sandbox*,int,lsb_output_buffer*) .../luasandbox_serialize.c 230 1 / 1 9 / 20 = 45% +heka_create_stream_reader(lua_State*) ...rc/heka/stream_reader.c 174 1 / 1 5 / 10 = 50% +lsb_get_output(lsb_lua_sandbox*,size_t*) ../src/luasandbox_output.c 110 1 / 1 2 / 4 = 50% +instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 119 1 / 1 1 / 2 = 50% +read_config(lua_State*) ../src/luasandbox.c 201 1 / 1 1 / 2 = 50% +set_random_seed() ../src/luasandbox.c 400 1 / 1 7 / 12 = 58% +lsb_read_file(const char*) ../src/util/util.c 49 1 / 1 6 / 10 = 60% +heka_encode_message_table(lsb_lua_sandbox*,int) ../src/heka/message.c 874 1 / 1 16 / 26 = 61% +min_expand(MatchState*,const char*,const char*,const char*) ...c/util/string_matcher.c 171 1 / 1 5 / 8 = 62% +encode_field_value(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 399 1 / 1 70 / 110 = 63% +preserve_global_data(lsb_lua_sandbox*) .../luasandbox_serialize.c 368 1 / 1 27 / 42 = 64% +lsb_init_heka_message(lsb_heka_message*,int) ../src/util/heka_message.c 429 1 / 1 4 / 6 = 66% +process_fields(lua_State*,const char*,const char*) ../src/heka/message.c 89 1 / 1 55 / 82 = 67% +encode_field_array(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 326 1 / 1 19 / 28 = 67% +lsb_heka_pm_input(lsb_heka_sandbox*,double,const char*,_Bool) ../src/heka/sandbox.c 564 1 / 1 11 / 16 = 68% +set_restrictions(lua_State*,lsb_heka_sandbox*) ../src/heka/sandbox.c 308 1 / 1 20 / 29 = 68% +read_string(lua_State*,int,const char*,const char*) ../src/heka/message.c 48 1 / 1 7 / 10 = 70% +lsb_init(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 528 1 / 1 27 / 38 = 71% +check_int(lua_State*,int,const char*,int) ../src/luasandbox.c 254 1 / 1 5 / 7 = 71% +lsb_outputfd(lsb_output_buffer*,double) ...rc/util/output_buffer.c 177 1 / 1 29 / 40 = 72% +encode_fields(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 614 1 / 1 27 / 36 = 75% +serialize_kvp(lsb_lua_sandbox*,serialization_data*,size_t) .../luasandbox_serialize.c 288 1 / 1 18 / 24 = 75% +decode_header(char*,size_t,size_t) ../src/util/heka_message.c 25 1 / 1 12 / 16 = 75% +read_message(lua_State*) ../src/heka/sandbox.c 41 1 / 1 6 / 8 = 75% +mm_eval(lua_State*) ../src/heka/sandbox.c 74 1 / 1 6 / 8 = 75% +hsr_decode_message(lua_State*) ...rc/heka/stream_reader.c 27 1 / 1 6 / 8 = 75% +process_varint(lua_State*,const char*,int,int,const char*,const char*) ../src/heka/message.c 68 1 / 1 3 / 4 = 75% +encode_int(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 283 1 / 1 3 / 4 = 75% +output(lua_State*) ../src/luasandbox.c 127 1 / 1 3 / 4 = 75% +lsb_stop_sandbox(lsb_lua_sandbox*) ../src/luasandbox.c 621 1 / 1 3 / 4 = 75% +lsb_clear_heka_message(lsb_heka_message*) ../src/util/heka_message.c 443 1 / 1 3 / 4 = 75% +lsb_outputc(lsb_output_buffer*,char) ...rc/util/output_buffer.c 78 1 / 1 3 / 4 = 75% +lsb_outputs(lsb_output_buffer*,const char*,size_t) ...rc/util/output_buffer.c 146 1 / 1 3 / 4 = 75% +lsb_set_tz(const char*) ../src/util/util.c 143 1 / 1 3 / 4 = 75% +lsb_decode_heka_message(lsb_heka_message*,const char*,size_t,lsb_logger*) ../src/util/heka_message.c 207 1 / 1 59 / 77 = 76% +process_message(lsb_heka_sandbox*,lsb_heka_message*,lua_State*,int,_Bool) ../src/heka/sandbox.c 478 1 / 1 33 / 43 = 76% +lsb_pcall_setup(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 721 1 / 1 14 / 18 = 77% +read_string(int,const char*,const char*,lsb_const_string*) ../src/util/heka_message.c 45 1 / 1 8 / 10 = 80% +read_string_value(const char*,const char*,int,lsb_read_value*) ../src/util/heka_message.c 63 1 / 1 8 / 10 = 80% +read_integer_value(const char*,const char*,int,lsb_read_value*) ../src/util/heka_message.c 84 1 / 1 8 / 10 = 80% +check_string(lua_State*,int,const char*,const char*) ../src/luasandbox.c 232 1 / 1 4 / 5 = 80% +matchbalance(MatchState*,const char*,const char*) ...c/util/string_matcher.c 137 1 / 1 13 / 16 = 81% +lsb_outputf(lsb_output_buffer*,const char*,...) ...rc/util/output_buffer.c 92 1 / 1 23 / 28 = 82% +heka_read_message(lua_State*,lsb_heka_message*) ../src/heka/message.c 925 1 / 1 48 / 58 = 82% +heka_decode_message(lua_State*) ../src/heka/message.c 689 1 / 1 54 / 65 = 83% +hsr_find_message(lua_State*) ...rc/heka/stream_reader.c 56 1 / 1 25 / 30 = 83% +lsb_heka_timer_event(lsb_heka_sandbox*,time_t,_Bool) ../src/heka/sandbox.c 811 1 / 1 15 / 18 = 83% +set_missing_headers(lua_State*,int,lsb_heka_sandbox*) ../src/heka/message.c 28 1 / 1 10 / 12 = 83% +lsb_heka_pm_analysis(lsb_heka_sandbox*,lsb_heka_message*,_Bool) ../src/heka/sandbox.c 670 1 / 1 10 / 12 = 83% +ignore_value_type(lsb_lua_sandbox*,serialization_data*,int,lua_CFunction*) .../luasandbox_serialize.c 115 1 / 1 10 / 12 = 83% +inject_message_analysis(lua_State*) ../src/heka/sandbox.c 191 1 / 1 5 / 6 = 83% +lsb_destroy(lsb_lua_sandbox*) ../src/luasandbox.c 634 1 / 1 5 / 6 = 83% +lsb_pb_output_varint(char*,unsigned long long) ../src/util/protobuf.c 56 1 / 1 5 / 6 = 83% +lsb_output(lsb_lua_sandbox*,int,int,int) ../src/luasandbox_output.c 42 1 / 1 27 / 32 = 84% +heka_encode_message(lua_State*) ../src/heka/message.c 816 1 / 1 11 / 13 = 84% +match_class(int,int) ...c/util/string_matcher.c 68 1 / 1 11 / 13 = 84% +match(MatchState*,const char*,const char*) ...c/util/string_matcher.c 183 1 / 1 39 / 46 = 84% +inject_message_input(lua_State*) ../src/heka/sandbox.c 112 1 / 1 18 / 21 = 85% +lsb_heka_pm_output(lsb_heka_sandbox*,lsb_heka_message*,void*,_Bool) ../src/heka/sandbox.c 785 1 / 1 12 / 14 = 85% +lsb_expand_input_buffer(lsb_input_buffer*,size_t) ../src/util/input_buffer.c 49 1 / 1 12 / 14 = 85% +numeric_test(match_node*,double) .../heka_message_matcher.c 67 1 / 1 6 / 7 = 85% +lsb_find_heka_message(lsb_heka_message*,lsb_input_buffer*,_Bool,size_t*,lsb_logger*) ../src/util/heka_message.c 331 1 / 1 31 / 36 = 86% +output_print(lua_State*) ../src/luasandbox.c 143 1 / 1 19 / 22 = 86% +load_sandbox_config(const char*,lsb_logger*) ../src/luasandbox.c 285 1 / 1 19 / 22 = 86% +matchbracketclass(int,const char*,const char*) ...c/util/string_matcher.c 99 1 / 1 19 / 22 = 86% +update_checkpoint(lua_State*) ../src/heka/sandbox.c 275 1 / 1 7 / 8 = 87% +lsb_init_input_buffer(lsb_input_buffer*,size_t) ../src/util/input_buffer.c 20 1 / 1 7 / 8 = 87% +lsb_outputd(lsb_output_buffer*,double) ...rc/util/output_buffer.c 160 1 / 1 7 / 8 = 87% +process_fields(lsb_heka_field*,const char*,const char*) ../src/util/heka_message.c 127 1 / 1 50 / 57 = 87% +copy_table(lua_State*,lua_State*,lsb_logger*) ../src/luasandbox.c 327 1 / 1 22 / 25 = 88% +lsb_expand_output_buffer(lsb_output_buffer*,size_t) ...rc/util/output_buffer.c 52 1 / 1 16 / 18 = 88% +inject_payload(lua_State*) ../src/heka/sandbox.c 216 1 / 1 18 / 20 = 90% +lsb_add_function(lsb_lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 710 1 / 1 9 / 10 = 90% +lsb_init_output_buffer(lsb_output_buffer*,size_t) ...rc/util/output_buffer.c 27 1 / 1 9 / 10 = 90% +eval_node(match_node*,lsb_heka_message*) .../heka_message_matcher.c 89 1 / 1 31 / 34 = 91% +eval_tree(match_node*,match_node*,lsb_heka_message*) .../heka_message_matcher.c 151 1 / 1 22 / 24 = 91% +memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 83 1 / 1 11 / 12 = 91% +lsb_read_heka_field(const lsb_heka_message*,lsb_const_string*,int,int,lsb_read_value*) ../src/util/heka_message.c 474 1 / 1 24 / 26 = 92% +classend(const char*) ...c/util/string_matcher.c 43 1 / 1 16 / 17 = 94% +string_test(match_node*,lsb_const_string*) .../heka_message_matcher.c 21 1 / 1 18 / 19 = 94% +encode_string(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 254 1 / 1 2 / 2 = 100% +encode_field_object(lsb_lua_sandbox*,lsb_output_buffer*) ../src/heka/message.c 377 1 / 1 4 / 4 = 100% +mm_check(lua_State*) ../src/heka/sandbox.c 58 1 / 1 0 / 0 +mm_gc(lua_State*) ../src/heka/sandbox.c 66 1 / 1 0 / 0 +mm_create(lua_State*) ../src/heka/sandbox.c 91 1 / 1 4 / 4 = 100% +lsb_heka_stop_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 761 1 / 1 0 / 0 +lsb_heka_terminate_sandbox(lsb_heka_sandbox*,const char*) ../src/heka/sandbox.c 767 1 / 1 0 / 0 +lsb_heka_destroy_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 773 1 / 1 2 / 2 = 100% +lsb_heka_get_error(lsb_heka_sandbox*) ../src/heka/sandbox.c 853 1 / 1 2 / 2 = 100% +lsb_heka_get_lua_file(lsb_heka_sandbox*) ../src/heka/sandbox.c 859 1 / 1 2 / 2 = 100% +lsb_heka_get_stats(lsb_heka_sandbox*) ../src/heka/sandbox.c 865 1 / 1 2 / 2 = 100% +lsb_heka_is_running(lsb_heka_sandbox*) ../src/heka/sandbox.c 886 1 / 1 4 / 4 = 100% +lsb_heka_get_message(lsb_heka_sandbox*) ../src/heka/sandbox.c 894 1 / 1 2 / 2 = 100% +lsb_heka_get_type(lsb_heka_sandbox*) ../src/heka/sandbox.c 901 1 / 1 2 / 2 = 100% +check_hsr(lua_State*,int) ...rc/heka/stream_reader.c 18 1 / 1 0 / 0 +hsr_read_message(lua_State*) ...rc/heka/stream_reader.c 142 1 / 1 6 / 6 = 100% +hsr_gc(lua_State*) ...rc/heka/stream_reader.c 154 1 / 1 0 / 0 +libsize(const luaL_Reg*) ../src/luasandbox.c 50 1 / 1 2 / 2 = 100% +preload_modules(lua_State*) ../src/luasandbox.c 57 1 / 1 2 / 2 = 100% +instruction_usage(lsb_lua_sandbox*) ../src/luasandbox.c 113 1 / 1 0 / 0 +unprotected_panic(lua_State*) ../src/luasandbox.c 215 1 / 1 0 / 0 +get_int(lua_State*,int,const char*) ../src/luasandbox.c 223 1 / 1 0 / 0 +stop_hook(lua_State*,lua_Debug*) ../src/luasandbox.c 613 1 / 1 0 / 0 +lsb_usage(lsb_lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 657 1 / 1 10 / 10 = 100% +lsb_get_error(lsb_lua_sandbox*) ../src/luasandbox.c 667 1 / 1 2 / 2 = 100% +lsb_set_error(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 673 1 / 1 4 / 4 = 100% +lsb_get_lua(lsb_lua_sandbox*) ../src/luasandbox.c 686 1 / 1 2 / 2 = 100% +lsb_get_lua_file(lsb_lua_sandbox*) ../src/luasandbox.c 692 1 / 1 2 / 2 = 100% +lsb_get_parent(lsb_lua_sandbox*) ../src/luasandbox.c 698 1 / 1 2 / 2 = 100% +lsb_get_state(lsb_lua_sandbox*) ../src/luasandbox.c 704 1 / 1 2 / 2 = 100% +lsb_pcall_teardown(lsb_lua_sandbox*) ../src/luasandbox.c 745 1 / 1 4 / 4 = 100% +lsb_terminate(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 759 1 / 1 6 / 6 = 100% +lsb_add_output_function(lua_State*,lua_CFunction) ../src/luasandbox_output.c 22 1 / 1 0 / 0 +lsb_get_output_function(lua_State*,int) ../src/luasandbox_output.c 30 1 / 1 0 / 0 +get_serialize_function(lua_State*,int) .../luasandbox_serialize.c 93 1 / 1 0 / 0 +find_table_ref(table_ref_array*,const void*) .../luasandbox_serialize.c 150 1 / 1 4 / 4 = 100% +get_preservation_version(lua_State*) .../luasandbox_serialize.c 196 1 / 1 4 / 4 = 100% +serialize_table(lsb_lua_sandbox*,serialization_data*,size_t) .../luasandbox_serialize.c 215 1 / 1 6 / 6 = 100% +file_exists(const char*) .../luasandbox_serialize.c 477 1 / 1 2 / 2 = 100% +lsb_add_serialize_function(lua_State*,lua_CFunction) .../luasandbox_serialize.c 523 1 / 1 0 / 0 +lsb_serialize_binary(lsb_output_buffer*,const void*,size_t) .../luasandbox_serialize.c 531 1 / 1 11 / 11 = 100% +lsb_serialize_double(lsb_output_buffer*,double) .../luasandbox_serialize.c 558 1 / 1 6 / 6 = 100% +read_double_value(const char*,const char*,int,lsb_read_value*) ../src/util/heka_message.c 103 1 / 1 2 / 2 = 100% +process_varint(int,const char*,const char*,long long*) ../src/util/heka_message.c 116 1 / 1 4 / 4 = 100% +lsb_free_heka_message(lsb_heka_message*) ../src/util/heka_message.c 463 1 / 1 2 / 2 = 100% +lsb_write_heka_uuid(lsb_output_buffer*,const char*,size_t) ../src/util/heka_message.c 519 1 / 1 22 / 22 = 100% +lsb_destroy_message_matcher(lsb_message_matcher*) .../heka_message_matcher.c 175 1 / 1 6 / 6 = 100% +lsb_eval_message_matcher(lsb_message_matcher*,lsb_heka_message*) .../heka_message_matcher.c 195 1 / 1 0 / 0 +lsb_free_input_buffer(lsb_input_buffer*) ../src/util/input_buffer.c 36 1 / 1 2 / 2 = 100% +lsb_free_output_buffer(lsb_output_buffer*) ...rc/util/output_buffer.c 42 1 / 1 2 / 2 = 100% +lsb_pb_read_key(const char*,int*,int*) ../src/util/protobuf.c 15 1 / 1 8 / 8 = 100% +lsb_pb_write_key(lsb_output_buffer*,unsigned char,unsigned char) ../src/util/protobuf.c 25 1 / 1 2 / 2 = 100% +lsb_pb_read_varint(const char*,const char*,long long*) ../src/util/protobuf.c 36 1 / 1 22 / 22 = 100% +lsb_pb_write_varint(lsb_output_buffer*,unsigned long long) ../src/util/protobuf.c 75 1 / 1 2 / 2 = 100% +lsb_pb_write_bool(lsb_output_buffer*,int) ../src/util/protobuf.c 85 1 / 1 4 / 4 = 100% +lsb_pb_write_double(lsb_output_buffer*,double) ../src/util/protobuf.c 99 1 / 1 2 / 2 = 100% +lsb_pb_write_string(lsb_output_buffer*,char,const char*,size_t) ../src/util/protobuf.c 114 1 / 1 6 / 6 = 100% +lsb_pb_update_field_length(lsb_output_buffer*,size_t) ../src/util/protobuf.c 127 1 / 1 8 / 8 = 100% +lsb_init_running_stats(lsb_running_stats*) ...rc/util/running_stats.c 13 1 / 1 0 / 0 +lsb_update_running_stats(lsb_running_stats*,double) ...rc/util/running_stats.c 21 1 / 1 4 / 4 = 100% +lsb_sd_running_stats(lsb_running_stats*) ...rc/util/running_stats.c 37 1 / 1 2 / 2 = 100% +lsb_init_const_string(lsb_const_string*) ../src/util/string.c 11 1 / 1 0 / 0 +singlematch(int,const char*,const char*) ...c/util/string_matcher.c 119 1 / 1 4 / 4 = 100% +max_expand(MatchState*,const char*,const char*,const char*) ...c/util/string_matcher.c 156 1 / 1 10 / 10 = 100% +lsb_string_match(const char*,size_t,const char*) ...c/util/string_matcher.c 261 1 / 1 10 / 10 = 100% +lsb_lp2(unsigned long long) ../src/util/util.c 35 1 / 1 2 / 2 = 100% +lsb_get_time() ../src/util/util.c 75 1 / 1 0 / 0 +lsb_get_timestamp() ../src/util/util.c 114 1 / 1 0 / 0 +------------------------------------------------------------------------------------------------ -------------------------------- ----- --------------------- +Total 100% 1568 / 2029 = 77% diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index 0d9b453..ad1e68d 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -1,7 +1,5 @@ # Analysis Sandbox Interface -[Available Analysis Sandboxes](/lua_sandbox/sandboxes/heka/analysis/index.html) - ## Recommendations Since the sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the diff --git a/docs/heka/input.md b/docs/heka/input.md index de627f1..0785f6b 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -1,7 +1,5 @@ # Input Sandbox Interface -[Available Input Sandboxes](/lua_sandbox/sandboxes/heka/input/index.html) - ## Recommendations Since the sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the @@ -58,7 +56,7 @@ to the values provide in the message table, if no value is provided it defaults to the appropriate configuration value. *Arguments* -* msg ([Heka message table](message.html), [Heka stream reader](stream_reader.html) or Heka protobuf string) +* msg ([Heka message table](message.html), [Heka stream reader](#heka-stream-reader-methods) or Heka protobuf string) * checkpoint (optional: number, string) - checkpoint to be returned in the `process_message` call *Return* diff --git a/docs/heka/output.md b/docs/heka/output.md index 758caed..a467cb4 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -1,7 +1,5 @@ # Output Sandbox Interface -[Available Output Sandboxes](/lua_sandbox/sandboxes/heka/output/index.html) - ## Recommendations Since the sandbox does not run in isolation there are some expectations of how the host infrastructure behaves. The current recommendation are based on the @@ -107,7 +105,7 @@ Returns a Heka protocol buffer message matcher; used to dynamic filter messages #### Example -See: [heka_tcp_matcher.lua](https://github.com/mozilla-services/lua_sandbox/blob/master/sandboxes/heka/output/heka_tcp_matcher.lua) +See: [heka_tcp_matcher.lua](https://github.com/mozilla-services/lua_sandbox_extensions/socket/sandboxes/heka/output/heka_tcp_matcher.html) ### update_checkpoint diff --git a/docs/sandbox.md b/docs/sandbox.md index 0724fdb..4e72119 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -49,14 +49,6 @@ By default only the base library is loaded additional libraries must be loaded w The following modules have been modified, as described, for use in the sandbox. - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) - The require() function has been modified to not expose any of the package table to the sandbox. - - [cjson](http://www.kyne.com.au/~mark/software/lua-cjson-manual.html) JSON parser with the following modifications: - - Loads the cjson module in a global cjson table - - The encode buffer is limited to the sandbox output_limit. - - The decode buffer will be roughly limited to one half of the sandbox memory_limit. - - The NULL value is not decoded to cjson.null it is simply discarded. - If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - - The new() function has been disabled so only a single cjson parser can be created. - - The encode_keep_buffer() function has been disabled (the buffer is always reused). - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. @@ -104,7 +96,7 @@ The best place to start is with some examples: ### Unit Test API -[Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_luasandbox.c) +[Unit Test API](/lua_sandbox/doxygen/test_2sandbox_8h.html) #### Lua Functions Exposed to C @@ -119,7 +111,7 @@ The best place to start is with some examples: - captures whatever is in the output buffer for use by the host application, appending any optional arguments (optional arguments have the same restriction as output). -### Heka Sandbox +### Heka Sandbox API -[Heka Sandbox Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/include/luasandbox/heka/sandbox.h) +[Heka Sandbox API](/lua_sandbox/doxygen/heka_2sandbox_8h.html) diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index be340bb..dec91ca 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -5,77 +5,6 @@ require "os" require "io" require "string" -require "table" - -local function get_path(s) - return s:match("(.+)/[^/]-$") -end - - -local function get_filename(s) - return s:match("/([^/]-)$") -end - - -local function strip_ext(s) - return s:sub(1, #s - 4) -end - - -local function sort_entries(t) - local a = {} - for n in pairs(t) do table.insert(a, n) end - table.sort(a) - local i = 0 -- iterator variable - local iter = function () -- iterator function - i = i + 1 - if a[i] == nil then - return nil - else - return a[i], t[a[i]] - end - end - return iter -end - - -local function create_index(path, dir) - local fh = assert(io.open("gh-pages/" .. path .. "/index.md", "w")) - fh:write(string.format("# %s\n", path)) - - for k,v in sort_entries(dir.entries) do - if k:match(".lua$") then - fh:write(string.format("* [%s](%s.html) - %s\n", k, strip_ext(k), v.title)) - else - fh:write(string.format("* [%s](%s/index.html)\n", k, k)) - end - end - - fh:close() -end - - -local function output_tree(fh, list, key, dir) - list[#list + 1] = key - local path = table.concat(list, "/") - create_index(path, dir) - if #list == 1 then fh:write("
                \n") end - fh:write(string.format('
              • %s
                • \n', path, key)) - - fh:write("
                  \n") - for k, v in sort_entries(dir.entries) do - if k:match(".lua$") then - fh:write(string.format('
                • %s
                  • \n', strip_ext(v.line), strip_ext(k))) - else - output_tree(fh, list, k, v) - end - end - fh:write("
                \n") - - if #list == 1 then fh:write("
              \n") end - table.remove(list) -end - local function output_css() local fh = assert(io.open("gh-pages/docs.css", "w")) @@ -188,41 +117,6 @@ local function output_menu(before, after, paths, version) - - ]]) - - output_tree(fh, {}, "modules", paths.entries.modules) - output_tree(fh, {}, "sandboxes", paths.entries.sandboxes) - - fh:write([[
              ]]) @@ -234,51 +128,6 @@ local function output_menu(before, after, paths, version) end -local function handle_path(paths, in_path, out_path) - local list = {} - local d = paths - for dir in string.gmatch(out_path, "[^/]+") do - list[#list + 1] = dir - if dir ~= "gh-pages" then - local full_path = table.concat(list, "/") - local cd = d.entries[dir] - if not cd then - os.execute(string.format("mkdir -p %s", full_path)) - local nd = {path = in_path, entries = {}} - d.entries[dir] = nd - d = nd - else - d = cd - end - end - end - return d -end - - -local function extract_lua_docs(paths) - local fh = assert(io.popen("find sandboxes modules -name \\*.lua")) - for line in fh:lines() do - local sfh = assert(io.open(line)) - local lua = sfh:read("*a") - sfh:close() - - local doc = lua:match("%-%-%[%[%s*(.-)%-%-%]%]") - local title = lua:match("#%s(.-)\n") - if not title then error("doc error, no title: " .. line) end - - local outfn = string.gsub("gh-pages/" .. line, "lua$", "md") - local p = handle_path(paths, get_path(line), get_path(outfn)) - local ofh = assert(io.open(outfn, "w")) - p.entries[get_filename(line)] = {line = line, title = title} - ofh:write(doc) - ofh:write(string.format("\n\nsource code: [%s](https://github.com/mozilla-services/lua_sandbox/blob/master/%s)\n", get_filename(line), line)) - ofh:close() - end - fh:close() -end - - local function md_to_html(paths, version) local before = "/tmp/before.html" local after = "/tmp/after.html" @@ -305,7 +154,6 @@ local function main() if rv ~= 0 then error"rsync" end output_css() local paths = {entries = {}} - extract_lua_docs(paths) md_to_html(paths, args[1]) end diff --git a/include/luasandbox/lauxlib.h b/include/luasandbox/lauxlib.h new file mode 100644 index 0000000..3425823 --- /dev/null +++ b/include/luasandbox/lauxlib.h @@ -0,0 +1,174 @@ +/* +** $Id: lauxlib.h,v 1.88.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions for building Lua libraries +** See Copyright Notice in lua.h +*/ + + +#ifndef lauxlib_h +#define lauxlib_h + + +#include +#include + +#include "lua.h" + + +#if defined(LUA_COMPAT_GETN) +LUALIB_API int (luaL_getn) (lua_State *L, int t); +LUALIB_API void (luaL_setn) (lua_State *L, int t, int n); +#else +#define luaL_getn(L,i) ((int)lua_objlen(L, i)) +#define luaL_setn(L,i,j) ((void)0) /* no op! */ +#endif + +#if defined(LUA_COMPAT_OPENLIB) +#define luaI_openlib luaL_openlib +#endif + + +/* extra error code for `luaL_load' */ +#define LUA_ERRFILE (LUA_ERRERR+1) + + +typedef struct luaL_Reg { + const char *name; + lua_CFunction func; +} luaL_Reg; + + + +LUALIB_API void (luaI_openlib) (lua_State *L, const char *libname, + const luaL_Reg *l, int nup); +LUALIB_API void (luaL_register) (lua_State *L, const char *libname, + const luaL_Reg *l); +LUALIB_API int (luaL_getmetafield) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_callmeta) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_typerror) (lua_State *L, int narg, const char *tname); +LUALIB_API int (luaL_argerror) (lua_State *L, int numarg, const char *extramsg); +LUALIB_API const char *(luaL_checklstring) (lua_State *L, int numArg, + size_t *l); +LUALIB_API const char *(luaL_optlstring) (lua_State *L, int numArg, + const char *def, size_t *l); +LUALIB_API lua_Number (luaL_checknumber) (lua_State *L, int numArg); +LUALIB_API lua_Number (luaL_optnumber) (lua_State *L, int nArg, lua_Number def); + +LUALIB_API lua_Integer (luaL_checkinteger) (lua_State *L, int numArg); +LUALIB_API lua_Integer (luaL_optinteger) (lua_State *L, int nArg, + lua_Integer def); + +LUALIB_API void (luaL_checkstack) (lua_State *L, int sz, const char *msg); +LUALIB_API void (luaL_checktype) (lua_State *L, int narg, int t); +LUALIB_API void (luaL_checkany) (lua_State *L, int narg); + +LUALIB_API int (luaL_newmetatable) (lua_State *L, const char *tname); +LUALIB_API void *(luaL_checkudata) (lua_State *L, int ud, const char *tname); + +LUALIB_API void (luaL_where) (lua_State *L, int lvl); +LUALIB_API int (luaL_error) (lua_State *L, const char *fmt, ...); + +LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def, + const char *const lst[]); + +LUALIB_API int (luaL_ref) (lua_State *L, int t); +LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref); + +LUALIB_API int (luaL_loadfile) (lua_State *L, const char *filename); +LUALIB_API int (luaL_loadbuffer) (lua_State *L, const char *buff, size_t sz, + const char *name); +LUALIB_API int (luaL_loadstring) (lua_State *L, const char *s); + +LUALIB_API lua_State *(luaL_newstate) (void); + + +LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s, const char *p, + const char *r); + +LUALIB_API const char *(luaL_findtable) (lua_State *L, int idx, + const char *fname, int szhint); + + + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define luaL_argcheck(L, cond,numarg,extramsg) \ + ((void)((cond) || luaL_argerror(L, (numarg), (extramsg)))) +#define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL)) +#define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL)) +#define luaL_checkint(L,n) ((int)luaL_checkinteger(L, (n))) +#define luaL_optint(L,n,d) ((int)luaL_optinteger(L, (n), (d))) +#define luaL_checklong(L,n) ((long)luaL_checkinteger(L, (n))) +#define luaL_optlong(L,n,d) ((long)luaL_optinteger(L, (n), (d))) + +#define luaL_typename(L,i) lua_typename(L, lua_type(L,(i))) + +#define luaL_dofile(L, fn) \ + (luaL_loadfile(L, fn) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_dostring(L, s) \ + (luaL_loadstring(L, s) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_getmetatable(L,n) (lua_getfield(L, LUA_REGISTRYINDEX, (n))) + +#define luaL_opt(L,f,n,d) (lua_isnoneornil(L,(n)) ? (d) : f(L,(n))) + +/* +** {====================================================== +** Generic Buffer manipulation +** ======================================================= +*/ + + + +typedef struct luaL_Buffer { + char *p; /* current position in buffer */ + int lvl; /* number of strings in the stack (level) */ + lua_State *L; + char buffer[LUAL_BUFFERSIZE]; +} luaL_Buffer; + +#define luaL_addchar(B,c) \ + ((void)((B)->p < ((B)->buffer+LUAL_BUFFERSIZE) || luaL_prepbuffer(B)), \ + (*(B)->p++ = (char)(c))) + +/* compatibility only */ +#define luaL_putchar(B,c) luaL_addchar(B,c) + +#define luaL_addsize(B,n) ((B)->p += (n)) + +LUALIB_API void (luaL_buffinit) (lua_State *L, luaL_Buffer *B); +LUALIB_API char *(luaL_prepbuffer) (luaL_Buffer *B); +LUALIB_API void (luaL_addlstring) (luaL_Buffer *B, const char *s, size_t l); +LUALIB_API void (luaL_addstring) (luaL_Buffer *B, const char *s); +LUALIB_API void (luaL_addvalue) (luaL_Buffer *B); +LUALIB_API void (luaL_pushresult) (luaL_Buffer *B); + + +/* }====================================================== */ + + +/* compatibility with ref system */ + +/* pre-defined references */ +#define LUA_NOREF (-2) +#define LUA_REFNIL (-1) + +#define lua_ref(L,lock) ((lock) ? luaL_ref(L, LUA_REGISTRYINDEX) : \ + (lua_pushstring(L, "unlocked references are obsolete"), lua_error(L), 0)) + +#define lua_unref(L,ref) luaL_unref(L, LUA_REGISTRYINDEX, (ref)) + +#define lua_getref(L,ref) lua_rawgeti(L, LUA_REGISTRYINDEX, (ref)) + + +#define luaL_reg luaL_Reg + +#endif + + diff --git a/include/luasandbox/lua.h b/include/luasandbox/lua.h new file mode 100644 index 0000000..75760ef --- /dev/null +++ b/include/luasandbox/lua.h @@ -0,0 +1,389 @@ +/* +** $Id: lua.h,v 1.218.1.7 2012/01/13 20:36:20 roberto Exp $ +** Lua - An Extensible Extension Language +** Lua.org, PUC-Rio, Brazil (http://www.lua.org) +** See Copyright Notice at the end of this file +*/ + + +#ifndef lua_h +#define lua_h + +#include +#include + + +#include "luaconf.h" + + +#define LUA_VERSION "Lua 5.1" +#define LUA_RELEASE "Lua 5.1.5" +#define LUA_VERSION_NUM 501 +#define LUA_COPYRIGHT "Copyright (C) 1994-2012 Lua.org, PUC-Rio" +#define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo & W. Celes" + + +/* mark for precompiled code (`Lua') */ +#define LUA_SIGNATURE "\033Lua" + +/* option for multiple returns in `lua_pcall' and `lua_call' */ +#define LUA_MULTRET (-1) + + +/* +** pseudo-indices +*/ +#define LUA_REGISTRYINDEX (-10000) +#define LUA_ENVIRONINDEX (-10001) +#define LUA_GLOBALSINDEX (-10002) +#define lua_upvalueindex(i) (LUA_GLOBALSINDEX-(i)) + + +/* thread status; 0 is OK */ +#define LUA_YIELD 1 +#define LUA_ERRRUN 2 +#define LUA_ERRSYNTAX 3 +#define LUA_ERRMEM 4 +#define LUA_ERRERR 5 + + +typedef struct lua_State lua_State; + +typedef int (*lua_CFunction) (lua_State *L); + + +/* +** functions that read/write blocks when loading/dumping Lua chunks +*/ +typedef const char * (*lua_Reader) (lua_State *L, void *ud, size_t *sz); + +typedef int (*lua_Writer) (lua_State *L, const void* p, size_t sz, void* ud); + + +/* +** prototype for memory-allocation functions +*/ +typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize); + + +/* +** basic types +*/ +#define LUA_TNONE (-1) + +#define LUA_TNIL 0 +#define LUA_TBOOLEAN 1 +#define LUA_TLIGHTUSERDATA 2 +#define LUA_TNUMBER 3 +#define LUA_TSTRING 4 +#define LUA_TTABLE 5 +#define LUA_TFUNCTION 6 +#define LUA_TUSERDATA 7 +#define LUA_TTHREAD 8 + + + +/* minimum Lua stack available to a C function */ +#define LUA_MINSTACK 20 + + +/* +** generic extra include file +*/ +#if defined(LUA_USER_H) +#include LUA_USER_H +#endif + + +/* type of numbers in Lua */ +typedef LUA_NUMBER lua_Number; + + +/* type for integer functions */ +typedef LUA_INTEGER lua_Integer; + + + +/* +** state manipulation +*/ +LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud); +LUA_API void (lua_close) (lua_State *L); +LUA_API lua_State *(lua_newthread) (lua_State *L); + +LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf); + + +/* +** basic stack manipulation +*/ +LUA_API int (lua_gettop) (lua_State *L); +LUA_API void (lua_settop) (lua_State *L, int idx); +LUA_API void (lua_pushvalue) (lua_State *L, int idx); +LUA_API void (lua_remove) (lua_State *L, int idx); +LUA_API void (lua_insert) (lua_State *L, int idx); +LUA_API void (lua_replace) (lua_State *L, int idx); +LUA_API int (lua_checkstack) (lua_State *L, int sz); + +LUA_API void (lua_xmove) (lua_State *from, lua_State *to, int n); + + +/* +** access functions (stack -> C) +*/ + +LUA_API int (lua_isnumber) (lua_State *L, int idx); +LUA_API int (lua_isstring) (lua_State *L, int idx); +LUA_API int (lua_iscfunction) (lua_State *L, int idx); +LUA_API int (lua_isuserdata) (lua_State *L, int idx); +LUA_API int (lua_type) (lua_State *L, int idx); +LUA_API const char *(lua_typename) (lua_State *L, int tp); + +LUA_API int (lua_equal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_rawequal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_lessthan) (lua_State *L, int idx1, int idx2); + +LUA_API lua_Number (lua_tonumber) (lua_State *L, int idx); +LUA_API lua_Integer (lua_tointeger) (lua_State *L, int idx); +LUA_API int (lua_toboolean) (lua_State *L, int idx); +LUA_API const char *(lua_tolstring) (lua_State *L, int idx, size_t *len); +LUA_API size_t (lua_objlen) (lua_State *L, int idx); +LUA_API lua_CFunction (lua_tocfunction) (lua_State *L, int idx); +LUA_API void *(lua_touserdata) (lua_State *L, int idx); +LUA_API lua_State *(lua_tothread) (lua_State *L, int idx); +LUA_API const void *(lua_topointer) (lua_State *L, int idx); + + +/* +** push functions (C -> stack) +*/ +LUA_API void (lua_pushnil) (lua_State *L); +LUA_API void (lua_pushnumber) (lua_State *L, lua_Number n); +LUA_API void (lua_pushinteger) (lua_State *L, lua_Integer n); +LUA_API void (lua_pushlstring) (lua_State *L, const char *s, size_t l); +LUA_API void (lua_pushstring) (lua_State *L, const char *s); +LUA_API const char *(lua_pushvfstring) (lua_State *L, const char *fmt, + va_list argp); +LUA_API const char *(lua_pushfstring) (lua_State *L, const char *fmt, ...); +LUA_API void (lua_pushcclosure) (lua_State *L, lua_CFunction fn, int n); +LUA_API void (lua_pushboolean) (lua_State *L, int b); +LUA_API void (lua_pushlightuserdata) (lua_State *L, void *p); +LUA_API int (lua_pushthread) (lua_State *L); + + +/* +** get functions (Lua -> stack) +*/ +LUA_API void (lua_gettable) (lua_State *L, int idx); +LUA_API void (lua_getfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawget) (lua_State *L, int idx); +LUA_API void (lua_rawgeti) (lua_State *L, int idx, int n); +LUA_API void (lua_createtable) (lua_State *L, int narr, int nrec); +LUA_API void *(lua_newuserdata) (lua_State *L, size_t sz); +LUA_API int (lua_getmetatable) (lua_State *L, int objindex); +LUA_API void (lua_getfenv) (lua_State *L, int idx); + + +/* +** set functions (stack -> Lua) +*/ +LUA_API void (lua_settable) (lua_State *L, int idx); +LUA_API void (lua_setfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawset) (lua_State *L, int idx); +LUA_API void (lua_rawseti) (lua_State *L, int idx, int n); +LUA_API int (lua_setmetatable) (lua_State *L, int objindex); +LUA_API int (lua_setfenv) (lua_State *L, int idx); + + +/* +** `load' and `call' functions (load and run Lua code) +*/ +LUA_API void (lua_call) (lua_State *L, int nargs, int nresults); +LUA_API int (lua_pcall) (lua_State *L, int nargs, int nresults, int errfunc); +LUA_API int (lua_cpcall) (lua_State *L, lua_CFunction func, void *ud); +LUA_API int (lua_load) (lua_State *L, lua_Reader reader, void *dt, + const char *chunkname); + +LUA_API int (lua_dump) (lua_State *L, lua_Writer writer, void *data); + + +/* +** coroutine functions +*/ +LUA_API int (lua_yield) (lua_State *L, int nresults); +LUA_API int (lua_resume) (lua_State *L, int narg); +LUA_API int (lua_status) (lua_State *L); + +/* +** garbage-collection function and options +*/ + +#define LUA_GCSTOP 0 +#define LUA_GCRESTART 1 +#define LUA_GCCOLLECT 2 +#define LUA_GCCOUNT 3 +#define LUA_GCCOUNTB 4 +#define LUA_GCSTEP 5 +#define LUA_GCSETPAUSE 6 +#define LUA_GCSETSTEPMUL 7 + +LUA_API int (lua_gc) (lua_State *L, int what, int data); + + +/* +** miscellaneous functions +*/ + +LUA_API int (lua_error) (lua_State *L); + +LUA_API int (lua_next) (lua_State *L, int idx); + +LUA_API void (lua_concat) (lua_State *L, int n); + +LUA_API lua_Alloc (lua_getallocf) (lua_State *L, void **ud); +LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud); + + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define lua_pop(L,n) lua_settop(L, -(n)-1) + +#define lua_newtable(L) lua_createtable(L, 0, 0) + +#define lua_register(L,n,f) (lua_pushcfunction(L, (f)), lua_setglobal(L, (n))) + +#define lua_pushcfunction(L,f) lua_pushcclosure(L, (f), 0) + +#define lua_strlen(L,i) lua_objlen(L, (i)) + +#define lua_isfunction(L,n) (lua_type(L, (n)) == LUA_TFUNCTION) +#define lua_istable(L,n) (lua_type(L, (n)) == LUA_TTABLE) +#define lua_islightuserdata(L,n) (lua_type(L, (n)) == LUA_TLIGHTUSERDATA) +#define lua_isnil(L,n) (lua_type(L, (n)) == LUA_TNIL) +#define lua_isboolean(L,n) (lua_type(L, (n)) == LUA_TBOOLEAN) +#define lua_isthread(L,n) (lua_type(L, (n)) == LUA_TTHREAD) +#define lua_isnone(L,n) (lua_type(L, (n)) == LUA_TNONE) +#define lua_isnoneornil(L, n) (lua_type(L, (n)) <= 0) + +#define lua_pushliteral(L, s) \ + lua_pushlstring(L, "" s, (sizeof(s)/sizeof(char))-1) + +#define lua_setglobal(L,s) lua_setfield(L, LUA_GLOBALSINDEX, (s)) +#define lua_getglobal(L,s) lua_getfield(L, LUA_GLOBALSINDEX, (s)) + +#define lua_tostring(L,i) lua_tolstring(L, (i), NULL) + + + +/* +** compatibility macros and functions +*/ + +#define lua_open() luaL_newstate() + +#define lua_getregistry(L) lua_pushvalue(L, LUA_REGISTRYINDEX) + +#define lua_getgccount(L) lua_gc(L, LUA_GCCOUNT, 0) + +#define lua_Chunkreader lua_Reader +#define lua_Chunkwriter lua_Writer + + +/* hack */ +LUA_API void lua_setlevel (lua_State *from, lua_State *to); + + +/* +** {====================================================================== +** Debug API +** ======================================================================= +*/ + + +/* +** Event codes +*/ +#define LUA_HOOKCALL 0 +#define LUA_HOOKRET 1 +#define LUA_HOOKLINE 2 +#define LUA_HOOKCOUNT 3 +#define LUA_HOOKTAILRET 4 + + +/* +** Event masks +*/ +#define LUA_MASKCALL (1 << LUA_HOOKCALL) +#define LUA_MASKRET (1 << LUA_HOOKRET) +#define LUA_MASKLINE (1 << LUA_HOOKLINE) +#define LUA_MASKCOUNT (1 << LUA_HOOKCOUNT) + +typedef struct lua_Debug lua_Debug; /* activation record */ + + +/* Functions to be called by the debuger in specific events */ +typedef void (*lua_Hook) (lua_State *L, lua_Debug *ar); + + +LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar); +LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar); +LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n); +LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n); + +LUA_API int lua_sethook (lua_State *L, lua_Hook func, int mask, int count); +LUA_API lua_Hook lua_gethook (lua_State *L); +LUA_API int lua_gethookmask (lua_State *L); +LUA_API int lua_gethookcount (lua_State *L); +LUA_API int lua_gethookcountremaining (lua_State *L); + + +struct lua_Debug { + int event; + const char *name; /* (n) */ + const char *namewhat; /* (n) `global', `local', `field', `method' */ + const char *what; /* (S) `Lua', `C', `main', `tail' */ + const char *source; /* (S) */ + int currentline; /* (l) */ + int nups; /* (u) number of upvalues */ + int linedefined; /* (S) */ + int lastlinedefined; /* (S) */ + char short_src[LUA_IDSIZE]; /* (S) */ + /* private part */ + int i_ci; /* active function */ +}; + +/* }====================================================================== */ + + +/****************************************************************************** +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ + + +#endif diff --git a/include/luasandbox/luaconf.h b/include/luasandbox/luaconf.h new file mode 100644 index 0000000..fa748aa --- /dev/null +++ b/include/luasandbox/luaconf.h @@ -0,0 +1,765 @@ +/* +** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ +** Configuration file for Lua +** See Copyright Notice in lua.h +*/ + + +#ifndef lconfig_h +#define lconfig_h + +#include +#include + + +/* +** ================================================================== +** Search for "@@" to find all configurable definitions. +** =================================================================== +*/ + + +/* +@@ LUA_ANSI controls the use of non-ansi features. +** CHANGE it (define it) if you want Lua to avoid the use of any +** non-ansi feature or library. +*/ +#if defined(__STRICT_ANSI__) +#define LUA_ANSI +#endif + + +#if !defined(LUA_ANSI) && defined(_WIN32) +#define LUA_WIN +#endif + +#if defined(LUA_USE_LINUX) +#define LUA_USE_POSIX +#define LUA_USE_DLOPEN /* needs an extra library: -ldl */ +#define LUA_USE_READLINE /* needs some extra libraries */ +#endif + +#if defined(LUA_USE_MACOSX) +#define LUA_USE_POSIX +#define LUA_DL_DYLD /* does not need extra library */ +#endif + + + +/* +@@ LUA_USE_POSIX includes all functionallity listed as X/Open System +@* Interfaces Extension (XSI). +** CHANGE it (define it) if your system is XSI compatible. +*/ +#if defined(LUA_USE_POSIX) +#define LUA_USE_MKSTEMP +#define LUA_USE_ISATTY +#define LUA_USE_POPEN +#define LUA_USE_ULONGJMP +#endif + + +/* +@@ LUA_PATH and LUA_CPATH are the names of the environment variables that +@* Lua check to set its paths. +@@ LUA_INIT is the name of the environment variable that Lua +@* checks for initialization code. +** CHANGE them if you want different names. +*/ +#define LUA_PATH "LUA_PATH" +#define LUA_CPATH "LUA_CPATH" +#define LUA_INIT "LUA_INIT" + + +/* +@@ LUA_PATH_DEFAULT is the default path that Lua uses to look for +@* Lua libraries. +@@ LUA_CPATH_DEFAULT is the default path that Lua uses to look for +@* C libraries. +** CHANGE them if your machine has a non-conventional directory +** hierarchy or if you want to install your libraries in +** non-conventional directories. +*/ +#if defined(_WIN32) +/* +** In Windows, any exclamation mark ('!') in the path is replaced by the +** path of the directory of the executable file of the current process. +*/ +#define LUA_LDIR "!\\lua\\" +#define LUA_CDIR "!\\" +#define LUA_PATH_DEFAULT \ + ".\\?.lua;" LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" \ + LUA_CDIR"?.lua;" LUA_CDIR"?\\init.lua" +#define LUA_CPATH_DEFAULT \ + ".\\?.dll;" LUA_CDIR"?.dll;" LUA_CDIR"loadall.dll" + +#else +#define LUA_ROOT "/usr/local/" +#define LUA_LDIR LUA_ROOT "share/lua/5.1/" +#define LUA_CDIR LUA_ROOT "lib/lua/5.1/" +#define LUA_PATH_DEFAULT \ + "./?.lua;" LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" \ + LUA_CDIR"?.lua;" LUA_CDIR"?/init.lua" +#define LUA_CPATH_DEFAULT \ + "./?.so;" LUA_CDIR"?.so;" LUA_CDIR"loadall.so" +#endif + + +/* +@@ LUA_DIRSEP is the directory separator (for submodules). +** CHANGE it if your machine does not use "/" as the directory separator +** and is not Windows. (On Windows Lua automatically uses "\".) +*/ +#if defined(_WIN32) +#define LUA_DIRSEP "\\" +#else +#define LUA_DIRSEP "/" +#endif + + +/* +@@ LUA_PATHSEP is the character that separates templates in a path. +@@ LUA_PATH_MARK is the string that marks the substitution points in a +@* template. +@@ LUA_EXECDIR in a Windows path is replaced by the executable's +@* directory. +@@ LUA_IGMARK is a mark to ignore all before it when bulding the +@* luaopen_ function name. +** CHANGE them if for some reason your system cannot use those +** characters. (E.g., if one of those characters is a common character +** in file/directory names.) Probably you do not need to change them. +*/ +#define LUA_PATHSEP ";" +#define LUA_PATH_MARK "?" +#define LUA_EXECDIR "!" +#define LUA_IGMARK "-" + + +/* +@@ LUA_INTEGER is the integral type used by lua_pushinteger/lua_tointeger. +** CHANGE that if ptrdiff_t is not adequate on your machine. (On most +** machines, ptrdiff_t gives a good choice between int or long.) +*/ +#define LUA_INTEGER ptrdiff_t + + +/* +@@ LUA_API is a mark for all core API functions. +@@ LUALIB_API is a mark for all standard library functions. +** CHANGE them if you need to define those functions in some special way. +** For instance, if you want to create one Windows DLL with the core and +** the libraries, you may want to use the following definition (define +** LUA_BUILD_AS_DLL to get it). +*/ +#if defined(LUA_BUILD_AS_DLL) + +#if defined(LUA_CORE) || defined(LUA_LIB) +#define LUA_API __declspec(dllexport) +#else +#define LUA_API __declspec(dllimport) +#endif + +#else +#if __GNUC__ >= 4 +#define LUA_API __attribute__ ((visibility ("default"))) +#else +#define LUA_API +#endif +#endif + +/* more often than not the libs go together with the core */ +#define LUALIB_API LUA_API + + +/* +@@ LUAI_FUNC is a mark for all extern functions that are not to be +@* exported to outside modules. +@@ LUAI_DATA is a mark for all extern (const) variables that are not to +@* be exported to outside modules. +** CHANGE them if you need to mark them in some special way. Elf/gcc +** (versions 3.2 and later) mark them as "hidden" to optimize access +** when Lua is compiled as a shared library. +*/ +#if defined(luaall_c) +#define LUAI_FUNC static +#define LUAI_DATA /* empty */ + +#elif defined(__GNUC__) && ((__GNUC__*100 + __GNUC_MINOR__) >= 302) && \ + defined(__ELF__) +#define LUAI_FUNC __attribute__((visibility("hidden"))) extern +#define LUAI_DATA LUAI_FUNC + +#else +#define LUAI_FUNC extern +#define LUAI_DATA extern +#endif + + + +/* +@@ LUA_QL describes how error messages quote program elements. +** CHANGE it if you want a different appearance. +*/ +#define LUA_QL(x) "'" x "'" +#define LUA_QS LUA_QL("%s") + + +/* +@@ LUA_IDSIZE gives the maximum size for the description of the source +@* of a function in debug information. +** CHANGE it if you want a different size. +*/ +#define LUA_IDSIZE 60 + + +/* +** {================================================================== +** Stand-alone configuration +** =================================================================== +*/ + +#if defined(lua_c) || defined(luaall_c) + +/* +@@ lua_stdin_is_tty detects whether the standard input is a 'tty' (that +@* is, whether we're running lua interactively). +** CHANGE it if you have a better definition for non-POSIX/non-Windows +** systems. +*/ +#if defined(LUA_USE_ISATTY) +#include +#define lua_stdin_is_tty() isatty(0) +#elif defined(LUA_WIN) +#include +#include +#define lua_stdin_is_tty() _isatty(_fileno(stdin)) +#else +#define lua_stdin_is_tty() 1 /* assume stdin is a tty */ +#endif + + +/* +@@ LUA_PROMPT is the default prompt used by stand-alone Lua. +@@ LUA_PROMPT2 is the default continuation prompt used by stand-alone Lua. +** CHANGE them if you want different prompts. (You can also change the +** prompts dynamically, assigning to globals _PROMPT/_PROMPT2.) +*/ +#define LUA_PROMPT "> " +#define LUA_PROMPT2 ">> " + + +/* +@@ LUA_PROGNAME is the default name for the stand-alone Lua program. +** CHANGE it if your stand-alone interpreter has a different name and +** your system is not able to detect that name automatically. +*/ +#define LUA_PROGNAME "lua" + + +/* +@@ LUA_MAXINPUT is the maximum length for an input line in the +@* stand-alone interpreter. +** CHANGE it if you need longer lines. +*/ +#define LUA_MAXINPUT 512 + + +/* +@@ lua_readline defines how to show a prompt and then read a line from +@* the standard input. +@@ lua_saveline defines how to "save" a read line in a "history". +@@ lua_freeline defines how to free a line read by lua_readline. +** CHANGE them if you want to improve this functionality (e.g., by using +** GNU readline and history facilities). +*/ +#if defined(LUA_USE_READLINE) +#include +#include +#include +#define lua_readline(L,b,p) ((void)L, ((b)=readline(p)) != NULL) +#define lua_saveline(L,idx) \ + if (lua_strlen(L,idx) > 0) /* non-empty line? */ \ + add_history(lua_tostring(L, idx)); /* add it to history */ +#define lua_freeline(L,b) ((void)L, free(b)) +#else +#define lua_readline(L,b,p) \ + ((void)L, fputs(p, stdout), fflush(stdout), /* show prompt */ \ + fgets(b, LUA_MAXINPUT, stdin) != NULL) /* get line */ +#define lua_saveline(L,idx) { (void)L; (void)idx; } +#define lua_freeline(L,b) { (void)L; (void)b; } +#endif + +#endif + +/* }================================================================== */ + + +/* +@@ LUAI_GCPAUSE defines the default pause between garbage-collector cycles +@* as a percentage. +** CHANGE it if you want the GC to run faster or slower (higher values +** mean larger pauses which mean slower collection.) You can also change +** this value dynamically. +*/ +#define LUAI_GCPAUSE 200 /* 200% (wait memory to double before next GC) */ + + +/* +@@ LUAI_GCMUL defines the default speed of garbage collection relative to +@* memory allocation as a percentage. +** CHANGE it if you want to change the granularity of the garbage +** collection. (Higher values mean coarser collections. 0 represents +** infinity, where each step performs a full collection.) You can also +** change this value dynamically. +*/ +#define LUAI_GCMUL 200 /* GC runs 'twice the speed' of memory allocation */ + + + +/* +@@ LUA_COMPAT_GETN controls compatibility with old getn behavior. +** CHANGE it (define it) if you want exact compatibility with the +** behavior of setn/getn in Lua 5.0. +*/ +#undef LUA_COMPAT_GETN + +/* +@@ LUA_COMPAT_LOADLIB controls compatibility about global loadlib. +** CHANGE it to undefined as soon as you do not need a global 'loadlib' +** function (the function is still available as 'package.loadlib'). +*/ +#undef LUA_COMPAT_LOADLIB + +/* +@@ LUA_COMPAT_VARARG controls compatibility with old vararg feature. +** CHANGE it to undefined as soon as your programs use only '...' to +** access vararg parameters (instead of the old 'arg' table). +*/ +#define LUA_COMPAT_VARARG + +/* +@@ LUA_COMPAT_MOD controls compatibility with old math.mod function. +** CHANGE it to undefined as soon as your programs use 'math.fmod' or +** the new '%' operator instead of 'math.mod'. +*/ +#define LUA_COMPAT_MOD + +/* +@@ LUA_COMPAT_LSTR controls compatibility with old long string nesting +@* facility. +** CHANGE it to 2 if you want the old behaviour, or undefine it to turn +** off the advisory error when nesting [[...]]. +*/ +#define LUA_COMPAT_LSTR 1 + +/* +@@ LUA_COMPAT_GFIND controls compatibility with old 'string.gfind' name. +** CHANGE it to undefined as soon as you rename 'string.gfind' to +** 'string.gmatch'. +*/ +#define LUA_COMPAT_GFIND + +/* +@@ LUA_COMPAT_OPENLIB controls compatibility with old 'luaL_openlib' +@* behavior. +** CHANGE it to undefined as soon as you replace to 'luaL_register' +** your uses of 'luaL_openlib' +*/ +#define LUA_COMPAT_OPENLIB + + + +/* +@@ luai_apicheck is the assert macro used by the Lua-C API. +** CHANGE luai_apicheck if you want Lua to perform some checks in the +** parameters it gets from API calls. This may slow down the interpreter +** a bit, but may be quite useful when debugging C code that interfaces +** with Lua. A useful redefinition is to use assert.h. +*/ +#if defined(LUA_USE_APICHECK) +#include +#define luai_apicheck(L,o) { (void)L; assert(o); } +#else +#define luai_apicheck(L,o) { (void)L; } +#endif + + +/* +@@ LUAI_BITSINT defines the number of bits in an int. +** CHANGE here if Lua cannot automatically detect the number of bits of +** your machine. Probably you do not need to change this. +*/ +/* avoid overflows in comparison */ +#if INT_MAX-20 < 32760 +#define LUAI_BITSINT 16 +#elif INT_MAX > 2147483640L +/* int has at least 32 bits */ +#define LUAI_BITSINT 32 +#else +#error "you must define LUA_BITSINT with number of bits in an integer" +#endif + + +/* +@@ LUAI_UINT32 is an unsigned integer with at least 32 bits. +@@ LUAI_INT32 is an signed integer with at least 32 bits. +@@ LUAI_UMEM is an unsigned integer big enough to count the total +@* memory used by Lua. +@@ LUAI_MEM is a signed integer big enough to count the total memory +@* used by Lua. +** CHANGE here if for some weird reason the default definitions are not +** good enough for your machine. (The definitions in the 'else' +** part always works, but may waste space on machines with 64-bit +** longs.) Probably you do not need to change this. +*/ +#if LUAI_BITSINT >= 32 +#define LUAI_UINT32 unsigned int +#define LUAI_INT32 int +#define LUAI_MAXINT32 INT_MAX +#define LUAI_UMEM size_t +#define LUAI_MEM ptrdiff_t +#else +/* 16-bit ints */ +#define LUAI_UINT32 unsigned long +#define LUAI_INT32 long +#define LUAI_MAXINT32 LONG_MAX +#define LUAI_UMEM unsigned long +#define LUAI_MEM long +#endif + + +/* +@@ LUAI_MAXCALLS limits the number of nested calls. +** CHANGE it if you need really deep recursive calls. This limit is +** arbitrary; its only purpose is to stop infinite recursion before +** exhausting memory. +*/ +#define LUAI_MAXCALLS 20000 + + +/* +@@ LUAI_MAXCSTACK limits the number of Lua stack slots that a C function +@* can use. +** CHANGE it if you need lots of (Lua) stack space for your C +** functions. This limit is arbitrary; its only purpose is to stop C +** functions to consume unlimited stack space. (must be smaller than +** -LUA_REGISTRYINDEX) +*/ +#define LUAI_MAXCSTACK 8000 + + + +/* +** {================================================================== +** CHANGE (to smaller values) the following definitions if your system +** has a small C stack. (Or you may want to change them to larger +** values if your system has a large C stack and these limits are +** too rigid for you.) Some of these constants control the size of +** stack-allocated arrays used by the compiler or the interpreter, while +** others limit the maximum number of recursive calls that the compiler +** or the interpreter can perform. Values too large may cause a C stack +** overflow for some forms of deep constructs. +** =================================================================== +*/ + + +/* +@@ LUAI_MAXCCALLS is the maximum depth for nested C calls (short) and +@* syntactical nested non-terminals in a program. +*/ +#define LUAI_MAXCCALLS 200 + + +/* +@@ LUAI_MAXVARS is the maximum number of local variables per function +@* (must be smaller than 250). +*/ +#define LUAI_MAXVARS 200 + + +/* +@@ LUAI_MAXUPVALUES is the maximum number of upvalues per function +@* (must be smaller than 250). +*/ +#define LUAI_MAXUPVALUES 60 + + +/* +@@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system. +*/ +#define LUAL_BUFFERSIZE BUFSIZ + +/* }================================================================== */ + + + + +/* +** {================================================================== +@@ LUA_NUMBER is the type of numbers in Lua. +** CHANGE the following definitions only if you want to build Lua +** with a number type different from double. You may also need to +** change lua_number2int & lua_number2integer. +** =================================================================== +*/ + +#define LUA_NUMBER_DOUBLE +#define LUA_NUMBER double + +/* +@@ LUAI_UACNUMBER is the result of an 'usual argument conversion' +@* over a number. +*/ +#define LUAI_UACNUMBER double + + +/* +@@ LUA_NUMBER_SCAN is the format for reading numbers. +@@ LUA_NUMBER_FMT is the format for writing numbers. +@@ lua_number2str converts a number to a string. +@@ LUAI_MAXNUMBER2STR is maximum size of previous conversion. +@@ lua_str2number converts a string to a number. +*/ +#define LUA_NUMBER_SCAN "%lf" +#define LUA_NUMBER_FMT "%.14g" +#define lua_number2str(s,n) sprintf((s), LUA_NUMBER_FMT, (n)) +#define LUAI_MAXNUMBER2STR 32 /* 16 digits, sign, point, and \0 */ +#define lua_str2number(s,p) strtod((s), (p)) + + +/* +@@ The luai_num* macros define the primitive operations over numbers. +*/ +#if defined(LUA_CORE) +#include +#define luai_numadd(a,b) ((a)+(b)) +#define luai_numsub(a,b) ((a)-(b)) +#define luai_nummul(a,b) ((a)*(b)) +#define luai_numdiv(a,b) ((a)/(b)) +#define luai_nummod(a,b) ((a) - floor((a)/(b))*(b)) +#define luai_numpow(a,b) (pow(a,b)) +#define luai_numunm(a) (-(a)) +#define luai_numeq(a,b) ((a)==(b)) +#define luai_numlt(a,b) ((a)<(b)) +#define luai_numle(a,b) ((a)<=(b)) +#define luai_numisnan(a) (!luai_numeq((a), (a))) +#endif + + +/* +@@ lua_number2int is a macro to convert lua_Number to int. +@@ lua_number2integer is a macro to convert lua_Number to lua_Integer. +** CHANGE them if you know a faster way to convert a lua_Number to +** int (with any rounding method and without throwing errors) in your +** system. In Pentium machines, a naive typecast from double to int +** in C is extremely slow, so any alternative is worth trying. +*/ + +/* On a Pentium, resort to a trick */ +#if defined(LUA_NUMBER_DOUBLE) && !defined(LUA_ANSI) && !defined(__SSE2__) && \ + (defined(__i386) || defined (_M_IX86) || defined(__i386__)) + +/* On a Microsoft compiler, use assembler */ +#if defined(_MSC_VER) + +#define lua_number2int(i,d) __asm fld d __asm fistp i +#define lua_number2integer(i,n) lua_number2int(i, n) + +/* the next trick should work on any Pentium, but sometimes clashes + with a DirectX idiosyncrasy */ +#else + +union luai_Cast { double l_d; long l_l; }; +#define lua_number2int(i,d) \ + { volatile union luai_Cast u; u.l_d = (d) + 6755399441055744.0; (i) = u.l_l; } +#define lua_number2integer(i,n) lua_number2int(i, n) + +#endif + + +/* this option always works, but may be slow */ +#else +#define lua_number2int(i,d) ((i)=(int)(d)) +#define lua_number2integer(i,d) ((i)=(lua_Integer)(d)) + +#endif + +/* }================================================================== */ + + +/* +@@ LUAI_USER_ALIGNMENT_T is a type that requires maximum alignment. +** CHANGE it if your system requires alignments larger than double. (For +** instance, if your system supports long doubles and they must be +** aligned in 16-byte boundaries, then you should add long double in the +** union.) Probably you do not need to change this. +*/ +#define LUAI_USER_ALIGNMENT_T union { double u; void *s; long l; } + + +/* +@@ LUAI_THROW/LUAI_TRY define how Lua does exception handling. +** CHANGE them if you prefer to use longjmp/setjmp even with C++ +** or if want/don't to use _longjmp/_setjmp instead of regular +** longjmp/setjmp. By default, Lua handles errors with exceptions when +** compiling as C++ code, with _longjmp/_setjmp when asked to use them, +** and with longjmp/setjmp otherwise. +*/ +#if defined(__cplusplus) +/* C++ exceptions */ +#define LUAI_THROW(L,c) throw(c) +#define LUAI_TRY(L,c,a) try { a } catch(...) \ + { if ((c)->status == 0) (c)->status = -1; } +#define luai_jmpbuf int /* dummy variable */ + +#elif defined(LUA_USE_ULONGJMP) +/* in Unix, try _longjmp/_setjmp (more efficient) */ +#define LUAI_THROW(L,c) _longjmp((c)->b, 1) +#define LUAI_TRY(L,c,a) if (_setjmp((c)->b) == 0) { a } +#define luai_jmpbuf jmp_buf + +#else +/* default handling with long jumps */ +#define LUAI_THROW(L,c) longjmp((c)->b, 1) +#define LUAI_TRY(L,c,a) if (setjmp((c)->b) == 0) { a } +#define luai_jmpbuf jmp_buf + +#endif + + +/* +@@ LUA_MAXCAPTURES is the maximum number of captures that a pattern +@* can do during pattern-matching. +** CHANGE it if you need more captures. This limit is arbitrary. +*/ +#define LUA_MAXCAPTURES 32 + + +/* +@@ lua_tmpnam is the function that the OS library uses to create a +@* temporary name. +@@ LUA_TMPNAMBUFSIZE is the maximum size of a name created by lua_tmpnam. +** CHANGE them if you have an alternative to tmpnam (which is considered +** insecure) or if you want the original tmpnam anyway. By default, Lua +** uses tmpnam except when POSIX is available, where it uses mkstemp. +*/ +#if defined(loslib_c) || defined(luaall_c) + +#if defined(LUA_USE_MKSTEMP) +#include +#define LUA_TMPNAMBUFSIZE 32 +#define lua_tmpnam(b,e) { \ + strcpy(b, "/tmp/lua_XXXXXX"); \ + e = mkstemp(b); \ + if (e != -1) close(e); \ + e = (e == -1); } + +#else +#define LUA_TMPNAMBUFSIZE L_tmpnam +#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } +#endif + +#endif + + +/* +@@ lua_popen spawns a new process connected to the current one through +@* the file streams. +** CHANGE it if you have a way to implement it in your system. +*/ +#if defined(LUA_USE_POPEN) + +#define lua_popen(L,c,m) ((void)L, fflush(NULL), popen(c,m)) +#define lua_pclose(L,file) ((void)L, (pclose(file) != -1)) + +#elif defined(LUA_WIN) + +#define lua_popen(L,c,m) ((void)L, _popen(c,m)) +#define lua_pclose(L,file) ((void)L, (_pclose(file) != -1)) + +#else + +#define lua_popen(L,c,m) ((void)((void)c, m), \ + luaL_error(L, LUA_QL("popen") " not supported"), (FILE*)0) +#define lua_pclose(L,file) ((void)((void)L, file), 0) + +#endif + +/* +@@ LUA_DL_* define which dynamic-library system Lua should use. +** CHANGE here if Lua has problems choosing the appropriate +** dynamic-library system for your platform (either Windows' DLL, Mac's +** dyld, or Unix's dlopen). If your system is some kind of Unix, there +** is a good chance that it has dlopen, so LUA_DL_DLOPEN will work for +** it. To use dlopen you also need to adapt the src/Makefile (probably +** adding -ldl to the linker options), so Lua does not select it +** automatically. (When you change the makefile to add -ldl, you must +** also add -DLUA_USE_DLOPEN.) +** If you do not want any kind of dynamic library, undefine all these +** options. +** By default, _WIN32 gets LUA_DL_DLL and MAC OS X gets LUA_DL_DYLD. +*/ +#if defined(LUA_USE_DLOPEN) +#define LUA_DL_DLOPEN +#endif + +#if defined(LUA_WIN) +#define LUA_DL_DLL +#endif + + +/* +@@ LUAI_EXTRASPACE allows you to add user-specific data in a lua_State +@* (the data goes just *before* the lua_State pointer). +** CHANGE (define) this if you really need that. This value must be +** a multiple of the maximum alignment required for your machine. +*/ +#define LUAI_EXTRASPACE 0 + + +/* +@@ luai_userstate* allow user-specific actions on threads. +** CHANGE them if you defined LUAI_EXTRASPACE and need to do something +** extra when a thread is created/deleted/resumed/yielded. +*/ +#define luai_userstateopen(L) ((void)L) +#define luai_userstateclose(L) ((void)L) +#define luai_userstatethread(L,L1) ((void)L) +#define luai_userstatefree(L) ((void)L) +#define luai_userstateresume(L,n) ((void)L) +#define luai_userstateyield(L,n) ((void)L) + + +/* +@@ LUA_INTFRMLEN is the length modifier for integer conversions +@* in 'string.format'. +@@ LUA_INTFRM_T is the integer type correspoding to the previous length +@* modifier. +** CHANGE them if your system supports long long or does not support long. +*/ + +#if defined(LUA_USELONGLONG) + +#define LUA_INTFRMLEN "ll" +#define LUA_INTFRM_T long long + +#else + +#define LUA_INTFRMLEN "l" +#define LUA_INTFRM_T long + +#endif + + + +/* =================================================================== */ + +/* +** Local configuration. You can use this space to add your redefinitions +** without modifying the main part of the file. +*/ + + + +#endif + diff --git a/include/luasandbox/lualib.h b/include/luasandbox/lualib.h new file mode 100644 index 0000000..ae047d9 --- /dev/null +++ b/include/luasandbox/lualib.h @@ -0,0 +1,55 @@ +/* +** $Id: lualib.h,v 1.36.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua standard libraries +** See Copyright Notice in lua.h +*/ + + +#ifndef lualib_h +#define lualib_h + +#include "lua.h" + + +/* Key to file-handle type */ +#define LUA_FILEHANDLE "FILE*" + +#define LUA_BASELIBNAME "" +LUALIB_API int (luaopen_base) (lua_State *L); + +#define LUA_COLIBNAME "coroutine" +LUALIB_API int (luaopen_coroutine) (lua_State *L); + +#define LUA_TABLIBNAME "table" +LUALIB_API int (luaopen_table) (lua_State *L); + +#define LUA_IOLIBNAME "io" +LUALIB_API int (luaopen_io) (lua_State *L); + +#define LUA_OSLIBNAME "os" +LUALIB_API int (luaopen_os) (lua_State *L); + +#define LUA_STRLIBNAME "string" +LUALIB_API int (luaopen_string) (lua_State *L); + +#define LUA_MATHLIBNAME "math" +LUALIB_API int (luaopen_math) (lua_State *L); + +#define LUA_DBLIBNAME "debug" +LUALIB_API int (luaopen_debug) (lua_State *L); + +#define LUA_LOADLIBNAME "package" +LUALIB_API int (luaopen_package) (lua_State *L); + + +/* open all previous libraries */ +LUALIB_API void (luaL_openlibs) (lua_State *L); + + + +#ifndef lua_assert +#define lua_assert(x) ((void)0) +#endif + + +#endif diff --git a/include/luasandbox/test/sandbox.h b/include/luasandbox/test/sandbox.h index 3009e54..ea962d0 100644 --- a/include/luasandbox/test/sandbox.h +++ b/include/luasandbox/test/sandbox.h @@ -15,14 +15,12 @@ #include "../error.h" #ifdef _WIN32 -#define LSB_TEST_MODULE_PATH "path = '.\\\\?.lua';cpath = '.\\\\?.dll'\n" #ifdef luasandboxtest_EXPORTS #define LSB_TEST_EXPORT __declspec(dllexport) #else #define LSB_TEST_EXPORT __declspec(dllimport) #endif #else -#define LSB_TEST_MODULE_PATH "path = './?.lua';cpath = './?.so'\n" #if __GNUC__ >= 4 #define LSB_TEST_EXPORT __attribute__ ((visibility ("default"))) #else @@ -36,12 +34,53 @@ extern "C" #endif #include "../lua.h" +/** + * Global variable to store the test output + * + */ LSB_TEST_EXPORT extern const char *lsb_test_output; + +/** + * Global variable storing the length of the current test output string + * + */ LSB_TEST_EXPORT extern size_t lsb_test_output_len; + +/** + * Generaly purpose stderr logger + * + */ LSB_TEST_EXPORT extern lsb_logger lsb_test_logger; -LSB_TEST_EXPORT int lsb_test_process(lsb_lua_sandbox *lsb, double ts); +/** + * Function to emulate processing data + * + * @param lsb Pointer to a lua sandbox + * @param tc Test case number to control how the sandbox state is updated + * + * @return LSB_TEST_EXPORT int Status value returned from the sandbox + */ +LSB_TEST_EXPORT int lsb_test_process(lsb_lua_sandbox *lsb, double tc); + +/** + * Function to emulate outputting summary data + * + * @param lsb Pointer to a lua sandbox + * @param tc Test case number to control what data is returned and how the state + * is updated + * + * @return LSB_TEST_EXPORT int 0 on success, 1 on failure + */ LSB_TEST_EXPORT int lsb_test_report(lsb_lua_sandbox *lsb, double tc); + +/** + * Callback for collecting output, places a pointer to the results in the global + * lsb_test_output variable )not thread safe) + * + * @param lua Lua state + * + * @return LSB_TEST_EXPORT int 0 or lua_error + */ LSB_TEST_EXPORT int lsb_test_write_output(lua_State *lua); #ifdef __cplusplus diff --git a/modules/heka/elasticsearch.lua b/modules/heka/elasticsearch.lua deleted file mode 100644 index 62af0c5..0000000 --- a/modules/heka/elasticsearch.lua +++ /dev/null @@ -1,120 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Elasticsearch Utility Functions - -## Module Configuration Table (common options) -```lua --- Boolean flag, if true then any time interpolation (often used to generate the --- ElasticSeach index) will use the timestamp from the processed message rather --- than the system time. -es_index_from_timestamp == false -- optional, default shown - --- String to use as the `_index` key's value in the generated JSON. --- Supports field interpolation as described below. -index = "heka-%{%Y.%m.%d}" -- optional, default shown - --- String to use as the `_type` key's value in the generated JSON. --- Supports field interpolation as described below. -type_name = "message" -- optional, default shown - --- String to use as the `_id` key's value in the generated JSON. --- Supports field interpolation as described below. -id = nil -- optional, default shown - -``` - -## Functions - -### bulkapi_index_json - -Returns a simple JSON 'index' structure satisfying the [ElasticSearch BulkAPI](http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html) - -*Arguments* -* index (string or nil) - Used as the `_index` key's value in the generated JSON - or nil to omit the key. Supports field interpolation as described below. -* type_name (string or nil) - Used as the `_type` key's value in the generated - JSON or nil to omit the key. Supports field interpolation as described below. -* id (string or nil) - Used as the `_id` key's value in the generated JSON or - nil to omit the key. Supports field interpolation as described below. -* ns (number or nil) - Nanosecond timestamp to use for any strftime field - interpolation into the above fields. Current system time will be used if nil. - -*Return* -* JSON - String suitable for use as ElasticSearch BulkAPI index directive. - -*See* -[Field Interpolation](msg_interpolate.html) ---]] - -local cjson = require "cjson" -local mi = require "heka.msg_interpolate" -local string = require "string" -local assert = assert -local type = type -local read_message = read_message -local read_config = read_config - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module. - -local result_inner = { - _index = nil, - _type = nil, - _id = nil -} - ---[[ Public Interface --]] - -function bulkapi_index_json(index, type_name, id, ns) - local secs - if ns then - secs = ns / 1e9 - end - if index then - result_inner._index = string.lower(mi.interpolate(index, secs)) - else - result_inner._index = nil - end - if type_name then - result_inner._type = mi.interpolate(type_name, secs) - else - result_inner._type = nil - end - if id then - result_inner._id = mi.interpolate(id, secs) - else - result_inner._id = nil - end - return cjson.encode({index = result_inner}) -end - - -function load_encoder_cfg() - local cfg = read_config("encoder_cfg") - - if cfg.es_index_from_timestamp == nil then - cfg.es_index_from_timestamp = false - else - assert(type(cfg.es_index_from_timestamp) == "boolean", - "es_index_from_timestamp must be nil or boolean") - end - - if cfg.index == nil then - cfg.index = "heka-%{%Y.%m.%d}" - else - assert(type(cfg.index) == "string", "index must be nil or a string") - end - - if cfg.type_name == nil then - cfg.type_name = "message" - else - assert(type(cfg.type_name) == "string", "type_name must be nil or a string") - end - - return cfg -end - -return M diff --git a/modules/heka/elasticsearch/payload.lua b/modules/heka/elasticsearch/payload.lua deleted file mode 100644 index 24740cd..0000000 --- a/modules/heka/elasticsearch/payload.lua +++ /dev/null @@ -1,55 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Elasticsearch Encoder for Heka Payload-only Messages - -The message payload must be pre-formatted JSON in an ElasticSearch compatible -format. - -## Module Configuration Table - -[Common Options](../elasticsearch.html) - -## Functions - -### encode - -Creates the ElasticSearch bulk API index JSON and combines it with the -pre-formatted JSON from the message payload (a new line is added if necessary). - -*Arguments* -- none - -*Return* -- JSON (string) - -## Sample Output -```json -{"index":{"_index":"mylogger-2014.06.05","_type":"mytype-host.domain.com"}} -{"json":"data","extracted":"from","message":"payload"} -``` ---]] - --- Imports -local string = require "string" -local es = require "heka.elasticsearch" -local read_message = read_message -local cfg = es.load_encoder_cfg() - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -function encode() - local ns - if cfg.es_index_from_timestamp then ns = read_message("Timestamp") end - local idx_json = es.bulkapi_index_json(cfg.index, cfg.type_name, cfg.id, ns) - local payload = read_message("Payload") or "{}" - if string.match(payload, "\n$", -1) then - return string.format("%s\n%s", idx_json, payload) - end - return string.format("%s\n%s\n", idx_json, payload) -end - -return M diff --git a/modules/heka/msg_interpolate.lua b/modules/heka/msg_interpolate.lua deleted file mode 100644 index 6631228..0000000 --- a/modules/heka/msg_interpolate.lua +++ /dev/null @@ -1,99 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Simple Templating to Transform Message Fields into Strings - -## Functions - -### interpolate - -Interpolates values from the currently processed message into the provided -string value. A `%{}` enclosed field name will be replaced by the field value -from the current message. All message header fields are supported ("Uuid", -"Timestamp", "Type", "Logger", "Severity", "Payload", "EnvVersion", "Pid", -"Hostname"). Any other values will be checked against the defined dynamic -message fields. If no field matches, then a -[C strftime](http://man7.org/linux/man-pages/man3/strftime.3.html) (\*nix) -or -[C89 strftime](http://msdn.microsoft.com/en-us/library/fe06s4ak.aspx) (Windows) -time substitution will be attempted. The time used for time substitution will be -the seconds-from-epoch timestamp passed in as the `secs` argument, if provided. -If `secs` is nil, local system time is used. Note that the message timestamp is -*not* automatically used; if you want to use the message timestamp for time -substitutions, then you need to manually extract it and convert it from -nanoseconds to seconds (i.e. divide by 1e9). - -*Arguments* -* value (string) - String into which message values should be interpolated. -* secs (number or nil) - Timestamp (in seconds since epoch) to use for time - substitutions. If nil, system time will be used. - -*Return* -* string - Original string value with any interpolated message values. ---]] - -local date = require "os".date -local floor = require "math".floor -local string = require "string" -local read_message = read_message -local tostring = tostring -local type = type - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module. - -local function interpolate_match(header_fields, match, secs) - -- First see if it's a message header. - local fp = header_fields[match] - if fp then return fp() end - - -- Second check for a dynamic field. - local fname = string.format("Fields[%s]", match) - local fval = read_message(fname) - if type(fval) == "boolean" then - return tostring(fval) - elseif fval then - return fval - end - -- Finally try to use it as a strftime format string. - fval = date(match, secs) - if fval ~= match then -- Only return it if a substitution happened. - return fval - end -end - ---[[ Public Interface --]] - -header_fields = { - Uuid = function() return get_uuid(read_message("Uuid")) end, - Timestamp = function() return get_timestamp(read_message("Timestamp")) end, - Type = function() return read_message("Type") end, - Logger = function() return read_message("Logger") end, - Severity = function() return read_message("Severity") end, - Payload = function() return read_message("Payload") end, - EnvVersion = function() return read_message("EnvVersion") end, - Pid = function() return read_message("Pid") end, - Hostname = function() return read_message("Hostname") end -} - - -function get_uuid(uuid) - return string.format("%X%X%X%X-%X%X-%X%X-%X%X-%X%X%X%X%X", string.byte(uuid, 1, 16)) -end - - -function get_timestamp(ns) - local time_t = floor(ns / 1e9) - local frac = ns - time_t * 1e9 - local ds = date("!%Y-%m-%dT%H:%M:%S", time_t) - return string.format("%s.%09dZ", ds, frac) -end - - -function interpolate(value, secs) - return string.gsub(value, "%%{(.-)}", function(match) return interpolate_match(header_fields, match, secs) end) -end - -return M diff --git a/modules/heka/util.lua b/modules/heka/util.lua deleted file mode 100644 index 8d209c5..0000000 --- a/modules/heka/util.lua +++ /dev/null @@ -1,64 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Heka Sandbox Utility Module - -## Functions - -### table_to_fields - -Flattens a Lua table so it can be encoded as a protobuf fields object. - -*Arguments* -- hash (table) - table to flatten (not modified) -- fields (table) - table to receive the flattened output -- parent (string) - key prefix -- separator (string) - key separator (default = ".") i.e. 'foo.bar' -- max_depth (number) - maximum nesting before converting the remainder of the - structure to a JSON string - -*Return* -- none - in-place modification of `fields` ---]] - --- Imports -local pairs = pairs -local type = type -local string = require "string" -local cjson = require "cjson" - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -function table_to_fields(t, fields, parent, char, max_depth) - if type(char) ~= "string" then - char = "." - end - - for k,v in pairs(t) do - if parent then - full_key = string.format("%s%s%s", parent, char, k) - else - full_key = k - end - - if type(v) == "table" then - local _, sep_count = string.gsub(full_key, char, "") - local depth = sep_count + 1 - - if type(max_depth) == "number" and depth >= max_depth then - fields[full_key] = cjson.encode(v) - else - table_to_fields(v, fields, full_key, char, max_depth) - end - else - if type(v) ~= "userdata" then - fields[full_key] = v - end - end - end -end - -return M diff --git a/modules/lpeg/cbufd.lua b/modules/lpeg/cbufd.lua deleted file mode 100644 index 0117ab9..0000000 --- a/modules/lpeg/cbufd.lua +++ /dev/null @@ -1,70 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Circular Buffer Delta Module - -## Variables - -* `grammar` - LPEG grammar to parse cbufd output - -## Usage -```lua -local cbufd = require "cbufd".grammar - -function process_message() - local payload = read_message("Payload") - local t = cbufd:match(payload) - if not t then - return -1, "invalid cbufd string" - end - -- use t - return 0 -end -``` - -## Sample Input/Output - -### Circular Buffer Delta Input -``` -{"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]} -1379660520 12075 159901 -1379660280 11837 154880 -``` - -### Lua Table Output -```lua -{ -header='{"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]}', -{12075, 159901, time = 1379660520000000000}, -{11837, 154880, time = 1379660280000000000} -} -``` ---]] - --- Imports -local l = require "lpeg" -l.locale(l) -local tonumber = tonumber - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local function not_a_number() - return 0/0 -end - -local eol = l.P"\n" -local header = l.Cg((1 - eol)^1, "header") * eol -local timestamp = l.digit^1 / "%0000000000" / tonumber -local sign = l.P"-" -local float = l.digit^1 * "." * l.digit^1 -local nan = l.P"nan" / not_a_number -local number = (sign^-1 * (float + l.digit^1)) / tonumber -+ nan -local row = l.Ct(l.Cg(timestamp, "time") * ("\t" * number)^1 * eol) - -grammar = l.Ct(header * row^1) * -1 - -return M diff --git a/modules/lpeg/common_log_format.lua b/modules/lpeg/common_log_format.lua deleted file mode 100644 index 0ba25f8..0000000 --- a/modules/lpeg/common_log_format.lua +++ /dev/null @@ -1,483 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Common Log Format Module - -## Variables - -* `nginx_error_grammar` - LPEG grammar to parse an Nginx error log line - -## Functions - -### build_nginx_grammar - -Constructs an LPEG grammar based on the Nginx log_format configuration string. - -*Arguments* -- log_format (string) - Nginx log_format configuration string - -*Return* -- grammar (LPEG user data object) or an error is thrown - -### build_apache_grammar - -Constructs an LPEG grammar based on the Apache log_format configuration string. - -*Arguments* -- log_format (string) - Apache log_format configuration string - -*Return* -- grammar (LPEG user data object) or an error is thrown - -### normalize_user_agent - -Extracts the browser, version and os information from a user agent string. - -*Arguments* -- ua (string) - user agent string - -*Return* -- browser (string) -- version (number) -- os (string) ---]] - --- Imports -local l = require "lpeg" -l.locale(l) -local string = require "string" -local dt = require "lpeg.date_time" -local ip = require "lpeg.ip_address" -local tonumber = tonumber -local ipairs = ipairs -local pairs = pairs -local error = error -local type = type - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - ---[[ Nginx access log grammar generation and parsing --]] - -local pct_encoded = l.P"%" * l.xdigit * l.xdigit -local unreserved = l.alnum + l.S"-._~" -local sub_delims = l.S"!$&'()*+,;=" - -local last_literal = l.space -local integer = l.P"-"^-1 * l.digit^1 / tonumber -local double = l.digit^1 * "." * l.digit^1 / tonumber -local msec_time = double / dt.seconds_to_ns -local host = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) - + l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation")) - + l.Ct(l.Cg((unreserved + pct_encoded + sub_delims)^1, "value") * l.Cg(l.Cc"hostname", "representation")) -local http_status = l.digit * l.digit * l.digit / tonumber - -local nginx_error_levels = l.Cg(( - l.P"debug" / "7" -+ l.P"info" / "6" -+ l.P"notice" / "5" -+ l.P"warn" / "4" -+ l.P"error" / "3" -+ l.P"crit" / "2" -+ l.P"alert" / "1" -+ l.P"emerg" / "0") -/ tonumber, "Severity") - -local nginx_upstream_sep = ", " -local nginx_upstream_gsep = " : " -local nginx_upstream_addr = l.C((1 - l.S(nginx_upstream_sep))^1) -local nginx_upstream_addrs = nginx_upstream_addr * (nginx_upstream_sep * nginx_upstream_addr)^0 -local nginx_upstream_time = double + l.P"-" / function () return 0 end -local nginx_upstream_times = nginx_upstream_time * (nginx_upstream_sep * nginx_upstream_time)^0 -local nginx_upstream_lengths = integer * (nginx_upstream_sep * integer)^0 -local nginx_upstream_status = http_status + l.P"-" / function () return 0 end -local nginx_upstream_statuses = nginx_upstream_status * (nginx_upstream_sep * nginx_upstream_status)^0 - -local nginx_format_variables = { - --arg_* - --args - binary_remote_addr = l.P(1) * l.P(1) * l.P(1) * l.P(1) - , body_bytes_sent = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - , bytes_sent = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - , connection = integer - , connection_requests = integer - , content_length = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - --content_type - --cookie_* - --document_root - --document_uri - , host = host - , hostname = host - --http_* - , https = l.P"on"^-1 - , is_args = l.P"?"^-1 - --limit_rate - , msec = msec_time - , nginx_version = l.digit^1 * "." * l.digit^1 * "." * l.digit^1 - , pid = integer - , pipe = l.P"p" + "." - , proxy_protocol_addr = host - --query_string - --realpath_root - , remote_addr = host - , remote_port = integer - --remote_user - --request - --request_body - --request_body_file - , request_completion = l.P"OK"^-1 - --request_filename - , request_length = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - , request_method = l.P"GET" + "POST" + "HEAD" + "PUT" + "DELETE" + "OPTIONS" + "TRACE" + "CONNECT" - , request_time = l.Ct(l.Cg(double, "value") * l.Cg(l.Cc"s", "representation")) - --request_uri - , scheme = l.P"https" + "http" - --sent_http_* - , server_addr = host - , server_name = host - , server_port = integer - , server_protocol = l.P"HTTP/" * l.digit^1 * "." * l.digit^1 - , status = http_status - , tcpinfo_rtt = integer - , tcpinfo_rttvar = integer - , tcpinfo_snd_cwnd = integer - , tcpinfo_rcv_space = integer - , time_iso8601 = dt.rfc3339 / dt.time_to_ns - , time_local = dt.clf_timestamp / dt.time_to_ns - --uri - - -- Upstream Module Grammars - -- http://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables - , upstream_addr = l.Ct(nginx_upstream_addrs * (nginx_upstream_gsep * nginx_upstream_addrs)^0) + "-" - , upstream_cache_status = l.P"HIT" + "MISS" + "EXPIRED" + "BYPASS" + "STALE" + "UPDATING" + "REVALIDATED" + "-" - , upstream_cache_last_modified = dt.build_strftime_grammar("%a, %d %b %Y %T GMT") / dt.time_to_ns + "-" - , upstream_response_length = l.Ct(l.Cg(l.Ct(nginx_upstream_lengths * (nginx_upstream_gsep * nginx_upstream_lengths)^0), "value") * l.Cg(l.Cc"B", "representation")) + "-" - , upstream_response_time = l.Ct(l.Cg(l.Ct(nginx_upstream_times * (nginx_upstream_gsep * nginx_upstream_times)^0), "value") * l.Cg(l.Cc"s", "representation")) - , upstream_status = l.Ct(nginx_upstream_statuses * (nginx_upstream_gsep * nginx_upstream_statuses)^0) - --upstream_http_* handled by the generic grammar -} - -local function get_string_grammar(s) - if last_literal ~= '"' then - return l.Cg((l.P(1) - last_literal)^0, s) - else - return l.Cg((l.P'\\"' + (l.P(1) - l.P'"'))^0, s) - end -end - -local function nginx_lookup_grammar(var) - local g = nginx_format_variables[var] - if not g then - g = get_string_grammar(var) - elseif var == "time_local" or var == "time_iso8601" or var == "msec" then - g = l.Cg(g, "time") - else - g = l.Cg(g, var) - end - - return g -end - - -local apache_format_variables = { - ["%"] = l.P"%" - , a = l.Cg(host, "remote_addr") - , A = l.Cg(host, "server_addr") - , B = l.Cg(nginx_format_variables["body_bytes_sent"], "body_bytes_sent") - , b = l.Cg(l.Ct(l.Cg(integer + l.P"-" / function () return 0 end, "value") * l.Cg(l.Cc"B", "representation")), "body_bytes_sent") - , D = l.Cg(l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"us", "representation")), "request_time") - , f = "request_filename" - , h = l.Cg(host, "remote_addr") - , H = l.Cg(nginx_format_variables["server_protocol"], "server_protocol") - , k = l.Cg(integer, "connection_requests") - , l = "remote_user" - , L = "request_log_id" - , m = l.Cg(nginx_format_variables["request_method"], "request_method") - , p = l.Cg(integer, "server_port") - , P = l.Cg(integer, "pid") - , q = "query_string" - , r = "request" - , R = "request_handler" - , s = l.Cg(nginx_format_variables["status"], "status") - , t = l.P"[" * l.Cg(nginx_format_variables["time_local"], "time") * "]" - , T = l.Cg(l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"s", "representation")), "request_time") - , u = "remote_user" - , U = "uri" - , v = l.Cg(host, "server_name") - , V = l.Cg(host, "server_name") - , X = l.Cg(l.S"X+-", "connection_status") - , I = l.Cg(nginx_format_variables["request_length"], "request_length") - , O = l.Cg(nginx_format_variables["request_length"], "response_length") - , S = l.Cg(nginx_format_variables["request_length"], "total_length") -} - -local function port_format(a) - if a == "canonical" or a == "local" then - return l.Cg(integer, "server_port") - elseif a == "remote" then - return l.Cg(integer, "remote_port") - end - error("unknown port format: " .. a) -end - -local function pid_format(a) - if a == "pid" then - return l.Cg(integer, "pid") - elseif a == "tid" then - return l.Cg(integer, "tid") - elseif a == "hextid" then - return l.Cg(l.xdigit^1 / function (n) return tonumber(n, 16) end, "tid") - end - error("unknown pid format: " .. a) -end - -local function time_format(a) - local pos = 1 - if string.find(a, "^end:") then - pos = 5 - elseif string.find(a, "^begin:") then - pos = 7 - end - - if string.find(a, "^sec", pos) then - return l.Cg(l.digit^1 / dt.seconds_to_ns, "time") - elseif string.find(a, "^msec_frac", pos) or string.find(a, "^usec_frac", pos) then - return l.Cg(l.digit^1 / function (t) return tonumber("." .. t) end, "sec_frac") - elseif string.find(a, "^msec", pos) then - return l.Cg(l.digit^1 / function (t) return tonumber(t) * 1e6 end, "time") - elseif string.find(a, "^usec", pos) then - return l.Cg(l.digit^1 / function (t) return tonumber(t) * 1e3 end, "time") - end - - return l.Cg(dt.build_strftime_grammar(a) / dt.time_to_ns, "time") -end - -local apache_format_arguments = { - a = function(a) return apache_format_variables["a"] end - , C = function(a) return "cookie_" .. a end - , e = function(a) return "env_" .. a end - , i = function(a) return "http_" .. string.gsub(string.lower(a), "-", "_") end -- make it compatible with the Nginx naming convention - , n = function(a) return "module_" .. a end - , o = function(a) return "sent_http_" .. string.gsub(string.lower(a), "-", "_") end -- make it compatible with the Nginx naming convention - , p = port_format - , P = pid_format - , t = time_format - , x = function(a) return string.gsub(string.lower(a), "-", "_") end -- already prefixed by SSL_ -} - -local function apache_lookup_variable(var) - local g = apache_format_variables[var] - if not g then - error("unknown variable: " .. var) - elseif type(g) == "string" then - g = get_string_grammar(g) - end - - return g -end - -local function apache_lookup_argument(t) - local f = apache_format_arguments[t.variable] - if not f then - error("unknown variable: " .. t.variable) - end - - local g = f(t.argument) - if not g then - error(string.format("variable: %s invalid arguments: %s", t.variable, t.arguments)) - elseif type(g) == "string" then - g = get_string_grammar(g) - end - - return g -end - - -local function space_grammar() - last_literal = l.space - return l.space^1 -end - -local function literal_grammar(var) - last_literal = string.sub(var, -1) - return l.P(var) -end - - -local escape_chars = {t = "\9", n = "\10", ['"'] = '"', ["\\"] = "\\"} -local function substitute_escape(seq) - seq = string.sub(seq, 2) - local e = escape_chars[seq] - if not e then - e = string.char(tonumber(seq)) - end - return e -end - -local escape_sequence = l.Cs(((l.P"\\" * (l.S'tn"\\' + l.digit^-3)) / substitute_escape + 1)^0) -local function escape_literal_grammar(var) - local literal = l.match(escape_sequence, var) - last_literal = string.sub(literal, -1) - return l.P(literal) -end - ---[[ User Agent normalization -Derived from the Persona user agent parser -https://github.com/mozilla/persona/blob/dev/lib/coarse_user_agent_parser.js -]]-- - --- find the version of the last product in the user agent string -local function ua_basic(ua) - local cur, last = 0, nil - while cur do - cur = ua:find("/", cur+1) - if cur then - last = cur - end - end - - if last then - return tonumber(ua:match("^(%d+)", last+1)) - end -end - --- find the version of the product identified by the keyword -local function ua_keyword(kw) - return function(ua) - local i = ua:find(kw, 1, true) - if i then - local s = i + kw:len() - if s then - return tonumber(ua:match("^(%d+)", s)) - end - end - end -end - -local ua_os_matchers = { - -- search, replace - {"iPod" ,"iPod" } - , {"iPad" ,"iPad" } - , {"iPhone" ,"iPhone" } - , {"Android" ,"Android" } - , {"BlackBerry" ,"BlackBerry" } - , {"Linux" ,"Linux" } - , {"Macintosh" ,"Macintosh" } - , {"Mozilla/5.0 (Mobile;" ,"FirefoxOS" } - --http://en.wikipedia.org/wiki/Microsoft_Windows#Timeline_of_releases - , {"Windows NT 10.0" ,"Windows 10" } - , {"Windows NT 6.3" ,"Windows 8.1" } - , {"Windows NT 6.2" ,"Windows 8" } - , {"Windows NT 6.1" ,"Windows 7" } - , {"Windows NT 6.0" ,"Windows Vista"} - , {"Windows NT 5.1" ,"Windows XP" } - , {"Windows NT 5.0" ,"Windows 2000" } -} - - -local fx_sync_android = ua_keyword("Firefox AndroidSync ") -local fx_sync_ios = ua_keyword("Firefox-iOS-Sync/") -local ua_browser_matchers = { - {"Edge" , ua_keyword("Edge/")} - , {"Chrome" , ua_keyword("Chrome/")} - , {"Opera Mini" , ua_basic} - , {"Opera Mobi" , ua_basic} - , {"Opera" , ua_basic} - , {"MSIE" , ua_keyword("MSIE ")} - , {"Trident/7.0" , function() return 11, "MSIE" end} - , {"Safari" , ua_basic} - , {"Firefox AndroidSync" , function(ua) return fx_sync_android(ua), "FxSync", "Android" end} - , {"Firefox-iOS-Sync" , function(ua) return fx_sync_ios(ua), "FxSync", "iOS" end} - , {"Firefox" , ua_keyword("Firefox/")} -} - ---[[ Public Interface --]] - --- Returns an LPeg grammar based on the Nginx log_format configuration string. -function build_nginx_grammar(log_format) - last_literal = l.space - local ws = l.space^1 / space_grammar - local variable = l.P"$" * ((l.alnum + "_")^1 / nginx_lookup_grammar) - local literal = (l.P(1) - (ws + variable))^1 / literal_grammar - local item = ws + variable + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(log_format) - if not t then - error("could not parse the log_format configuration") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - --- Returns the user agent browser, version, and os -function normalize_user_agent(ua) - if not ua then return end - - local browser, version, os - for i, v in ipairs(ua_os_matchers) do - if ua:find(v[1], 1, true) then - os = v[2] - break - end - end - - for i, v in ipairs(ua_browser_matchers) do - if ua:find(v[1], 1, true) then - version, browser, aos = v[2](ua) - if not browser then browser = v[1] end - if aos then os = aos end - break - end - end - - return browser, version, os -end - --- Returns an LPeg grammar based on the Apache LogFormat configuration string. -function build_apache_grammar(log_format) - last_literal = l.space - local ws = l.space^1 / space_grammar - local http_status_code = l.digit * l.digit * l.digit - local http_status_codes = l.P"!"^-1 * http_status_code * ("," * http_status_code)^0 - local variable = l.alpha + "%" - local argument = "{" * l.Cg((l.P(1) - "}")^1, "argument") * "}" - local directive = l.P"%" * l.S"<>"^-1 * (variable / apache_lookup_variable) - + "%" * http_status_codes^-1 * l.Ct(argument * l.Cg(variable, "variable")) / apache_lookup_argument - local literal = (l.P(1) - (ws + directive))^1 / escape_literal_grammar - local item = ws + directive + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(log_format) - if not t then - error("could not parse the log_format configuration") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - -nginx_error_grammar = l.Ct(l.Cg(dt.build_strftime_grammar("%Y/%m/%d %T") / dt.time_to_ns, "Timestamp") - * l.space * "[" * nginx_error_levels * "]" - * l.space * l.Cg(l.digit^1 / tonumber, "Pid") * "#" - * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": " * (l.P"*" * l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") - * l.Cg(l.P(1)^0, "Payload")) - -return M diff --git a/modules/lpeg/date_time.lua b/modules/lpeg/date_time.lua deleted file mode 100644 index f1c959a..0000000 --- a/modules/lpeg/date_time.lua +++ /dev/null @@ -1,326 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Date Time Module - -## Variables -### LPEG Grammars -* `date_fullyear` - 4 digit year -* `date_month` - 2 digit month -* `date_mabbr` - month abbreviation to a numeric string -* `date_mfull` - month name to a numeric string -* `date_mday` - 2 digit month day (zero padded) -* `date_mday_sp` - 2 digit month day (space padded) -* `time_hour` - 2 digit hour -* `time_minute` - 2 digit minute -* `time_second` - 2 digit second (including leap second) -* `time_secfrac` - fraction seconds -* `timezone` - timezone abbreviations (US and UTC others create an offset of 0) -* `timezone_offset` - numeric time zone offset -* `clf_timestamp` - common log format timestamp -* `mysql_timestamp` - MySQL timestamp -* `pgsql_timestamp` - Postgres timestamp -* `rfc3164_timestamp` -* `rfc3339_time_numoffset` -* `rfc3339_time_offset` -* `rfc3339_partial_time` -* `rfc3339_full_date` -* `rfc3339_full_time` -* `rfc3339` - - -## Functions - -### build_strftime_grammar - -Constructs an LPEG grammar based on the strftime format. - -*Arguments* -- strftime (string) - strftime format specifier - -*Return* -- grammar (LPEG user data object) or an error is thrown - -### time_to_ns - -Converts time table to a time_ns - -*Arguments* -- t (table) - table returned by the various date/time grammars - -*Return* -- time_ns (number) - number of nanoseconds since the Unix epoch - -### seconds_to_ns - -Converts time_t to a time_ns - -*Arguments* -- time_t - -*Return* -- time_ns (number) - number of nanoseconds since the Unix epoch - ---]] - --- Imports -local l = require "lpeg" -l.locale(l) -local os = require "os" -local string = require "string" -local tonumber = tonumber -local ipairs = ipairs -local error = error -local type = type - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - ---[[ Utility Functions --]] -function time_to_ns(t) - if type(t) ~= "table" then return 0 end - if t.time_t then return t.time_t * 1e9 end - - local offset = 0 - if t.offset_hour then - offset = (t.offset_hour * 60 * 60) + (t.offset_min * 60) - if t.offset_sign == "+" then offset = offset * -1 end - end - - if t.period then - t.hour = tonumber(t.hour) - if t.period == "AM" then - if t.hour == 12 then - t.hour = 0 - end - elseif t.period == "PM" then - if t.hour < 12 then - t.hour = t.hour + 12 - end - end - end - - local frac = 0 - if t.sec_frac then - frac = t.sec_frac - end - - local ost = os.time(t) - if not ost then - return 0 - end - - return (ost + frac + offset) * 1e9 -end - ---Converts a second value into the number of nanoseconds since the UNIX epoch -function seconds_to_ns(sec) - return tonumber(sec) * 1e9 -end - - ---[[ Generic Grammars --]] -date_fullyear = l.Cg(l.digit * l.digit * l.digit * l.digit, "year") - -date_month = l.Cg(l.P"0" * l.R"19" - + "1" * l.R"02", "month") -date_mabbr = l.Cg( - l.P"Jan" /"1" - + l.P"Feb" /"2" - + l.P"Mar" /"3" - + l.P"Apr" /"4" - + l.P"May" /"5" - + l.P"Jun" /"6" - + l.P"Jul" /"7" - + l.P"Aug" /"8" - + l.P"Sep" /"9" - + l.P"Oct" /"10" - + l.P"Nov" /"11" - + l.P"Dec" /"12" - , "month") - -date_mfull = l.Cg( - l.P"January" /"1" - + l.P"February" /"2" - + l.P"March" /"3" - + l.P"April" /"4" - + l.P"May" /"5" - + l.P"June" /"6" - + l.P"July" /"7" - + l.P"August" /"8" - + l.P"September" /"9" - + l.P"October" /"10" - + l.P"November" /"11" - + l.P"December" /"12" - , "month") - -date_mday = l.Cg(l.P"0" * l.R"19" - + l.R"12" * l.R"09" - + "3" * l.R"01", "day") - -date_mday_sp = l.Cg(l.P" " * l.R"19" - + l.S"12" * l.digit - + "3" * l.S"01", "day") - -time_hour = l.Cg(l.R"01" * l.digit - + "2" * l.R"03", "hour") - -time_minute = l.Cg(l.R"05" * l.digit, "min") - -time_second = l.Cg(l.R"05" * l.digit - + "60", "sec") -- include leap second - -time_secfrac = l.Cg(l.P"." * l.digit^1 / tonumber, "sec_frac") - -timezone = - l.P"UTC" * l.Cg(l.Cc"+", "offset_sign") * l.Cg(l.Cc"00" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"PST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"08" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"PDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"07" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"MST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"07" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"MDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"06" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"CST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"06" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"CDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"05" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"EST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"05" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"EDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"04" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.alpha^-5 * l.Cg(l.Cc"+", "offset_sign") * l.Cg(l.Cc"00" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - -timezone_offset = l.Cg(l.S"+-", "offset_sign") * l.Cg(time_hour / tonumber, "offset_hour") * l.Cg(time_minute / tonumber, "offset_min") - ---[[ RFC3339 grammar -sample input: 1999-05-05T23:23:59.217-07:00 - -output table: -hour=23 (string) -min=23 (string) -year=1999 (string) -month=05 (string) -day=05 (string) -sec=59 (string) -*** conditional table members *** -sec_frac=0.217 (number) -offset_sign=- (string) -offset_hour=7 (number) -offset_min=0 (number) ---]] -rfc3339_time_numoffset = l.Cg(l.S"+-", "offset_sign") -* l.Cg(time_hour / tonumber, "offset_hour") -* ":" -* l.Cg(time_minute / tonumber, "offset_min") - -rfc3339_time_offset = l.P"Z" + rfc3339_time_numoffset -rfc3339_partial_time = time_hour * ":" * time_minute * ":" * time_second * time_secfrac^-1 -rfc3339_full_date = date_fullyear * "-" * date_month * "-" * date_mday -rfc3339_full_time = rfc3339_partial_time * rfc3339_time_offset -rfc3339 = l.Ct(rfc3339_full_date * "T" * rfc3339_full_time) - ---[[ strftime Grammars --]] -local century = os.date("%Y"):sub(1,2) -local strftime_specifiers = {} -strftime_specifiers["a"] = l.P"Mon" + "Tue" + "Wed" + "Thu" + "Fri" + "Sat" + "Sun" -strftime_specifiers["A"] = l.P"Monday" + "Tuesday" + "Wednesday" + "Thursday" + "Friday" + "Saturday" + "Sunday" -strftime_specifiers["b"] = date_mabbr -strftime_specifiers["B"] = date_mfull -strftime_specifiers["C"] = l.digit * l.digit -strftime_specifiers["y"] = l.Cg((l.digit * l.digit) / function (yy) return century .. yy end, "year") -strftime_specifiers["d"] = date_mday -strftime_specifiers["D"] = date_month * "/" * date_mday * "/" * strftime_specifiers["y"] -strftime_specifiers["e"] = date_mday_sp -strftime_specifiers["f"] = l.Cg(l.digit^1 / ".%0" / tonumber, "sec_frac") -strftime_specifiers["F"] = date_fullyear * "-" * date_month * "-" * date_mday -strftime_specifiers["g"] = strftime_specifiers["y"] -strftime_specifiers["G"] = date_fullyear -strftime_specifiers["h"] = date_mabbr -strftime_specifiers["H"] = time_hour -strftime_specifiers["I"] = l.Cg(l.P"0" * l.digit - + "1" * l.R"02", "hour") -strftime_specifiers["j"] = l.P"00" * l.R"19" - + "0" * l.digit * l.digit - + l.S"12" * l.digit * l.digit - + "3" * l.R"05" * l.digit - + "36" * l.R"06" -strftime_specifiers["k"] = l.Cg(l.S" 1" * l.digit - + "2" * l.R"03", "hour") -strftime_specifiers["l"] = l.Cg(l.space * l.digit - + "1" * l.R"02", "hour") -strftime_specifiers["m"] = date_month -strftime_specifiers["M"] = time_minute -strftime_specifiers["n"] = l.P"\n" -strftime_specifiers["p"] = l.Cg(l.P"AM" + "PM", "period") -strftime_specifiers["r"] = strftime_specifiers["I"] * ":" * time_minute * ":" * time_second * " " * strftime_specifiers["p"] -strftime_specifiers["R"] = time_hour * ":" * time_minute -strftime_specifiers["s"] = l.Cg(l.digit^1 / tonumber, "time_t") -strftime_specifiers["S"] = l.Cg(l.R"05" * l.digit - + "6" * l.S"01", "sec") -strftime_specifiers["t"] = l.P"\t" -strftime_specifiers["T"] = time_hour * ":" * time_minute * ":" * time_second -strftime_specifiers["u"] = l.R"17" -strftime_specifiers["U"] = l.R"04" * l.digit - + l.P"5" * l.R"03" -strftime_specifiers["V"] = strftime_specifiers["U"] -strftime_specifiers["w"] = l.R"06" -strftime_specifiers["W"] = strftime_specifiers["U"] -strftime_specifiers["x"] = strftime_specifiers["D"] -strftime_specifiers["X"] = strftime_specifiers["T"] -strftime_specifiers["Y"] = date_fullyear -strftime_specifiers["z"] = timezone_offset -strftime_specifiers["Z"] = timezone -strftime_specifiers["%"] = l.P"%" -if os.date("%c"):find("^%d") then -- windows - strftime_specifiers["c"] = strftime_specifiers["D"] * " " * strftime_specifiers["T"] - strftime_specifiers["z"] = timezone -- todo depending on the registry this could be the full name and not the abbreviation -else - strftime_specifiers["c"] = strftime_specifiers["a"] * " " * date_mabbr * " " * date_mday_sp * " " * strftime_specifiers["T"] * " " * date_fullyear -end - -local function strftime_lookup_grammar(var) - local g = strftime_specifiers[var] - if not g then - error("unknown strftime specifier: " .. var) - end - return g -end - --- Returns an LPeg grammar based on the strftime format. -function build_strftime_grammar(format) - local ws = l.space / function () return l.space end - local variable = l.P"%" * ((l.alnum + "%") / strftime_lookup_grammar) - local literal = (l.P(1) - (ws + variable))^1 / function (lit) return l.P(lit) end - local item = ws + variable + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(format) - if not t then - error("could not parse the strftime format") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - - ---[[ Common Log Format Grammars --]] --- strftime format %d/%b/%Y:%H:%M:%S %z. -clf_timestamp = l.Ct(date_mday * "/" * date_mabbr * "/" * date_fullyear * ":" * strftime_specifiers["T"] * " " * timezone_offset) - - ---[[ RFC3164 Grammars --]] -local sp = l.P" " --- since we consume multiple spaces this will also handle date-rfc3164-buggyday -rfc3164_timestamp = l.Ct(date_mabbr * sp^1 * l.Cg(l.digit^-2, "day") * sp^1 * strftime_specifiers["T"] * l.Cg(l.Cc("") / function() return os.date("%Y") end, "year")) - - ---[[ Database Grammars --]] -mysql_timestamp = l.Ct(date_fullyear * date_month * date_mday * time_hour * time_minute * time_second) - -pgsql_timestamp = l.Ct(rfc3339_full_date * sp * strftime_specifiers["T"] ) - -return M diff --git a/modules/lpeg/ip_address.lua b/modules/lpeg/ip_address.lua deleted file mode 100644 index 8f26326..0000000 --- a/modules/lpeg/ip_address.lua +++ /dev/null @@ -1,59 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# IP Address Module - -## Variables -### LPEG Grammars -* `v4` - IPv4 address matcher -* `v6` - IPv6 address matcher -* `hostname` - URI reg-name -* `host` - v6 | v4 | hostname -* `host_field` - returns the host as a Heka message field table `{value="127.0.0.1", representation="ipv4"}` ---]] - --- Imports -local string = require "string" -local l = require "lpeg" -l.locale(l) - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local d8 = "1" * l.digit * l.digit - + "2" * l.R"04" * l.digit - + "25" * l.R"05" - + l.R"19" * l.digit - + l.digit -v4 = d8 * "." * d8 * "." * d8 * "." * d8 - - -local h16 = l.xdigit * l.xdigit^-3 -local hg = h16 * ":" -local ls32 = hg * h16 - + v4 - -v6 = hg * hg * hg * hg * hg * hg * ls32 -+ "::" * hg * hg * hg * hg * hg * ls32 -+ h16^-1 * "::" * hg * hg * hg * hg * ls32 -+ (hg * hg^-1 + ":") * ":" * hg * hg * hg * ls32 -+ (hg * hg^-2 + ":") * ":" * hg * hg * ls32 -+ (hg * hg^-3 + ":") * ":" * hg * ls32 -+ (hg * hg^-4 + ":") * ":" * ls32 -+ (hg * hg^-5 + ":") * ":" * h16 -+ (hg * hg^-6 + ":") * ":" - - -local unreserved = l.alnum + l.S"-._~" -local pct_encoded = l.P"%" * l.xdigit * l.xdigit -local sub_delims = l.S"!$&'()*+,;=" - -hostname = (unreserved + pct_encoded + sub_delims)^1 / string.lower -host = v6 + v4 + hostname -host_field = l.Ct(l.Cg(v6, "value") * l.Cg(l.Cc"ipv6", "representation")) - + l.Ct(l.Cg(v4, "value") * l.Cg(l.Cc"ipv4", "representation")) - + l.Ct(l.Cg(hostname, "value") * l.Cg(l.Cc"hostname", "representation")) - -return M diff --git a/modules/lpeg/mysql.lua b/modules/lpeg/mysql.lua deleted file mode 100644 index 1a5ccee..0000000 --- a/modules/lpeg/mysql.lua +++ /dev/null @@ -1,129 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# MySQL Module - -## Variables -### LPEG Grammars -* `slow_query_grammar` -* `mariadb_slow_query_grammar` -* `percona_slow_query_grammar` -* `short_slow_query_grammar` -* `mariadb_short_slow_query_grammar` ---]] - --- Imports -local l = require "lpeg" -l.locale(l) -local tonumber = tonumber -local ip = require "lpeg.ip_address" - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local unreserved = l.alnum + l.S"-._~" -local space = l.space^1 -local sep = l.P"\n" -local sql_end = l.P";" * (l.P"\n" + -1) -local line = (l.P(1) - sep)^0 * sep -local float = l.digit^1 * "." * l.digit^1 / tonumber -local integer = l.P"-"^-1 * l.digit^1 / tonumber - -local time = l.P"# Time: " * line - -local user_legal = (1 - l.S"[]") -local user_name = user_legal^0 * "[" * l.Cg(user_legal^0, "Username") * "]" -local host_name = ip.hostname^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" -local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep - -local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Query_time") -local lock_time = l.P"Lock_time: " * l.Cg(l.Ct(l.Cg(float, "value") * l.Cg(l.Cc"s", "representation")), "Lock_time") -local rows_sent = l.P"Rows_sent: " * l.Cg(integer, "Rows_sent") -local rows_examined = l.P"Rows_examined: " * l.Cg(integer, "Rows_examined") -local query = query_time * space * lock_time * space * rows_sent * space * rows_examined * sep - -local use_db = l.P"use " * line - -local last_insert = l.P"last_insert_id=" * l.Cg(integer, "Last_insert") * "," -local insert = l.P"insert_id=" * l.Cg(integer, "Insert_id") * "," -local timestamp = l.P"timestamp=" * l.Cg((l.digit^1 / "%0000000000"), "Timestamp") -local set = l.P"SET " * last_insert^0 * insert^0 * timestamp * ";" * sep - -local admin = l.P"# administrator command: " * line - -local sql = l.Cg((l.P(1) - sql_end)^0 * sql_end, "Payload") - --- Maria DB extensions -local yes_no = l.C(l.P"Yes" + "No") -local thread_id = l.P"# Thread_id: " * l.Cg(integer, "Thread_id") - * l.P" Schema: " * l.Cg(unreserved^0, "Schema") - * l.P" QC_hit: " * l.Cg(yes_no, "QC_hit") * sep - -local full_scan = l.P"# Full_scan: " * l.Cg(yes_no, "Full_scan") - * " Full_join: " * l.Cg(yes_no, "Full_join") - * " Tmp_table: " * l.Cg(yes_no, "Tmp_table") - * " Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep - * l.P"# Filesort: " * l.Cg(yes_no, "Filesort") - * " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk") - * " Merge_passes: " * l.Cg(integer, "Merge_passes") * sep - --- Percona extensions -local percona_user = "# User@Host: " * user_name * space * "@" * space * host_name * sep^0 -local conn_id = " Id:" * space * l.Cg(integer, "Connection_id") * sep - -local info = l.P(("# Thread_id: " * l.Cg(integer, "Thread_id") - * " Schema: " * l.Cg(unreserved^0, "Schema") - * " Last_errno: " * l.Cg(integer, "Last_errno") - * " Killed: " * l.Cg(integer, "Killed")) - + ("# Schema: " * l.Cg(unreserved^0, "Schema") - * " Last_errno: " * l.Cg(integer, "Last_errno") - * " Killed: " * l.Cg(integer, "Killed"))) * sep - -local percona_query = "# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") - * l.Cg(l.Cc"s", "representation")), "Query_time") - * " Lock_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") - * l.Cg(l.Cc"s", "representation")), "Lock_time") - * " Rows_sent: " * l.Cg(integer, "Rows_sent") - * " Rows_examined: " * l.Cg(integer, "Rows_examined") - * " Rows_affected: " * l.Cg(integer, "Rows_affected") - * (" Rows_read: " * l.Cg(integer, "Rows_read"))^0 * sep - -local memory_footprint = "# Bytes_sent: " * l.Cg(l.Ct(l.Cg(integer, "value") - * l.Cg(l.Cc"B", "representation")), "Bytes_sent") - * (" Tmp_tables: " * l.Cg(integer, "Tmp_tables") - * " Tmp_disk_tables: " * l.Cg(integer, "Tmp_disk_tables") - * " Tmp_table_sizes: " * l.Cg(l.Ct(l.Cg(integer, "value") - * l.Cg(l.Cc"B", "representation")), "Tmp_table_sizes"))^0 * sep - -local stored_routine = "# Stored routine: " * l.Cg((l.alnum + l.S"_.-")^1, "Stored_routine") * sep - -local query_plan_info = "# QC_Hit: " * l.Cg(yes_no, "QC_hit") - * " Full_scan: " * l.Cg(yes_no, "Full_scan") - * " Full_join: " * l.Cg(yes_no, "Full_join") - * " Tmp_table: " * l.Cg(yes_no, "Tmp_table") - * " Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep - * "# Filesort: " * l.Cg(yes_no, "Filesort") - * " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk") - * " Merge_passes: " * l.Cg(integer, "Merge_passes") * sep - -local innodb_usage = "# InnoDB_IO_r_ops: " * l.Cg(integer, "InnoDB_IO_r_ops") - * " InnoDB_IO_r_bytes: " * l.Cg(l.Ct(l.Cg(integer, "value") - * l.Cg(l.Cc"B", "representation")), "InnoDB_IO_r_bytes") - * " InnoDB_IO_r_wait: " * l.Cg(l.Ct(l.Cg(float, "value") - * l.Cg(l.Cc"s", "representation")), "InnoDB_IO_r_wait") * sep - * "# InnoDB_rec_lock_wait: " * l.Cg(l.Ct(l.Cg(float, "value") - * l.Cg(l.Cc"s", "representation")), "InnoDB_rec_lock_wait") - * " InnoDB_queue_wait: " * l.Cg(l.Ct(l.Cg(float, "value") - * l.Cg(l.Cc"s", "representation")), "InnoDB_queue_wait") * sep - * "# InnoDB_pages_distinct: " * l.Cg(integer, "InnoDB_pages_distinct") * sep - -slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) -mariadb_slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) -percona_slow_query_grammar = l.Ct(time^0 * percona_user * l.Cg(l.Ct(conn_id^0 * info^0 * percona_query * memory_footprint^0 * stored_routine^0 * query_plan_info^0 * innodb_usage^0), "Fields") * use_db^0 * set * admin^0 * sql) - -short_slow_query_grammar = l.Ct(l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) -mariadb_short_slow_query_grammar = l.Ct(l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) - -return M diff --git a/modules/lpeg/postfix.lua b/modules/lpeg/postfix.lua deleted file mode 100644 index 23dce4a..0000000 --- a/modules/lpeg/postfix.lua +++ /dev/null @@ -1,540 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Copyright 2015 Mathieu Parent - ---[[ -# Postfix Module - -## Function -### function postfix_match - -Parses a variety of postfix log message types. - -*Arguments* -- programname (string) - name of the logging program e.g., 'smtpd' -- message (string) - message string to match -- extract_keyvalue_data (bool) - true if key/value pairs should be split out into table entries - -*Return* -- matched results (table) or nil if no match ---]] - -local string = require 'string' -local ip = require 'lpeg.ip_address' -local l = require 'lpeg' -l.locale(l) -local ipairs = ipairs -local pairs = pairs -local rawset = rawset -local tonumber = tonumber -local type = type - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - --- Basic types -local notspace = l.P(1) - l.space -local integer = l.digit^1 / tonumber -local number = (l.digit^1 * (l.P'.' * l.digit^1)^-1) / tonumber - --- keyvalue_data -local kv_key = l.C(l.alpha^1) -local kv_pair_end = (l.P', ' * l.alpha^1 * l.P'=') + (l.P',' * -1) + -1 -local kv_value = l.P'<' * l.C((l.P(1) - (l.P'>' * kv_pair_end))^0) * l.P'>' - + l.C((l.P(1) - kv_pair_end)^0) -local kv_sep = l.P', ' + (l.P',' * -1) + -1 -local kv_pair = l.Cg(kv_key * '=' * kv_value) * kv_sep^-1 -local kv_grammar = l.Cf(l.Ct'' * kv_pair^0, rawset) - --- Hostname and IPs -local ipv46 = l.Ct(l.Cg(ip.v4, 'value') * l.Cg(l.Cc'ipv4', 'representation')) - + l.Ct(l.Cg(ip.v6, 'value') * l.Cg(l.Cc'ipv6', 'representation')) -local hostname = l.Ct(l.Cg((l.P(1) - l.S'[] ')^1, 'value') * l.Cg(l.Cc'hostname', 'representation')) -local host = ipv46 + hostname -local port = l.Ct(l.Cg(integer, 'value') * l.Cg(l.Cc'2', 'value_type')) - --- Postfix grammar --- Inspired by https://github.com/whyscream/postfix-grok-patterns --- 2015-05-26 8fdd562d159845b97bf8d9c273e5357f3a143a94 --- Original under BSD-3-clause license --- Adaptation 2015 Mathieu Parent -local postfix_queueid = l.xdigit ^ 6 + l.alnum ^ 15 + l.P'NOQUEUE' -local postfix_queueid_cg = l.Cg(postfix_queueid, 'postfix_queueid') - -local postfix_client_hostname_cg = l.Cg(host, 'postfix_client_hostname') -local postfix_client_ip_cg = l.Cg(ipv46, 'postfix_client_ip') -local postfix_client_port_cg = l.Cg(port, 'postfix_client_port') -local postfix_client_info_cg = postfix_client_hostname_cg^-1 - * l.P'[' * postfix_client_ip_cg * l.P']' - * (l.P':' * postfix_client_port_cg)^-1 -local postfix_client_info_ct = l.Ct(postfix_client_info_cg) - -local postfix_relay_hostname_cg = l.Cg(host, 'postfix_relay_hostname') -local postfix_relay_ip_cg = l.Cg(ipv46, 'postfix_relay_ip') -local postfix_relay_service_cg = l.Cg((l.P(1)-l.P']')^1, 'postfix_relay_service') -local postfix_relay_port_cg = l.Cg(port, 'postfix_relay_port') -local postfix_relay_info_cg = postfix_relay_hostname_cg^-1 - * l.P'[' * (postfix_relay_ip_cg + postfix_relay_service_cg) * l.P']' - * (l.P':' * postfix_relay_port_cg)^-1 -local postfix_relay_info_ct = l.Ct(postfix_relay_info_cg) - -local postfix_smtp_stage = l.P'CONNECT' - + l.P'HELO' - + l.P'EHLO' - + l.P'STARTTLS' - + l.P'AUTH' - + l.P'MAIL' - + l.P'RCPT' - + l.P'DATA' - + l.P'RSET' - + l.P'UNKNOWN' - + l.P'END-OF-MESSAGE' - + l.P'VRFY' - + l.P'.' -local postfix_smtp_stage_cg = l.Cg(postfix_smtp_stage, 'postfix_smtp_stage') -local postfix_proxy_smtp_stage_cg = l.Cg(postfix_smtp_stage, 'postfix_proxy_smtp_stage') - -local postfix_action = l.P'reject' - + l.P'defer' - + l.P'accept' - + l.P'header-redirect' -local postfix_action_cg = l.Cg(postfix_action, 'postfix_action') -local postfix_proxy_result_cg = l.Cg(postfix_action, 'postfix_proxy_result') - -local postfix_status_code =( l.digit * l.digit * l.digit) / tonumber -local postfix_status_code_cg = l.Cg(postfix_status_code, 'postfix_status_code') -local postfix_status_code_enhanced = l.digit * l.P'.' * l.digit * l.P'.' * l.digit -local postfix_status_code_enhanced_cg = l.Cg(postfix_status_code_enhanced, 'postfix_status_code_enhanced') -local postfix_dnsbl_message_cg = l.P'Service unavailable; ' - * (l.P(1) - l.P'[')^0 - * l.P'[' - * l.Cg((l.P(1) - l.P']')^0, 'postfix_status_data') - * l.P'] ' - * l.Cg((l.P(1)-(l.P'; ' * l.alpha^1 * l.P'='))^1, 'postfix_status_message') - * l.P';' -local postfix_ps_access_action = l.P'DISCONNECT' - + l.P'BLACKLISTED' - + l.P'WHITELISTED' - + l.P'WHITELIST VETO' - + l.P'PASS NEW' - + l.P'PASS OLD' -local postfix_ps_violation_cg = l.Cg(l.P'BARE NEWLINE' - + l.P'COMMAND TIME LIMIT' - + l.P'COMMAND COUNT LIMIT' - + l.P'COMMAND LENGTH LIMIT' - + l.P'COMMAND PIPELINING' - + l.P'DNSBL' - + l.P'HANGUP' - + l.P'NON-SMTP COMMAND' - + l.P'PREGREET', - 'postfix_postscreen_violation') -local postfix_time_unit = l.digit^1 * l.S'smhd' -local postfix_keyvalue_greedy_cg = postfix_queueid_cg - * l.P': ' - * l.Cg(l.P(1)^1, 'postfix_keyvalue_data') -local postfix_warning_cg = (l.P'warning' + l.P'fatal') * ': ' * l.Cg(l.P(1)^1, 'postfix_warning') -local postfix_tlsconn_cg = (l.P'Anonymous' + l.P'Trusted' + l.P'Untrusted' + l.P'Verified') - * l.P' TLS connection established ' - * ((l.P'to ' * postfix_relay_info_cg) + (l.P'from ' * postfix_client_info_cg)) * l.P': ' - * l.Cg(notspace^0, 'postfix_tls_version') - * l.P' with cipher ' - * l.Cg(notspace^0, 'postfix_tls_cipher') - * l.P' (' * l.Cg(notspace^0, 'postfix_tls_cipher_size') * l.P' bits)' -local postfix_delays_ct = l.Ct(l.Cg(number, 'postfix_delay_before_qmgr') - * l.P'/' - * l.Cg(number, 'postfix_delay_in_qmgr') - * l.P'/' - * l.Cg(number, 'postfix_delay_conn_setup') - * l.P'/' - * l.Cg(number, 'postfix_delay_transmission')) -local postfix_lostconn = l.P'lost connection' - + l.P'timeout' - + l.P'Connection timed out' -local postfix_smtpd_lostconn_data_cg = l.Cg(postfix_lostconn, 'postfix_smtpd_lostconn_data') -local postfix_proxy_message_cg = (l.Cg(postfix_status_code, 'postfix_proxy_status_code') * l.P' ')^-1 * (l.Cg(postfix_status_code_enhanced, 'postfix_proxy_status_code_enhanced') * l.P' ')^-1 * (l.P(1) - l.P';')^0 - --- smtpd patterns -local postfix_smtpd_connect_cg = l.P'connect from ' - * postfix_client_info_cg -local postfix_smtpd_disconnect_cg = l.P'disconnect from ' - * postfix_client_info_cg -local postfix_smtpd_lostconn_cg = (postfix_smtpd_lostconn_data_cg - * l.P' after ' - * postfix_smtp_stage_cg - * (l.P' (' * l.digit^1 * l.P' bytes)') ^-1 - * l.P' from ' - * postfix_client_info_cg) - + (l.Cg((l.P(1) - l.P' from ')^1, 'postfix_action') - * l.P' from ' - * postfix_client_info_cg - * l.P': ' - * postfix_smtpd_lostconn_data_cg) -local postfix_smtpd_noqueue_cg = l.P'NOQUEUE: ' - * postfix_action_cg - * l.P': ' - * postfix_smtp_stage_cg - * l.P' from ' - * postfix_client_info_cg - * l.P': ' - * postfix_status_code_cg - * l.P' ' - * postfix_status_code_enhanced_cg - * (l.P' <' * l.Cg((l.P(1) - l.P'>')^1, 'postfix_status_data') * l.P'>:')^-1 - * l.P' ' - * (postfix_dnsbl_message_cg + (l.Cg((l.P(1)-l.P';')^0, 'postfix_status_message') * l.P';')) - * l.P' ' - * l.Cg(l.P(1)^1, 'postfix_keyvalue_data') -local postfix_smtpd_pipelining_cg = l.P'improper command pipelining after ' * postfix_smtp_stage_cg * l.P' from ' * postfix_client_info_cg * l.P':' -local postfix_smtpd_proxy_cg = l.P'proxy-' - * postfix_proxy_result_cg - * l.P': ' - * postfix_proxy_smtp_stage_cg - * l.P': ' - * postfix_proxy_message_cg -- FIXME this should fill postfix_proxy_message - * l.P'; ' - * l.Cg(l.P(1)^1, 'postfix_keyvalue_data') - --- cleanup patterns -local postfix_cleanup_milter_cg = postfix_queueid_cg - * l.P': ' - * l.P'milter-' - * l.Cg(postfix_action, 'postfix_milter_result') - * l.P': ' - * l.Cg((l.P(1)-l.P';')^1, 'postfix_milter_message') - * l.P'; ' - * l.Cg((l.P(1)-l.P':')^1, 'postfix_keyvalue_data') - *(l.P': ' * l.Cg(l.P(1)^1, 'postfix_milter_data'))^-1 - --- qmgr patterns -local postfix_qmgr_removed_cg = postfix_queueid_cg * l.P': removed' -local postfix_qmgr_active_cg = postfix_queueid_cg - * l.P': ' - * l.Cg((l.P(1) - l.P' (')^1, 'postfix_keyvalue_data') - * l.P' (queue active)' - --- pipe patterns -local postfix_pipe_delivered_cg = postfix_queueid_cg - * l.P': ' - * l.Cg((l.P(1) - l.P' (')^1, 'postfix_keyvalue_data') - * l.P' (delivered via ' - * l.Cg(l.alnum^1, 'postfix_pipe_service') -local postfix_pipe_forward_cg = postfix_queueid_cg - * l.P': ' - * l.Cg((l.P(1) - l.P' (')^1, 'postfix_keyvalue_data') - * l.P' (mail forwarding loop for ' - * l.Cg((l.P(1) - l.P')')^1, 'postfix_to') - --- postscreen patterns -local postfix_ps_connect_cg = l.P'CONNECT from ' - * postfix_client_info_cg - * l.P' to [' - * l.Cg(ipv46, 'postfix_server_ip') - * l.P']:' - * l.Cg(number, 'postfix_server_port') -local postfix_ps_access_cg = l.Cg(postfix_ps_access_action, 'postfix_postscreen_access') - * l.P' ' - * postfix_client_info_cg -local postfix_ps_noqueue_cg =postfix_smtpd_noqueue_cg -local postfix_ps_toobusy_cg = l.P'NOQUEUE: reject: CONNECT from ' - * postfix_client_info_cg - * l.P': ' - * l.Cg(l.P(1)^1, 'postfix_postscreen_toobusy_data') -local postfix_ps_dnsbl_cg = postfix_ps_violation_cg - * l.P' rank ' - * l.Cg(integer, 'postfix_postscreen_dnsbl_rank') - * l.P' for ' - * postfix_client_info_cg -local postfix_ps_cache_cg = l.P'cache ' - * (l.P(1)-l.P' ')^1 - * l.P' full cleanup: retained=' - * l.Cg(integer, 'postfix_postscreen_cache_retained') - * l.P' dropped=' - * l.Cg(integer, 'postfix_postscreen_cache_dropped') - * l.P' entries' -local postfix_ps_violations_cg = postfix_ps_violation_cg - * (l.P' ' * l.digit^1)^-1 - * (l.P' after ' * l.Cg(number, 'postfix_postscreen_violation_time'))^-1 - * l.P' from ' - * postfix_client_info_cg - * (l.P' after ' * postfix_smtp_stage_cg)^-1 - --- dnsblog patterns -local postfix_dnsblog_listing_cg = l.P'addr ' - * l.Cg(ipv46, 'postfix_client_ip') - * l.P' listed by domain ' - * l.Cg(host, 'postfix_dnsbl_domain') - * l.P' as ' - * l.Cg(ipv46, 'postfix_dnsbl_result') - --- tlsproxy patterns -local postfix_tlsproxy_conn_cg = (l.P'DISCONNECT' - + l.P'CONNECT') - * (l.P' from')^-1 - * l.P' ' - * postfix_client_info_cg - --- anvil patterns -local postfix_service_cg = (l.Cg((l.P(1)-l.S':')^1, 'postfix_service') - * l.P':' - * l.Cg(ipv46, 'postfix_client_ip')) - + (l.Cg((l.P(1)-l.S':')^1 * l.P':' * l.digit^1, 'postfix_service') - * l.P':' - * l.Cg(ipv46, 'postfix_client_ip')) - -local postfix_anvil_conn_rate_cg = l.P'statistics: max connection rate ' - * l.Cg(integer, 'postfix_anvil_conn_rate') - * l.P'/' - * l.Cg(postfix_time_unit, 'postfix_anvil_conn_period') - * l.P' for (' - * postfix_service_cg - * l.P') at ' - * l.Cg(l.P(1)^1, 'postfix_anvil_timestamp') -local postfix_anvil_conn_cache_cg = l.P'statistics: max cache size ' - * l.Cg(integer, 'postfix_anvil_cache_size') - * l.P' at ' - * l.Cg(l.P(1)^1, 'postfix_anvil_timestamp') -local postfix_anvil_conn_count_cg = l.P'statistics: max connection count ' - * l.Cg(integer, 'postfix_anvil_conn_count') - * l.P' for (' - * postfix_service_cg - * l.P') at ' - * l.Cg(l.P(1)^1, 'postfix_anvil_timestamp') - --- smtp patterns -local postfix_smtp_delivery_cg = postfix_queueid_cg - * l.P': ' - * l.Cg((l.P(1) - l.P' status=')^1, 'postfix_keyvalue_data') - * l.P' status=' - * l.Cg(l.alpha^1, 'postfix_status') - * (l.P' (' * l.Cg((l.P(1) - (l.P')' * -1))^1, 'postfix_smtp_response') * l.P')')^-1 -local postfix_smtp_connerr_cg = l.P'connect to ' - * postfix_relay_info_cg - * l.P': ' - * (l.P'Connection timed out' + l.P'No route to host' + l.P'Connection refused') -local postfix_smtp_lostconn_cg = postfix_queueid_cg - * l.P': ' - * postfix_lostconn - * l.P' with ' - * postfix_relay_info_cg - --- master patterns -local postfix_master_start_cg = (l.P'daemon started' + l.P'reload') - * l.P' -- version ' - * l.Cg((l.P(1)-l.P',')^1, 'postfix_version') - * l.P', configuration ' - * l.Cg(l.P(1)^1, 'postfix_config_path') -local postfix_master_exit_cg = l.P'terminating on signal ' - * l.Cg(integer, 'postfix_termination_signal') - --- bounce patterns -local postfix_bounce_notification_cg = postfix_queueid_cg - * l.P': ' - * l.P'sender ' - * (l.P'non-delivery' + l.P'delivery status' + l.P'delay') - * l.P' notification: ' - * l.Cg(postfix_queueid, 'postfix_bounce_queueid') - --- scache patterns -local postfix_scache_lookups_cg = l.P'statistics: ' - * (l.P'address' + l.P'domain') - * l.P' lookup hits=' - * l.Cg(integer, 'postfix_scache_hits') - * l.P' miss=' - * l.Cg(integer, 'postfix_scache_miss') - * l.P' success=' - * l.Cg(integer, 'postfix_scache_success') -local postfix_scache_simultaneous_cg = l.P'statistics: max simultaneous domains=' - * l.Cg(integer, 'postfix_scache_domains') - * l.P' addresses=' - * l.Cg(integer, 'postfix_scache_addresses') - * l.P' connection=' - * l.Cg(integer, 'postfix_scache_connection') -local postfix_scache_timestamp_cg = l.P'statistics: start interval ' - * l.Cg(l.P(1)^1, 'postfix_scache_timestamp') - --- aggregate all patterns -local postfix_patterns = { - smtpd = l.Ct(postfix_smtpd_connect_cg - + postfix_smtpd_disconnect_cg - + postfix_smtpd_lostconn_cg - + postfix_smtpd_noqueue_cg - + postfix_smtpd_pipelining_cg - + postfix_tlsconn_cg - + postfix_warning_cg - + postfix_smtpd_proxy_cg - + postfix_keyvalue_greedy_cg), - cleanup = l.Ct(postfix_cleanup_milter_cg - + postfix_warning_cg - + postfix_keyvalue_greedy_cg), - qmgr = l.Ct(postfix_qmgr_removed_cg - + postfix_qmgr_active_cg - + postfix_warning_cg), - pipe = l.Ct(postfix_pipe_delivered_cg - + postfix_pipe_forward_cg), - postscreen = l.Ct(postfix_ps_connect_cg - + postfix_ps_access_cg - + postfix_ps_noqueue_cg - + postfix_ps_toobusy_cg - + postfix_ps_cache_cg - + postfix_ps_dnsbl_cg - + postfix_ps_violations_cg - + postfix_warning_cg), - dnsblog = l.Ct(postfix_dnsblog_listing_cg), - anvil = l.Ct(postfix_anvil_conn_rate_cg - + postfix_anvil_conn_cache_cg - + postfix_anvil_conn_count_cg), - smtp = l.Ct(postfix_smtp_delivery_cg - + postfix_smtp_connerr_cg - + postfix_smtp_lostconn_cg - + postfix_tlsconn_cg - + postfix_warning_cg), - discard = l.Ct(postfix_queueid_cg - * l.P': ' - * l.Cg((l.P(1) - l.P' status=')^1, 'postfix_keyvalue_data') - * l.P' status=' - * l.Cg(l.alnum^1, 'postfix_status')), - -- lmtp = smtp, - pickup = l.Ct(postfix_queueid_cg - * l.P': uid=' - * l.Cg((l.P(1) - l.P' from=')^1, 'postfix_uid') - * l.P' from=<' - * l.Cg((l.P(1) - l.P'>')^1, 'postfix_from') - * (l.P'> orig_id=' - * l.Cg(l.P(1)^1, 'postfix_orig_id'))^-1 - ), - tlsproxy = l.Ct(postfix_tlsproxy_conn_cg), - master = l.Ct(postfix_master_start_cg - + postfix_master_exit_cg), - bounce = l.Ct(postfix_bounce_notification_cg), - sendmail = l.Ct(postfix_warning_cg), - postdrop = l.Ct(postfix_warning_cg), - scache = l.Ct(postfix_scache_lookups_cg - + postfix_scache_simultaneous_cg - + postfix_scache_timestamp_cg), - trivial_rewrite = l.Ct(postfix_warning_cg), - tlsmgr = l.Ct(postfix_warning_cg), - ['local'] = l.Ct(postfix_keyvalue_greedy_cg), -} -postfix_patterns['lmtp'] = postfix_patterns['smtp'] - ---local name = l.C(l.alpha^1) * space ---local value = l.C((l.P(1) - l.P',')^1) ---local sep = l.S(',;') * space ---local pair = l.Cg(name * '=' * space * value) * sep^-1 ---local queueid = lpeg.Cf(lpeg.Ct'' * l.Cg(lpeg.Cc'queueid' * l.C(l.xdigit ^ 1)) * l.P': ', rawset) ---local postfix_grammar = l.Cf(queueid * pair^0, rawset) - - - -local postfix_programname = (l.P(1) - l.P'/')^1 - * '/' - * l.Cg(l.P(1)^1) - -local integer_fields = { - 'postfix_anvil_cache_size', - 'postfix_anvil_conn_count', - 'postfix_anvil_conn_rate', - 'postfix_client_port', - 'postfix_nrcpt', - 'postfix_postscreen_cache_dropped', - 'postfix_postscreen_cache_retained', - 'postfix_postscreen_dnsbl_rank', - 'postfix_relay_port', - 'postfix_server_port', - 'postfix_size', - 'postfix_status_code', - 'postfix_termination_signal', - 'postfix_uid', -} -local float_fields = { - 'postfix_delay', - --'postfix_delay_before_qmgr', - --'postfix_delay_conn_setup', - --'postfix_delay_in_qmgr', - --'postfix_delay_transmission', - --'postfix_postscreen_violation_time', -} - - -function postfix_match(programname, message, extract_keyvalue_data) - local daemon_name = postfix_programname:match(programname) - local ret = nil - if not daemon_name then - return nil - end - if postfix_patterns[daemon_name] then - ret = postfix_patterns[daemon_name]:match(message) - -- below are fake patterns used by tests - elseif daemon_name == 'relay_info' then - ret = postfix_relay_info_ct:match(message) - elseif daemon_name == 'delays' then - ret = postfix_delays_ct:match(message) - elseif daemon_name == 'keyvalue_data' then - ret = kv_grammar:match(message) - end - -- extract postfix_keyvalue_data - if extract_keyvalue_data and ret and ret.postfix_keyvalue_data then - local kv = kv_grammar:match(ret.postfix_keyvalue_data) - if kv then - ret.postfix_keyvalue_data = nil - for k,v in pairs(kv) do - ret[string.format('postfix_%s', k)] = v - end - if ret.postfix_client then - local cl = postfix_client_info_ct:match(ret.postfix_client) - if cl then - ret.postfix_client = nil - for k2,v2 in pairs(cl) do - ret[k2] = v2 - end - end - end - if ret.postfix_relay then - local relay = postfix_relay_info_ct:match(ret.postfix_relay) - if relay then - ret.postfix_relay = nil - for k2,v2 in pairs(relay) do - ret[k2] = v2 - end - end - end - if ret.postfix_delays then - local delays = postfix_delays_ct:match(ret.postfix_delays) - if delays then - ret.postfix_delays = nil - for k2,v2 in pairs(delays) do - ret[k2] = v2 - end - end - end - end - end - if ret then - for i, v in ipairs(integer_fields) do - if ret[v] then - local value = ret[v] - if type(value) == 'table' then - value = value.value - end - ret[v] = { - value = tonumber(value), - value_type = 2, - } - end - end - for i, v in ipairs(float_fields) do - if ret[v] then - local value = ret[v] - if type(value) == 'table' then - value = value.value - end - ret[v] = tonumber(value) - end - end - end - return ret -end - -return M diff --git a/modules/lpeg/syslog.lua b/modules/lpeg/syslog.lua deleted file mode 100644 index df6afd1..0000000 --- a/modules/lpeg/syslog.lua +++ /dev/null @@ -1,281 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Syslog Module - -## Variables - -* `severity` - LPEG grammar to parse a syslog severity string and return the numeric value - -## Functions - -### build_rsyslog_grammar - -Constructs an LPEG grammar based on the rsyslog template configuration string. - -*Arguments* -- template (string) - http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html - -*Return* -- grammar (LPEG user data object) or an error is thrown ---]] - - --- Imports -local l = require "lpeg" -l.locale(l) -local math = require "math" -local string = require "string" -local dt = require "lpeg.date_time" -local ip = require "lpeg.ip_address" -local tonumber = tonumber -local error = error -local type = type -local ipairs = ipairs -local rawset = rawset - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local function unescape_param_value(param_value) - return string.gsub(param_value, '\\([]"\\])', '%1') -end - -local function prefix_param_name(param_name) - return '_' .. param_name -end - --- http://tools.ietf.org/html/rfc5424#page-8 -local octet = l.P(1) -local utf8_string = octet^0 -local sp = l.P" " -local printusascii = l.R"!~" -local nonzero_digit = l.R"19" -local digit = l.R"09" -local nilvalue = l.P"-" -local bom = l.P("\239\187\191") -local msg_any = octet^0 -local msg_utf8 = bom * utf8_string -local msg = msg_utf8 + msg_any -local hostname = nilvalue + printusascii^-255 -local sd_name = (printusascii - l.S'=]" ')^-32 -local param_name = sd_name / prefix_param_name -local esc = l.P'\\' -local param_value_esc = l.S'"\\]' -local param_value = ((octet - param_value_esc) + (esc * octet))^0 / unescape_param_value -local sd_id = l.Cg(l.Cc"id" * l.C(sd_name)) -local sd_param = l.Cg(param_name * '="' * param_value * '"') -local sd_params = l.Cf(l.Ct"" * sd_id * (sp * sd_param)^0, rawset) -local sd_element = l.P"[" * sd_params * "]" -local syslog_facility = digit^-3 / tonumber -local syslog_severity = digit / tonumber - -local function convert_pri(pri) - pri = tonumber(pri) - local facility = math.floor(pri/8) - local severity = pri % 8 - - return {facility = facility, severity = severity} -end -local pri = digit^-3 / convert_pri - --- https://github.com/rsyslog/rsyslog/blob/35ef2408dfec0e8abdebd33c74578f9bb3299f20/runtime/msg.c#L309 -local syslog_severity_text = ( - (l.P"debug" + "DEBUG") / "7" -+ (l.P"info" + "INFO") / "6" -+ (l.P"notice" + "NOTICE") / "5" -+ (l.P"warning" + "WARNING") / "4" -+ (l.P"warn" + "WARN") / "4" -+ (l.P"error" + "ERROR") / "3" -+ (l.P"err" + "ERR") / "3" -+ (l.P"crit" + "CRIT") / "2" -+ (l.P"alert" + "ALERT") / "1" -+ (l.P"emerg" + "EMERG") / "0" -+ (l.P"panic" + "PANIC") / "0" -) / tonumber - -local syslog_facility_text = ( - (l.P"kern" + "KERN") / "0" -+ (l.P"user" + "USER") / "1" -+ (l.P"mail" + "MAIL") / "2" -+ (l.P"daemon" + "DAEMON") / "3" -+ (l.P"auth" + "AUTH") / "4" -+ (l.P"security" + "SECURITY") / "4" -+ (l.P"syslog" + "SYSLOG") / "5" -+ (l.P"lpr" + "LPR") / "6" -+ (l.P"news" + "NEWS") / "7" -+ (l.P"uucp" + "UUCP") / "8" -+ (l.P"cron" + "CRON") / "9" -+ (l.P"authpriv" + "AUTHPRIV") / "10" -+ (l.P"ftp" + "FTP") / "11" -+ (l.P"ntp" + "NTP") / "12" -+ (l.P"audit" + "AUDIT") / "13" -+ (l.P"alert" + "ALERT") / "14" -+ (l.P"clock" + "CLOCK") / "15" -+ (l.P"local0" + "LOCAL0") / "16" -+ (l.P"local1" + "LOCAL1") / "17" -+ (l.P"local2" + "LOCAL2") / "18" -+ (l.P"local3" + "LOCAL3") / "19" -+ (l.P"local4" + "LOCAL4") / "20" -+ (l.P"local5" + "LOCAL5") / "21" -+ (l.P"local6" + "LOCAL6") / "22" -+ (l.P"local7" + "LOCAL7") / "23" -) / tonumber - -local time_formats = { -["date-rfc3164"] = dt.rfc3164_timestamp, -["date-rfc3164-buggyday"] = dt.rfc3164_timestamp, -["date-mysql"] = dt.mysql_timestamp, -["date-pgsql"] = dt.pgsql_timestamp, -["date-rfc3339"] = dt.rfc3339, -["date-unixtimestamp"] = digit^1, -["date-subseconds"] = digit^1 -} - -local function lookup_time_format(property) - local f - if property.options then - local format = string.lower(property.options) - f = time_formats[format] - end - if property.name == "timereported" then - property.name = "timestamp" - end - - if not f then - f = time_formats["date-rfc3164"] - end - - if format == "date-unixtimestamp" then - f = f / dt.seconds_to_ns - elseif format == "date-subseconds" then - f = f / tonumber - else - f = f / dt.time_to_ns - end - - return f -end - --- http://rsyslog-5-8-6-doc.neocities.org/property_replacer.html -local programname = (printusascii - l.S" :[")^0 -local rsyslog_properties = { - -- special case msg since the rsyslog template can break the rfc5424 msg rules - rawmsg = octet^0, - hostname = hostname, - source = hostname, - fromhost = hostname, - ["fromhost-ip"] = ip.v4 + ip.v6, - syslogtag = l.Ct(l.Cg(programname, "programname") * ("[" * l.Cg(l.digit^1 / tonumber, "pid") * "]")^-1 * l.P":"^-1), - programname = programname, - pri = pri, -- pri table with facility and severity keys - ["pri-text"] = syslog_facility_text * "." * syslog_severity_text * "<" * pri * ">", - iut = l.digit^1, - syslogfacility = syslog_facility, - ["syslogfacility-text"] = syslog_facility_text, - syslogseverity = syslog_severity, - ["syslogseverity-text"] = syslog_severity_text, - syslogpriority = syslog_severity, - ["syslogpriority-text"] = syslog_severity_text, - timegenerated = lookup_time_format, - timereported = lookup_time_format, - timestamp = lookup_time_format, - ["protocol-version"] = digit^1, - ["structured-data"] = (l.Ct"" * nilvalue) + sd_element, - ["app-name"] = nilvalue + printusascii^-48, - procid = nilvalue + printusascii^-128, - msgid = nilvalue + printusascii^-32, - inputname = printusascii^0, -- doesn't appear to generate output (don't use it) - ["$bom"] = bom, - ["$now"] = dt.rfc3339_full_date, - ["$year"] = dt.date_fullyear, - ["$month"] = dt.date_month, - ["$day"] = dt.date_mday, - ["$hour"] = dt.time_hour, - ["$hhour"] = l.P"0" * l.S"01", - ["$qhour"] = l.P"0" * l.R"03", - ["$minute"] = dt.time_minute -} - -local function space_grammar() - return l.space -end - -local last_literal = l.P"\n" -local function rsyslog_lookup(property) - if property.options and property.options == "sp-if-no-1st-sp" then - return sp^1 - end - - property.name = string.lower(property.name) - local g - if property.name == "msg" then - g = l.Cg((l.P(1) - last_literal)^0, property.name) -- todo account for an escaped literal - else - g = rsyslog_properties[property.name] - end - - if not g then - error(string.format("invalid rsyslog property: '%s'", property.name)) - end - - if type(g) == "function" then - g = g(property) - end - last_literal = l.P"\n" - return l.Cg(g, property.name) -end - -local escape_chars = {t = "\9", n = "\10", r = "\13", ['"'] = '"', ["'"] = "'" - , ["\\"] = "\\", ["%"] = "%", a = "\7", b = "\8", f = "\12", v = "\11"} -local function substitute_escape(seq) - seq = string.sub(seq, 2) - local e = escape_chars[seq] - if not e then - e = string.char(tonumber(seq)) - end - return e -end -local escape_sequence = l.Cs(((l.P"\\" * (l.S'tnr"\'\\%abfv' + digit^-3)) / substitute_escape + 1)^0) -local function literal_grammar(var) - local literal = l.match(escape_sequence, var) - last_literal = string.sub(literal, -1) - return l.P(literal) -end - --- --- Public Interface --- -function build_rsyslog_grammar(template) - local ws = l.space / space_grammar - local options = l.P":" * l.Cg((1 - l.P"%")^0, "options") -- todo support multiple options - local tochar = l.Cg((1 - l.S":%")^0, "tochar") - local fromchar = l.Cg((1 - l.P":")^0, "fromchar") - local substr = l.P":" * fromchar * ":" * tochar - local propname = l.Cg((l.alnum + l.S"-$")^1, "name") - local property = l.P"%" * l.Ct(propname * substr^-1 * options^-1) / rsyslog_lookup * l.P"%" - local literal = (l.P(1) - (ws + property))^1 / literal_grammar - local item = ws + property + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(template) - if not t then - error("could not parse the rsyslog template") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - -severity = syslog_severity_text - -return M diff --git a/modules/lpeg/syslog_message.lua b/modules/lpeg/syslog_message.lua deleted file mode 100644 index 6b61559..0000000 --- a/modules/lpeg/syslog_message.lua +++ /dev/null @@ -1,640 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Copyright 2015 Mathieu Parent - ---[[ -# Syslog Message Module - - -## Functions - -### get_prog_grammar - -Retrieves the parser for a particular program. - -*Arguments* -- prog (string) - program name e.g. "CRON", "dhclient", "dhcpd"... - -*Return* -- grammar (LPEG user data object) or nil if the `programname` isn't found - -### get_wildcard_grammar - -*Arguments* -- prog (string) - program name, currently only accepts "PAM" - -*Return* -- grammar (LPEG user data object) or nil if the `programname` isn't found ---]] - - -local string = require "string" -local ip = require "lpeg.ip_address" -local l = require "lpeg" -l.locale(l) -local tonumber = tonumber -local type = type - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local prog_grammar = {} -local wildcard_grammar = {} - --- LPEG helpers -local integer = (l.P"-"^-1 * l.digit^1) / tonumber -local float = (l.P"-"^-1 * l.digit^1 * (l.P"." * l.digit^1)^1) / tonumber -local ipv4 = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) -local ipv6 = l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation")) -local ipv46 = ipv4 + ipv6 - -local function capture_until(var, txt) - return l.Cg((l.P(1) - l.P(txt))^0, var) -end -local function capture_followed_by(var, txt) - return capture_until(var, txt) * l.P(txt) -end - --- programname=CRON -prog_grammar["CRON"] = l.Ct( - l.P"(" - * capture_followed_by("cron_username", ") ") - * capture_followed_by("cron_event", " (") - * capture_followed_by("cron_detail", ")") - * l.P(-1) - ) - --- programname=crontab -prog_grammar["crontab"] = prog_grammar["CRON"] - --- programname=dhclient -prog_grammar["dhclient"] = l.Ct( - ( -- "DHCPDISCOVER on %s to %s port %d interval %ld" - l.Cg(l.P"DHCPDISCOVER", "dhcp_type") - * l.P" on " - * capture_followed_by("dhcp_client_interface", " to ") - * l.Cg(ipv4, "dhcp_server_addr") - * l.P" port " - * l.Cg(l.digit^1 / tonumber, "dhcp_server_port") - * l.P" interval " - * l.Cg(l.digit^1 / tonumber, "dhcp_client_interval_seconds") - * l.P(-1) - ) + ( -- "DHCPREQUEST on %s to %s port %d" - -- "DHCPDECLINE on %s to %s port %d" - -- "DHCPRELEASE on %s to %s port %d" - l.Cg(l.P"DHCPREQUEST" + l.P"DHCPDECLINE" + l.P"DHCPRELEASE", "dhcp_type") - * l.P" on " - * capture_followed_by("dhcp_client_interface", " to ") - * l.Cg(ipv4, "dhcp_server_addr") - * l.P" port " - * l.Cg(l.digit^1 / tonumber, "dhcp_server_port") - * l.P(-1) - ) + ( -- "DHCPACK from %s" - l.Cg(l.P"DHCPACK", "dhcp_type") - * l.P" from " - * l.Cg(ipv4, "dhcp_server_addr") - * l.P(-1) - ) + ( -- "bound to %s -- renewal in %ld seconds." - l.P"bound to " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" -- renewal in " - * l.Cg(l.digit^1 / tonumber, "dhcp_client_renewal_seconds") - * l.P" seconds." - * l.P(-1) - )) - --- programname=dhcpd -local dhcpd_hw_addr = l.xdigit * l.xdigit * (l.S":" * l.xdigit * l.xdigit)^0 -prog_grammar["dhcpd"] = l.Ct( - ( - l.Cg(l.P"BOOTREQUEST", "dhcp_type") - * l.P" from " - * l.Cg(dhcpd_hw_addr, "dhcp_client_hw_addr") - * l.P" via " - * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") - * l.P(-1) - ) + ( - l.Cg(l.P"BOOTREPLY", "dhcp_type") - * l.P" for " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" to " - * capture_followed_by("dhcp_client_addr", " (") - * l.Cg(dhcpd_hw_addr, "dhcp_client_hw_addr") - * l.P") via " - * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") - * l.P(-1) - ) + ( - l.Cg(l.P"DHCPDISCOVER", "dhcp_type") - * l.P" from " - * l.Cg(dhcpd_hw_addr + l.P"", "dhcp_client_hw_addr") - * l.P" " - * (l.P"(" * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hostname") * l.P") ")^-1 - * l.P"via " - * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") - * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 - * l.P(-1) - ) + ( - l.Cg(l.P"DHCPOFFER" + l.P"DHCPACK" + l.P"BOOTREPLY", "dhcp_type") - * l.P" on " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" to " - * l.Cg((l.P(1)-l.P" ")^1, "dhcp_client_hw_addr") - * l.P" " - * (l.P"(" * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hostname") * l.P") ")^-1 - * l.P"via " - * l.Cg((l.P(1)-l.P" ")^1, "dhcp_source") - * ( l.P" [" - * l.Cg(integer, "dhcp_lease_time") - * l.P"]")^-1 - * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 - * l.P(-1) - ) + ( - l.Cg(l.P"DHCPACK", "dhcp_type") - * l.P" to " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" (" - * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hw_addr") - * l.P") via " - * l.Cg((l.P(1)-l.P" ")^1, "dhcp_source") - * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 - * ( l.P" [" - * l.Cg(integer, "dhcp_lease_time") - * l.P"]")^-1 - * l.P(-1) - ) + ( - l.Cg(l.P"DHCPNAK", "dhcp_type") - * l.P" on " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" to " - * l.Cg((l.P(1)-l.P" ")^1, "dhcp_client_hw_addr") - * l.P" via " - * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") - * l.P(-1) - ) + ( - l.Cg(l.P"DHCPREQUEST", "dhcp_type") - * l.P" for " - * l.Cg(ipv4, "dhcp_client_addr") - * (l.P" (" * l.Cg((l.P(1)-l.P")")^1, "dhcp_server_addr") * l.P")")^-1 - * l.P" from " - * l.Cg(dhcpd_hw_addr + l.P"", "dhcp_client_hw_addr") - * l.P" " - * (l.P"(" * l.Cg((l.P(1)-l.P")")^1, "dhcp_client_hostname") * l.P") ")^-1 - * l.P"via " - * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") - * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 - * l.P(-1) - ) + ( - l.Cg(l.P"DHCPINFORM", "dhcp_type") - * l.P" from " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" via " - * l.Cg((l.P(1)-l.P":")^1, "dhcp_source") - * (l.P": " * l.Cg(l.P(1)^1, "dhcp_message"))^-1 - * l.P(-1) - ) + ( - l.Cg(l.P"balancing", "dhcp_pool_result") - * l.P" pool" - * capture_followed_by("dhcp_pool_id", " ") - * capture_followed_by("dhcp_network", " total ") - * l.Cg(integer, "dhcp_lease_count") - * l.P" free " - * l.Cg(integer, "dhcp_free_leases") - * l.P" backup " - * l.Cg(integer, "dhcp_backup_leases") - * l.P" lts " - * l.Cg(integer, "dhcp_lts") - * l.P" max-own (+/-)" - * l.Cg(integer, "dhcp_hold") - * (l.P" (requesting peer rebalance!)")^-1 - * l.P(-1) - ) + ( - l.Cg(l.P"balanced" + l.P"IMBALANCED", "dhcp_pool_result") - * l.P" pool" - * capture_followed_by("dhcp_network", " total ") - * l.Cg(integer, "dhcp_lease_count") - * l.P" free " - * l.Cg(integer, "dhcp_free_leases") - * l.P" backup " - * l.Cg(integer, "dhcp_backup_leases") - * l.P" lts " - * l.Cg(integer, "dhcp_lts") - * l.P" max-misbal " - * l.Cg(integer, "dhcp_thresh") - * l.P(-1) - ) + ( - l.P"bind update on " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" from " - * capture_followed_by("dhcp_failover_peer", " ") - * l.Cg(l.P"rejected", "dhcp_bind_update_status") - * l.P": " - * l.Cg(l.P(1)^1, "dhcp_message") - ) + ( - l.P"bind update on " - * l.Cg(ipv4, "dhcp_client_addr") - * l.P" " - * l.Cg(l.P"got ack", "dhcp_bind_update_status") - * l.P" from " - * capture_followed_by("dhcp_failover_peer", ": ") - * l.Cg(l.P(1)^1, "dhcp_message") - )) - --- programname=login -prog_grammar["login"] = l.Ct( - ( - l.P"FAILED LOGIN (" - * l.Cg(l.digit^1 / tonumber, "failcount") - * l.P")" - * l.P" on '" - * capture_followed_by("tty", "'") - * (l.P" from '" * capture_followed_by("from", "'"))^-1 - * l.P" FOR '" - * capture_followed_by("user", "', ") - * l.Cg(l.P(1)^1, "pam_error") - ) + ( - l.P"ROOT LOGIN " - * l.P" on '" - * capture_followed_by("tty", "'") - * (l.P" from '" * capture_followed_by("from", "'"))^-1 - * l.P(-1) - )) - --- programname=groupadd -prog_grammar["groupadd"] = l.Ct( - ( - l.P"new group: name=" - * capture_followed_by("group_name", ", GID=") - * l.Cg(l.digit^1 / tonumber, "gid") - * l.P(-1) - )) - --- programname=groupdel -prog_grammar["groupdel"] = l.Ct( - ( - l.P"group '" - * capture_followed_by("group_name", "' removed") - * (l.P" from " * l.Cg(l.P(1)^1, "group_dbname"))^-1 - )) - --- programname=named -prog_grammar["named"] = l.Ct( - ( -- "lame server resolving "%s" (in "%s"?): %s" - l.Cg(l.P"lame server", "dns_error") - * l.P" resolving '" - * capture_followed_by("dns_name", "' (in '") - * capture_followed_by("dns_domain", "'?): ") - * l.Cg(ipv46, "dns_addr") - * l.P"#" - * l.Cg(l.digit^1 / tonumber, "dns_port") - * l.P(-1) - ) + ( -- "error (%s%s%s) resolving "%s/%s/%s": %s" before 1d761cb453c76353deb8423c78e98d00c5f86ffa - l.P"error (" - * capture_followed_by("dns_error", ") resolving '") - * capture_followed_by("dns_name", "/") - * capture_followed_by("dns_type", "/") - * capture_followed_by("dns_class", "': ") - * l.Cg(ipv46, "dns_addr") - * l.P"#" - * l.Cg(l.digit^1 / tonumber, "dns_port") - * l.P(-1) - ) + ( -- "%s%s%s resolving "%s/%s/%s": %s" after 1d761cb453c76353deb8423c78e98d00c5f86ffa - capture_followed_by("dns_error", " resolving '") - * capture_followed_by("dns_name", "/") - * capture_followed_by("dns_type", "/") - * capture_followed_by("dns_class", "': ") - * l.Cg(ipv46, "dns_addr") - * l.P"#" - * l.Cg(l.digit^1 / tonumber, "dns_port") - * l.P(-1) - ) + ( -- "DNS format error from %s resolving %s%s%s: %s" - l.Cg(l.P"DNS format error", "dns_error") - * l.P" from " - * l.Cg(ipv46, "dns_addr") - * l.P"#" - * l.Cg(l.digit^1 / tonumber, "dns_port") - * l.P" resolving " - * capture_followed_by("dns_name", "/") - * capture_until("dns_type", l.P" for client " + l.P":") - * (l.P" for client " - * l.Cg(ipv46, "dns_client_addr") - * l.P"#" - * l.Cg(l.digit^1 / tonumber, "dns_client_port") - )^-1 - * l.P": " - * l.Cg(l.P(1)^1, "dns_message") - ) + ( -- "skipping nameserver '%s' because it is a CNAME, while resolving '%s'" - l.Cg(l.P"skipping nameserver", "dns_error") - * l.P" '" - * capture_followed_by("dns_nameserver", "' because it is a CNAME, while resolving '") - * capture_followed_by("dns_name", "'") - * l.P(-1) - ) + ( -- "client %s%s%s%s%s%s%s%s: %s" - l.P"client " - * l.Cg(ipv46, "dns_client_addr") - * l.P"#" - * l.Cg(l.digit^1 / tonumber, "dns_client_port") - * (l.P"/key " * l.Cg((l.P(1)-l.S" :")^1, "dns_client_signer"))^-1 - * (l.P" (" * l.Cg((l.P(1)-l.P")")^1, "dns_name") * l.P")")^-1 - * (l.P": view " * l.Cg((l.P(1)-l.P": ")^1, "dns_view"))^-1 - * l.P": " - * l.Cg(l.P(1)^1, "dns_message") - ) + ( -- "success resolving "%s" (in "%s"?) after %s" - l.P"success resolving '" - * capture_followed_by("dns_name", "/") - * capture_followed_by("dns_type", "' (in '") - * capture_followed_by("dns_domain", "'?) after ") - * l.Cg(l.P(1)^1, "dns_message") - ) + ( -- "sending notifies (serial %u)" - l.P"zone " - * capture_followed_by("dns_domain", "/") - * capture_followed_by("dns_class", "/") - * capture_followed_by("dns_view", ": ") - * l.Cg("sending notifies", "dns_message") - * l.P" (serial " - * l.Cg(l.digit^1 / tonumber, "dns_serial") - * l.P")" - * l.P(-1) - ) + ( -- "clients-per-query decreased to %u" - l.P"clients-per-query decreased to " - * l.Cg(l.digit^1 / tonumber, "dns_clients_per_query") - * l.P(-1) - )) - --- programname=puppet-agent --- see http://docs.puppetlabs.com/puppet/latest/reference/lang_reserved.html#classes-and-defined-types -local puppet_namespace_segment = l.upper - * (l.lower + l.digit + l.P"_")^0 -local puppet_type = -- example: Mod::Config - puppet_namespace_segment - * (l.P"::" * puppet_namespace_segment)^0 -local puppet_resource = ( -- example: Mod::Config[foo] - puppet_type - * l.P"[" - * (l.P(1)-l.P"]")^1 - * l.P"]" - ) -local puppet_resource_path = ( -- example: /Stage[main]/Profile_one/Mod::Config[foo] - (l.P"/" * (puppet_resource + puppet_type))^1 - ) ---http://docs.puppetlabs.com/puppet/latest/reference/lang_reserved.html#parameters -local puppet_parameter = ( -- example: /Stage[main]/Mod::Config[foo]/ensure - (l.lower + l.digit + l.P"_")^1 - ) -local puppet_resource_message_cg = (-- "Triggered "#{callback}" from #{events.length} events" - -- "Would have triggered "#{callback}" from #{events.length} events" - l.Cg((l.P"Would have triggered" * l.Cg(l.Cc(true), "puppet_noop")) + l.P"Triggered", "puppet_msg") - * l.P" '" - * capture_followed_by("puppet_callback","' from ") - * l.Cg(integer, "puppet_events_count") - * l.P" events" - * l.P(-1) - ) + ( - l.P"Scheduling " - * capture_followed_by("puppet_callback"," of ") -- most probably "refresh" - * l.Cg(puppet_resource, "puppet_callback_target") - * l.P(-1) - ) + ( - l.P"Unscheduling " - * capture_followed_by("puppet_callback"," on ") -- most probably "refresh" - * l.Cg(puppet_resource, "puppet_callback_target") - * l.P(-1) - ) + ( - l.P"Filebucketed " - * capture_followed_by("puppet_file_path"," to ") - * capture_followed_by("puppet_bucket"," with sum ") - * capture_followed_by("puppet_file_sum",l.P(-1)) - ) -local puppet_parameter_message_cg = ( - l.P"current_value " - * capture_followed_by("puppet_current_value", ", should be ") - * capture_followed_by("puppet_should_value", " (noop)") - * (l.P" (previously recorded value was " *l.Cg(l.P(1)^1, "puppet_historical_value"))^-1 - * l.Cg(l.Cc(true), "puppet_noop") - * l.P(-1) - ) + ( - l.Cg(puppet_parameter, "puppet_ensure_parameter") - * l.P" changed '" - * capture_followed_by("puppet_old_value", "' to '") - * capture_followed_by("puppet_new_value", "'" * l.P(-1)) - ) + ( - l.Cg(l.P"executed successfully", "puppet_change") - * l.P(-1) - ) -prog_grammar["puppet-agent"] = l.Ct( - ( - l.P"(" - * l.Cg(puppet_resource_path, "puppet_resource_path") - * l.P"/" - * l.Cg(puppet_parameter, "puppet_parameter") - * l.P")" - * ( - (l.P" " * puppet_parameter_message_cg) - + l.P(1)^0 -- parameter can send arbitrary message - ) - ) + ( - l.P"(" - * capture_followed_by("puppet_resource_path", ") ") - * puppet_resource_message_cg - ) + (-- msg + (" in %0.2f seconds" % seconds) - l.Cg(l.P"Finished catalog run", "puppet_msg") - * l.P" in " - * l.Cg(float, "puppet_benchmark_seconds") - * l.P" seconds" - * l.P(-1) - ) + ( -- Keep as is - (l.P"Retrieving pluginfacts" * l.P(-1)) - + (l.P"Retrieving plugin" * l.P(-1)) - + (l.P"Loading facts" * l.P(-1)) - + (l.P"Caching catalog for ") - + (l.P"Applying configuration version '") - + (l.P"Computing checksum on file ") - + (l.P"Run of Puppet configuration client already in progress; skipping (") -- /var/lib/puppet/state/agent_catalog_run.lock exists) - )) --- programname=sshd -prog_grammar["sshd"] = l.Ct( - ( - l.Cg(l.P"Accepted" + l.P"Failed" + l.P"Partial" + l.P"Postponed", "sshd_authmsg") - * l.P" " - * l.Cg((l.P(1)-l.S"/ ")^1, "sshd_method") - * (l.P"/" * l.Cg((l.P(1)-l.S"/ ")^1, "sshd_submethod"))^-1 - * l.P" for " - * l.P"invalid user "^-1 - * capture_followed_by("remote_user", " from ") - * l.Cg(ipv46, "remote_addr") - * l.P" port " - * l.Cg(l.digit^1 / tonumber, "remote_port") - * l.P" " - * (l.P"ssh2" + l.P"ssh1") - * (l.P": " * l.Cg(l.P(1)^1, "sshd_info"))^-1 - ) + ( - l.P"Received disconnect from " - * l.Cg(ipv46, "remote_addr") - * l.P": " - * (l.Cg(l.digit^1 / tonumber, "disconnect_reason") * l.P": ")^-1 - * l.Cg(l.P(1)^1, "disconnect_msg") - ) + ( - l.P"reverse mapping checking getaddrinfo for " - * capture_followed_by("remote_host", "[") - * l.Cg(ipv46, "remote_addr") - * l.P"] failed - POSSIBLE BREAK-IN ATTEMPT!" - * l.P(-1) - ) + ( - l.P"subsystem request for " - * capture_followed_by("sshd_subsystem", " by user " + l.P(-1)) - * l.Cg((l.P(1)-l.S" ")^0, "remote_user") - * l.P(-1) - ) + ( - l.P"Connection closed by " - * l.Cg(ipv46, "remote_addr") - * l.P" [preauth]" - * l.P(-1) - ) + ( - l.P"Invalid user " - * capture_followed_by("remote_user", " from ") - * l.Cg(ipv46, "remote_addr") - * l.P(-1) - ) + ( - l.P"input_userauth_request: invalid user " - * capture_followed_by("remote_user", " [preauth]") - * l.P(-1) - ) + ( - l.P"Exiting on signal " - * l.Cg(l.digit^1, "signal") - * l.P(-1) - ) + ( - l.P"Received signal " - * l.Cg(l.digit^1, "signal") - * l.P"; terminating." - * l.P(-1) - ) + ( - l.P"Server listening on " - * l.Cg((l.P(1)-l.S" ")^1, "listen_address") - * l.P" port " - * l.Cg(l.digit^1, "listen_port") - * l.P"." - * l.P(-1) - ) + ( - l.P"Did not receive identification string from " - * l.Cg(ipv46, "remote_addr") - * l.P(-1) - ) + ( - l.Cg(l.P"error" + l.P"fatal", "sshd_errorlevel") - * l.P": " - * l.Cg(l.P(1)^1, "sshd_error") - )) - --- programname=su -prog_grammar["su"] = l.Ct( - ( - l.Cg(l.P"Successful" + l.P"FAILED", "su_status") - * l.P" su for " - * capture_followed_by("su_name", " by ") - * l.Cg(l.P(1)^1, "su_oldname") - ) + ( - l.P"pam_authenticate: " - * l.Cg(l.P(1)^1, "pam_error") - ) + ( - l.S"+-" - * l.P" " -- FIXME what to capture? - )) - --- programname=sudo -local function sudo_field(name) - return (l.P(name) * l.P"=" * capture_followed_by("sudo_" .. string.lower(name), " ; "))^-1 -end -prog_grammar["sudo"] = l.Ct( - capture_followed_by("sudo_message", l.P" : ") - * sudo_field("TTY") - * sudo_field("PWD") - * sudo_field("USER") - * sudo_field("GROUP") - * sudo_field("TSID") - * sudo_field("ENV") - * l.P"COMMAND=" * l.Cg(l.P(1)^1, "sudo_command") - ) - --- programname=systemd-logind -prog_grammar["systemd-logind"] = l.Ct( - ( - l.P"New session " - * l.Cg(integer, "session_id") - * l.P" of user " - * capture_followed_by("user_id", "." * l.P(-1)) - * l.Cg(l.Cc("SESSION_START"), "sd_message") - ) + ( - l.P"Removed session " - * l.Cg(integer, "session_id") - * l.P"." - * l.Cg(l.Cc("SESSION_STOP"), "sd_message") - * l.P(-1) - )) - --- programname=useradd -prog_grammar["useradd"] = l.Ct( - ( - l.P"new user: name=" - * capture_followed_by("user_name", ", UID=") - * l.Cg(l.digit^1 / tonumber, "uid") - * l.P", GID=" - * l.Cg(l.digit^1 / tonumber, "gid") - * l.P", home=" - * capture_followed_by("user_home", ", shell=") - * l.Cg(l.P(1)^1, "user_shell") - ) + ( - l.P"add '" - * capture_followed_by("user_name", l.P"' to group '" + l.P"' to shadow group '") - * capture_followed_by("group_name", "'") - * l.P(-1) - )) - --- PAM -local pam_header = capture_followed_by("pam_module", "(") - * capture_followed_by("pam_service", ":") - * capture_followed_by("pam_type", "): ") -wildcard_grammar["PAM"] = l.Ct( - ( pam_header - * l.Cg(l.P"session opened", "pam_action") - * l.P" for user " - * capture_followed_by("user_name", " by ") - * capture_followed_by("login_name", "(uid=") - * l.Cg(l.digit^1 / tonumber, "uid") - * l.P")" - ) + ( - pam_header - * l.Cg(l.P"session closed", "pam_action") - * l.P" for user " - * l.Cg(l.P(1)^1, "user_name") - ) + ( - pam_header - * l.Cg(l.P"authentication failure", "pam_action") - * l.P"; logname=" - * capture_followed_by("logname", " uid=") - * l.Cg(l.digit^1 / tonumber, "uid") - * l.P" euid=" - * l.Cg(l.digit^1 / tonumber, "euid") - * l.P" tty=" - * capture_followed_by("tty", " ruser=") - * capture_followed_by("ruser", " rhost=") - * l.Cg((l.P(1) - l.P(" user="))^0, "rhost") - * l.P" " -- duplicate space - * (l.P" user=" * l.Cg(l.P(1)^1, "user"))^-1 - ) + ( - pam_header - * l.P"check pass; user " - * (l.P"(" * capture_followed_by("user_name", ") "))^-1 - * l.P"unknown" - )) - - -function get_prog_grammar(prog) - return prog_grammar[prog] -end - - -function get_wildcard_grammar(prog) - return wildcard_grammar[prog] -end - -return M diff --git a/modules/lsb/util.lua b/modules/lsb/util.lua deleted file mode 100644 index 953018f..0000000 --- a/modules/lsb/util.lua +++ /dev/null @@ -1,77 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Lua Sandbox Utility Module - -## Functions - -### behead_array - -Effectively removes all array values up to the provided index from an array -by copying end values to the array head and setting now unused entries at -the end of the array to `nil`. - -*Arguments* -- index (number) - remove the values in the array up to the index value -- array (table) - array to behead - -*Return* -- none - in-place operation - -### pairs_by_key - -Sorts the keys into an array, and then iterates on the array. - -*Arguments* -- hash (table) - hash table to iterate in sorted key order -- sort (function) - function to specify an alternate sort order - -*Return* -- function - iterator that traverses the table keys in sort function order ---]] - --- Imports -local pairs = pairs -local table = require "table" - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -function behead_array(idx, array) - if idx <= 1 then return end - local array_len = #array - local start_nil_idx = 1 -- If idx > #array we zero it out completely. - if idx <= array_len then - -- Copy values to lower indexes. - local difference = idx - 1 - for i = idx, array_len do - array[i-difference] = array[i] - end - start_nil_idx = array_len - difference + 1 - end - -- Empty out the end of the array. - for i = start_nil_idx, array_len do - array[i] = nil - end -end - --- http://www.lua.org/pil/19.3.html -function pairs_by_key(t, f) - local a = {} - for n in pairs(t) do table.insert(a, n) end - table.sort(a, f) - local i = 0 -- iterator variable - local iter = function () -- iterator function - i = i + 1 - if a[i] == nil then - return nil - else - return a[i], t[a[i]] - end - end - return iter -end - -return M diff --git a/sandboxes/heka/analysis/throughput.lua b/sandboxes/heka/analysis/throughput.lua deleted file mode 100644 index 6f05478..0000000 --- a/sandboxes/heka/analysis/throughput.lua +++ /dev/null @@ -1,48 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Throughput Analysis - -Simple message throughput counter/visualization. - -## Sample Configuration -```lua -filename = "throughput.lua" -message_matcher = "TRUE" -ticker_interval = 60 -preserve_data = true -preservation_version = 0 - --- rows (integer) - Number of aggregate count buckets --- rows = 10080 -- one week - --- sec_per_row (integer) - Number of seconds each aggregate bucket represents --- sec_per_row = 60 - --- preservation_version (integer) - This must be incremented if the rows or --- sec_per_row value is change when using preservation (the old data is cleared) --- preservation_version = 0 -``` ---]] -_PRESERVATION_VERSION = read_config("preservation_version") or 0 - -require "circular_buffer" -require "os" -local time = os.time - -local rows = read_config("rows") or 10080 -local sec_per_row = read_config("sec_per_row") or 60 - -local cb = circular_buffer.new(rows, 1, sec_per_row) -cb:set_header(1, "messages") - -function process_message() - cb:add(time() * 1e9, 1, 1) - return 0 -end - -function timer_event(ns) - inject_payload("cbuf", "counts", cb) -end diff --git a/sandboxes/heka/input/heka_kafka.lua b/sandboxes/heka/input/heka_kafka.lua deleted file mode 100644 index 2349a99..0000000 --- a/sandboxes/heka/input/heka_kafka.lua +++ /dev/null @@ -1,60 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "kafka" - ---[[ -# Heka Kafka Consumer Input - -## Sample Configuration -```lua -filename = "heka_kafka.lua" -output_limit = 8 * 1024 * 1024 -brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 - --- in balanced consumer group mode a consumer can only subscribe on topics, not topics:partitions. --- The partition syntax is only used for manual assignments (without balanced consumer groups). -topics = {"test"} -ticker_interval = 60 - --- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#global-configuration-properties -consumer_conf = { - ["group.id"] = "test_group", -- must always be provided (a single consumer is considered a group of one - -- in that case make this a unique identifier) - ["message.max.bytes"] = output_limit, -} - --- https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md#topic-configuration-properties -topic_conf = { - -- ["auto.commit.enable"] = true, -- cannot be overridden - -- ["offset.store.method"] = "broker, -- cannot be overridden -} -``` ---]] -local brokerlist = read_config("brokerlist") or error("brokerlist must be set") -local topics = read_config("topics") or error("topics must be set") -local consumer_conf = read_config("consumer_conf") -local topic_conf = read_config("topic_conf") - -local consumer = kafka.consumer(brokerlist, topics, consumer_conf, topic_conf) - -local err_msg = { - Logger = read_config("Logger"), - Type = "error", - Payload = nil, -} - -function process_message() - while true do - local msg, topic, partition, key = consumer:receive() - if msg then - local ok, err = pcall(inject_message, msg) - if not ok then - err_msg.Payload = err - pcall(inject_message, err_msg) - end - end - end - return 0 -- unreachable but here for consistency -end diff --git a/sandboxes/heka/input/heka_stdin.lua b/sandboxes/heka/input/heka_stdin.lua deleted file mode 100644 index 747a6d2..0000000 --- a/sandboxes/heka/input/heka_stdin.lua +++ /dev/null @@ -1,32 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Heka Protobuf Stream Input (stdin) - -## Sample Configuration -```lua -filename = "heka_stdin.lua" -``` ---]] - -local stdin = require "io".stdin -require "string" - -local hsr = create_stream_reader(read_config("Logger")) - -function process_message() - local cnt = 0 - local found, consumed, read - repeat - repeat - found, consumed, read = hsr:find_message(stdin) - if found then - inject_message(hsr) - cnt = cnt + 1 - end - until not found - until read == 0 - return 0, string.format("processed %d messages", cnt) -end diff --git a/sandboxes/heka/input/heka_tcp.lua b/sandboxes/heka/input/heka_tcp.lua deleted file mode 100644 index 9947503..0000000 --- a/sandboxes/heka/input/heka_tcp.lua +++ /dev/null @@ -1,107 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Heka Compatible TCP Input - -## Sample Configuration -```lua -filename = "heka_tcp.lua" -instruction_limit = 0 - -address = "127.0.0.1" -port = 5565 - -ssl_params = { - mode = "server", - protocol = "tlsv1", - key = "/etc/hindsight/certs/serverkey.pem", - certificate = "/etc/hindsight/certs/server.pem", - cafile = "/etc/hindsight/certs/CA.pem", - verify = {"peer", "fail_if_no_peer_cert"}, - options = {"all", "no_sslv3"} -} -``` ---]] - -require "coroutine" -local socket = require "socket" -require "string" -require "table" - -local address = read_config("address") or "127.0.0.1" -local port = read_config("port") or 5565 -local ssl_params = read_config("ssl_params") -local ssl_ctx = nil -if ssl_params then - require "ssl" - ssl_ctx = assert(ssl.newcontext(ssl_params)) -end -local server = assert(socket.bind(address, port)) -server:settimeout(0) -local threads = {} -local sockets = {server} - -local function handle_client(client, caddr, cport) - local found, consumed, need = false, 0, 8192 * 4 - local hsr = create_stream_reader(string.format("%s:%d -> %s:%d", caddr, cport, address, port)) - client:settimeout(0) - while client do - local buf, err, partial = client:receive(need) - if partial then buf = partial end - if not buf then break end - - repeat - found, consumed, need = hsr:find_message(buf) - if found then inject_message(hsr) end - buf = nil - until not found - - if err == "closed" then break end - - coroutine.yield() - end -end - -function process_message() - while true do - local ready = socket.select(sockets, nil, 1) - if ready then - for _, s in ipairs(ready) do - if s == server then - local client = s:accept() - if client then - local caddr, cport = client:getpeername() - if not caddr then - caddr = "unknown" - cport = 0 - end - if ssl_ctx then - client = ssl.wrap(client, ssl_ctx) - client:dohandshake() - end - sockets[#sockets + 1] = client - threads[client] = coroutine.create( - function() handle_client(client, caddr, cport) end) - end - else - if threads[s] then - local status = coroutine.resume(threads[s]) - if not status then - s:close() - for i = #sockets, 2, -1 do - if s == sockets[i] then - table.remove(sockets, i) - break - end - end - threads[s] = nil - end - end - end - end - end - end - return 0 -end diff --git a/sandboxes/heka/input/syslog_udp.lua b/sandboxes/heka/input/syslog_udp.lua deleted file mode 100644 index bd4f139..0000000 --- a/sandboxes/heka/input/syslog_udp.lua +++ /dev/null @@ -1,99 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Syslog UDP Input - -## Sample Configuration -```lua -filename = "syslog_udp.lua" -instruction_limit = 0 - --- address (string) - IP address or * for all interfaces --- address = "127.0.0.1" - --- port (integer) - IP port to listen on --- port = 514 - --- template (string) - The 'template' configuration string from rsyslog.conf --- see http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html --- template = "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" -- RSYSLOG_TraditionalForwardFormat -``` ---]] - -local syslog = require "lpeg.syslog" -local socket = require "socket" - -local address = read_config("address") or "127.0.0.1" -local port = read_config("port") or 514 -local hostname_keep = read_config("hostname_keep") -local template = read_config("template") or "<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%" -local grammar = syslog.build_rsyslog_grammar(template) - -local msg = { -Timestamp = nil, -Type = read_config("type"), -Hostname = nil, -Payload = nil, -Pid = nil, -Severity = nil, -Fields = nil -} - -local err_msg = { - Logger = read_config("Logger"), - Type = "error", - Payload = nil, -} - -local server = assert(socket.udp()) -assert(server:setsockname(address, port)) -server:settimeout(1) - -function process_message() - while true do - local data, ip, port = server:receivefrom() - if data then - local fields = grammar:match(data) - if fields then - if fields.pri then - msg.Severity = fields.pri.severity - fields.syslogfacility = fields.pri.facility - fields.pri = nil - else - msg.Severity = fields.syslogseverity or fields["syslogseverity-text"] - or fields.syslogpriority or fields["syslogpriority-text"] - - fields.syslogseverity = nil - fields["syslogseverity-text"] = nil - fields.syslogpriority = nil - fields["syslogpriority-text"] = nil - end - - if fields.syslogtag then - fields.programname = fields.syslogtag.programname - msg.Pid = fields.syslogtag.pid - fields.syslogtag = nil - end - - msg.Hostname = fields.hostname or fields.source - fields.hostname = nil - fields.source = nil - - msg.Payload = fields.msg - fields.msg = nil - - fields.sender_ip = ip - fields.sender_port = {value = port, value_type = 2} - - msg.Fields = fields - pcall(inject_message, msg) - end - elseif ip ~= "timeout" then - err_msg.Payload = ip - pcall(inject_message, err_msg) - end - end - return 0 -end diff --git a/sandboxes/heka/output/debug.lua b/sandboxes/heka/output/debug.lua deleted file mode 100644 index 37652c4..0000000 --- a/sandboxes/heka/output/debug.lua +++ /dev/null @@ -1,57 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# RST Heka Message Output - -Writes a user friendly version (RST format) of the full Heka message to stdout - -## Sample Configuration -```lua -filename = "debug.lua" -message_matcher = "TRUE" -``` ---]] - -local write = require "io".write -local flush = require "io".flush -local concat = require "table".concat -local mi = require "heka.msg_interpolate" - -function process_message() - local raw = read_message("raw") - local msg = decode_message(raw) - write(":Uuid: ", mi.get_uuid(msg.Uuid), "\n") - write(":Timestamp: ", mi.get_timestamp(msg.Timestamp), "\n") - write(":Type: ", msg.Type or "", "\n") - write(":Logger: ", msg.Logger or "", "\n") - write(":Severity: ", msg.Severity or 7, "\n") - write(":Payload: ", msg.Payload or "", "\n") - write(":EnvVersion: ", msg.EnvVersion or "", "\n") - write(":Pid: ", msg.Pid or "", "\n") - write(":Hostname: ", msg.Hostname or "", "\n") - write(":Fields:\n") - for i, v in ipairs(msg.Fields or {}) do - write(" | name: ", v.name, - " type: ", v.value_type or 0, - " representation: ", v.representation or "", - " value: ") - if v.value_type == 4 then - for j, w in ipairs(v.value) do - if j ~= 1 then write(",") end - if w then write("true") else write("false") end - end - write("\n") - else - write(concat(v.value, ","), "\n") - end - end - write("\n") - flush() - return 0 -end - -function timer_event(ns) - -- no op -end diff --git a/sandboxes/heka/output/elasticsearch_bulk_api.lua b/sandboxes/heka/output/elasticsearch_bulk_api.lua deleted file mode 100644 index 74a8fcc..0000000 --- a/sandboxes/heka/output/elasticsearch_bulk_api.lua +++ /dev/null @@ -1,178 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Elasticsearch Bulk API Output - -## Sample Configuration -```lua -filename = "elasticsearch_bulk_api.lua" -message_matcher = "Type == 'nginx'" -ticker_interval = 10 -- flush every 10 seconds or flush_count (50000) messages -memory_limit = 200e6 - -address = "127.0.0.1" -port = 9200 -timeout = 10 -flush_count = 50000 -flush_on_shutdown = false -preserve_data = not flush_on_shutdown --in most cases this should be the inverse of flush_on_shutdown - --- See the elasticsearch module directory for the various encoders and configuration documentation. -encoder_module = "heka.elasticsearch.moz_telemetry" -encoder_cfg = { - es_index_from_timestamp = true, - index = "%{Logger}-%{%Y.%m.%d}", - type_name = "%{Type}-%{Hostname}", - fields = {"Fields[request]", "Fields[http_user_agent]"}, -} -``` ---]] - -require "table" -require "rjson" -require "string" -local ltn12 = require "ltn12" -local time = require "os".time -local em = require(read_config("encoder_module")) -local socket = require "socket" -local http = require("socket.http") -local address = read_config("address") or "127.0.0.1" -local port = read_config("port") or 9200 -local timeout = read_config("timeout") or 10 - -local batch_file = string.format("%s/%s.batch", read_config("output_path"), read_config("Logger")) -local flush_on_shutdown = read_config("flush_on_shutdown") -local ticker_interval = read_config("ticker_interval") -local flush_count = read_config("flush_count") or 50000 -local last_flush = time() - -local client -local function create_client() - local client = http.open(address, port) - client.c:setoption("tcp-nodelay", true) - client.c:setoption("keepalive", true) - client.c:settimeout(timeout) - return client -end -local pcreate_client = socket.protect(create_client); - - -local req_headers = { - ["user-agent"] = http.USERAGENT, - ["content-length"] = 0, - ["host"] = address .. ":" .. port, - ["accept"] = "application/json", - ["connection"] = "keep-alive", -} - -local function send_request() -- hand coded since socket.http doesn't support keep-alive connections - local fh = assert(io.open(batch_file, "r")) - req_headers["content-length"] = fh:seek("end") - client:sendrequestline("POST", "/_bulk") - client:sendheaders(req_headers) - fh:seek("set") - client:sendbody(req_headers, ltn12.source.file(fh, "invalid file handle")) - local code = client:receivestatusline() - local headers - while code == 100 do -- ignore any 100-continue messages - headers = client:receiveheaders() - code = client:receivestatusline() - end - headers = client:receiveheaders() - local ok, err, ret = true, nil, 0 - if code ~= 204 and code ~= 304 and not (code >= 100 and code < 200) then - if code == 200 and string.match(headers["content-type"], "^application/json") then - local body = {} - local sink = ltn12.sink.table(body) - client:receivebody(headers, sink) - local response = table.concat(body) - local ok, doc = pcall(rjson.parse, response) - if ok then - if doc:value(doc:find("errors")) then - ret = -1 - err = string.format("ElasticSearch server reported errors processing the submission") - end - else - ret = -3 - err = string.format("HTTP response didn't contain valid JSON. err: %s", doc) - end - else - client:receivebody(headers, ltn12.sink.null()) - end - - if not err and code > 304 then - ret = -1 - err = string.format("HTTP response error. Status: %d", code) - end - end - - if headers.connection == "close" then - client:close() - client = nil - end - - return true, err, ret -end -local psend_request = socket.protect(function(client) return send_request(client) end) - - -local batch = assert(io.open(batch_file, "a+")) -local function send_batch() - local err - if not client then - client, err = pcreate_client() - end - if err then return -3, err end -- retry indefinitely - - batch:flush() - local ok, err, ret = psend_request(client) - if not ok then -- network error - client = nil - return -3, err - end - last_flush = time() - return ret, err -end - -batch_count = 0 -retry = false - -function process_message() - if not retry then - batch_count = batch_count + 1 - batch:write(em.encode()) - end - - if batch_count == flush_count then - local ret, err = send_batch() - if ret == 0 then - retry = false - batch_count = 0 - batch:close() - batch = assert(io.open(batch_file, "w")) - elseif ret == -3 then - retry = true - client = nil - end - return ret, err - end - return 0 -end - - -function timer_event(ns, shutdown) - local timedout = (ns / 1e9 - last_flush) >= ticker_interval - if (timedout or (shutdown and flush_on_shutdown)) and batch_count > 0 then - local ret, err = send_batch() - if ret == 0 then - retry = false - batch_count = 0 - batch:close() - if not shutdown then - batch = assert(io.open(batch_file, "w")) - end - end - end -end diff --git a/sandboxes/heka/output/heka_kafka.lua b/sandboxes/heka/output/heka_kafka.lua deleted file mode 100644 index 1bbdf68..0000000 --- a/sandboxes/heka/output/heka_kafka.lua +++ /dev/null @@ -1,63 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "kafka" - ---[[ -# Heka Kafka Producer Output - -## Sample Configuration -```lua -filename = "heka_kafka.lua" -message_matcher = "TRUE" -output_limit = 8 * 1024 * 1024 -brokerlist = "localhost:9092" -- see https://github.com/edenhill/librdkafka/blob/master/src/rdkafka.h#L2205 -ticker_interval = 60 -async_buffer_size = 20000 - -topic_constant = "test" -producer_conf = { - ["queue.buffering.max.messages"] = async_buffer_size, - ["batch.num.messages"] = 200, - ["message.max.bytes"] = output_limit, - ["queue.buffering.max.ms"] = 10, - ["topic.metadata.refresh.interval.ms"] = -1, -} -``` ---]] -local brokerlist = read_config("brokerlist") or error("brokerlist must be set") -local topic_constant = read_config("topic_constant") -local topic_variable = read_config("topic_variable") or "Logger" -local producer_conf = read_config("producer_conf") - -local producer = kafka.producer(brokerlist, producer_conf) - -function process_message(sequence_id) - local topic = topic_constant - if not topic then - topic = read_message(topic_variable) or "unknown" - end - producer:create_topic(topic) -- creates the topic if it does not exist - - producer:poll() - local ret = producer:send(topic, -1, sequence_id) -- sends the current message - - if ret ~= 0 then - if ret == 105 then - return -3, "queue full" -- retry - elseif ret == 90 then - return -1, "message too large" -- fail - elseif ret == 2 then - error("unknown topic: " .. topic) - elseif ret == 3 then - error("unknown partition") - end - end - - return -5 -- asynchronous checkpoint management -end - -function timer_event(ns) - producer:poll() -end diff --git a/sandboxes/heka/output/heka_log_rolling.lua b/sandboxes/heka/output/heka_log_rolling.lua deleted file mode 100644 index 943e8e3..0000000 --- a/sandboxes/heka/output/heka_log_rolling.lua +++ /dev/null @@ -1,53 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "io" -require "string" - ---[[ -# Heka Protobuf Stream Output (rolled by size) - -Outputs a Heka protobuf stream rolling the log file every time it reaches the `roll_size`. - -## Sample Configuration -```lua -filename = "heka_log_rolling.lua" -message_matcher = "TRUE" -ticker_interval = 0 -preserve_data = true - ---location where the payload is written -output_dir = "/tmp" -roll_size = 1024 * 1024 * 1024 -``` ---]] - -file_num = 0 - -local output_dir = read_config("output_dir") or "/tmp" -local output_prefix = read_config("Logger") -local roll_size = read_config("roll_size") or 1e9 -local fh - -function process_message() - if not fh then - local fn = string.format("%s/%s.%d.log", output_dir, output_prefix, file_num) - fh, err = io.open(fn, "a") - if err then return -1, err end - end - - local msg = read_message("framed") - fh:write(msg) - - if fh:seek() >= roll_size then - fh:close() - fh = nil - file_num = file_num + 1 - end - return 0 -end - -function timer_event(ns) - -- no op -end diff --git a/sandboxes/heka/output/heka_s3_partition.lua b/sandboxes/heka/output/heka_s3_partition.lua deleted file mode 100644 index 3236b36..0000000 --- a/sandboxes/heka/output/heka_s3_partition.lua +++ /dev/null @@ -1,281 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "cjson" -require "io" -require "os" -require "string" -require "table" - -files = {} -local fh_cnt = 0 -local time_t = 0 -local buffer_cnt = 0 - -local hostname = read_config("Hostname") -local batch_path = read_config("batch_path") or error("batch_path must be specified") -local s3_path = read_config("s3_path") or error("s3_path must be specified") -local max_file_handles = read_config("max_file_handles") or 1000 -local max_file_size = read_config("max_file_size") or 1024 * 1024 * 500 -local max_file_age = read_config("max_file_age") or 60 * 60 -local flush_on_shutdown = read_config("flush_on_shutdown") - ---[[ -# Heka Protobuf Stream S3 Output Partitioner - -Batches message data into Heka protobuf stream files based on the specified path -dimensions and copies them to S3 when they reach the maximum size or maximum -age. - -## Configuration - -### Dimension Specification File -This file contains a JSON specification for how the data should be partitioned -into files and the location each file will reside at in S3. The JSON is shared -with other tools so it is not directly included in the configuration. - -TODO: This specifification should be expanded (when necessary) to include -- header values -- field/array index support -- patterns instead of exact matches - -#### Sample Specification File -```json - { - "version": 1, - "dimensions": [ - {"field_name": "submissionDate", - "allowed_values": {"min": "20140120", "max": "20140125"}}, - {"field_name": "sourceName", - "allowed_values": "*"}, - {"field_name": "sourceVersion", - "allowed_values": "*"}, - {"field_name": "reason", - "allowed_values": ["idle-daily","saved-session"]}, - {"field_name": "appName", - "allowed_values": ["Firefox", "Fennec", "Thunderbird", "FirefoxOS", "B2G"]}, - {"field_name": "appUpdateChannel", - "allowed_values": ["default", "nightly", "aurora", "beta", "release", "esr"]}, - {"field_name": "appVersion", - "allowed_values": "*"} - ] - } -``` - -### Sample Configuration - -```lua -filename = "heka_s3_partition.lua" - --- see the specification above -dimension_file = "foobar.json" - --- directory location to store the intermediate output files -batch_path = "/var/tmp/foobar" - --- Specifies how many data files to keep open at once. If there are more --- "current" files than this, the least-recently used file will be closed --- and then re-opened if more messages arrive before it is copied to S3. The --- default is 1000. A value of 0 means no maximum. -max_file_handles = 1000 - --- Specifies how much data (in bytes) can be written to a single file before --- it is copied to s3 (default 500MB) -max_file_size = 1024 * 1024 * 500 - --- Specifies how long (in seconds) to wait before it is copied to s3 --- (default 1 hour). Idle files are only checked every ticker_interval seconds. -max_file_age = 60 * 60 - --- Specifies that all local files will be copied S3 before exiting (default false). -flush_on_shutdown = true -preserve_data = not flush_on_shutdown -- should always be the inverse of flush_on_shutdown - -s3_path = "s3://foo" - -ticker_interval = 60 -``` ---]] - -local function sanitize_dimension(d) - if d ~= nil then - return string.gsub(tostring(d), "[^a-zA-Z0-9_.]", "_") - end -end - - -local function is_in_list(v, av) - for i, j in ipairs(av) do - if v == j then return true end - end - return false -end - - -local function is_in_range(v, min, max) - if min and v < min then return false end - if max and v > max then return false end - return true -end - - -local function validate_dimensions(fn) - local fh = assert(io.open(fn)) - local json = fh:read("*a") - fh:close() - local df = cjson.decode(json) - for i,d in ipairs(df.dimensions) do - local name = string.format("Fields[%s]", d.field_name) - d.field_name = name - local av = d.allowed_values - if type(av) == "string" then - if av == "*" then - d.matcher = function (v) return true end - else - av = sanitize_dimension(av) - d.matcher = function (v) return v == av end - end - elseif type(av) == "table" then - if av[1] ~= nil then - for m,n in ipairs(av) do - if type(n) ~= "string" then - error(string.format("field '$s' allowed_values array must contain only strings", name)) - end - av[m] = sanitize_dimension(n) - end - d.matcher = function (v) return is_in_list(v, av) end - else - if not av.min and not av.max then - error(string.format("field '%s' allowed_values range must have a 'min' or 'max'", name)) - end - if av.min and type(av.min) ~= "string" - or av.max and type(av.max) ~= "string" then - error(string.format("field '%s' allowed_values range min/max must be a string", name)) - end - d.matcher = function (v) return is_in_range(v, av.min, av.max) end - end - else - error(string.format("field '%s' allowed_values invalid type: %s", type(av))) - end - end - return df.dimensions -end - - -local function get_fqfn(path) - return string.format("%s/%s", batch_path, path) -end - - -local function close_fh(entry) - if not entry[2] then return end - entry[2]:close() - entry[2] = nil - fh_cnt = fh_cnt - 1 -end - - -local function copy_file(path, entry) - close_fh(entry) - local t = os.time() - local cmd - if t == time_t then - buffer_cnt = buffer_cnt + 1 - else - time_t = t - buffer_cnt = 0 - end - - local src = get_fqfn(path) - cmd = string.format("aws s3 cp %s %s/%s/%d_%d_%s", src, s3_path, - string.gsub(path, "+", "/"), time_t, buffer_cnt, hostname) - - local ret = os.execute(cmd) - if ret ~= 0 then - return string.format("ret: %d, cmd: %s", ret, cmd) - end - files[path] = nil - - local ok, err = os.remove(src); - if not ok then - return string.format("os.remove('%s') failed: %s", path, err) - end -end - - -local function get_entry(path) - local ct = os.time() - local t = files[path] - if not t then - t = {ct, nil} -- last active, file handle - files[path] = t - else - t[1] = ct - end - - if not t[2] then - if max_file_handles ~= 0 then - if fh_cnt >= max_file_handles then - local oldest = ct + 60 - local entry - for k,v in pairs(files) do -- if we max out file handles a lot we will want to make this more efficient - local et = v[1] - if v[2] and et < oldest then - entry = v - oldest = et - end - end - if entry then close_fh(entry) end - end - end - t[2] = assert(io.open(get_fqfn(path), "a")) - fh_cnt = fh_cnt + 1 - end - return t -end - -local dimensions = validate_dimensions(read_config("dimension_file")) - -function process_message() - local dims = {} - for i,d in ipairs(dimensions) do - local v = sanitize_dimension(read_message(d.field_name)) - if v then - if d.matcher(v) then - dims[i] = v - else - dims[i] = "OTHER" - end - else - dims[i] = "UNKNOWN" - end - end - local path = table.concat(dims, "+") -- the plus will be converted to a path separator '/' on copy - local entry = get_entry(path) - local fh = entry[2] - fh:write(read_message("framed")) - local size = fh:seek() - if size >= max_file_size then - local err = copy_file(path, entry) - if err then return -1, err end - end - return 0 -end - - -function timer_event(ns, shutdown) - local err - local ct = os.time() - for k,v in pairs(files) do - if (shutdown and flush_on_shutdown) or (ct - v[1] >= max_file_age) then - local e = copy_file(k, v) - if e then err = e end - elseif shutdown then - close_fh(v) - end - end - if shutdown and flush_on_shutdown and err then - error(string.format("flush on shutdown failed, last error: %s", err)) - end -end diff --git a/sandboxes/heka/output/heka_tcp.lua b/sandboxes/heka/output/heka_tcp.lua deleted file mode 100644 index 342eca6..0000000 --- a/sandboxes/heka/output/heka_tcp.lua +++ /dev/null @@ -1,84 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Heka Compatible TCP Output - -## Sample Configuration -```lua -filename = "heka_tcp.lua" -message_matcher = "TRUE" - -address = "127.0.0.1" -port = 5565 -timeout = 10 - -ssl_params = { - mode = "client", - protocol = "tlsv1", - key = "/etc/hindsight/certs/clientkey.pem", - certificate = "/etc/hindsight/certs/client.pem", - cafile = "/etc/hindsight/certs/CA.pem", - verify = "peer", - options = {"all", "no_sslv3"} -} -``` ---]] - -local socket = require "socket" - -local address = read_config("address") or "127.0.0.1" -local port = read_config("port") or 5565 -local timeout = read_config("timeout") or 10 -local ssl_params = read_config("ssl_params") - -local ssl_ctx = nil -if ssl_params then - require "ssl" - ssl_ctx = assert(ssl.newcontext(ssl_params)) -end - -local function create_client() - local c, err = socket.connect(address, port) - if c then - c:setoption("tcp-nodelay", true) - c:setoption("keepalive", true) - c:settimeout(timeout) - if ssl_ctx then - c, err = ssl.wrap(c, ssl_ctx) - if c then - c:dohandshake() - end - end - end - return c, err -end - -local client, err = create_client() - -local function send_message(msg, i) - local len, err, i = client:send(msg, i) - if not len then - if err == "timeout" or err == "closed" then - client:close() - client = nil - return -3, err - end - return send_message(msg, i) - end - return 0 -end - -function process_message() - if not client then - client, err = create_client() - end - if not client then return -3, err end -- retry indefinitely - local ret, err = send_message(read_message("framed"), 1) - return ret, err -end - -function timer_event(ns) - -- no op -end diff --git a/sandboxes/heka/output/heka_tcp_matcher.lua b/sandboxes/heka/output/heka_tcp_matcher.lua deleted file mode 100644 index 58a4791..0000000 --- a/sandboxes/heka/output/heka_tcp_matcher.lua +++ /dev/null @@ -1,117 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - ---[[ -# Heka Compatible TCP Output with Dynamic Matching - -Used for generating a debug stream of messages for as long as the connection is -maintained. - -## Sample Configuration -```lua -filename = "heka_tcp_matcher.lua" -instruction_limit = 0 -messsage_matcher = "TRUE" -ticker_interval = 1 - -address = "127.0.0.1" -port = 5566 - -ssl_params = { - mode = "server", - protocol = "tlsv1", - key = "/etc/hindsight/certs/serverkey.pem", - certificate = "/etc/hindsight/certs/server.pem", - cafile = "/etc/hindsight/certs/CA.pem", - verify = {"peer", "fail_if_no_peer_cert"}, - options = {"all", "no_sslv3"} -} -``` ---]] - -require "string" -local socket = require "socket" -require "table" - -local address = read_config("address") or "127.0.0.1" -local port = read_config("port") or 5566 -local ssl_params = read_config("ssl_params") -local ssl_ctx = nil -if ssl_params then - require "ssl" - ssl_ctx = assert(ssl.newcontext(ssl_params)) -end -local server = assert(socket.bind(address, port)) -server:settimeout(0) -local sockets = {server} -local subscribers = {} - -local function send_message(socket, msg, i) - local len, err, i = socket:send(msg, i) - if not len then - if err == "timeout" or err == "closed" then - return false - end - return send_message(msg, i) - end - return true -end - - -local num_subs = 0 -function process_message() - local msg - for i = num_subs, 1, -1 do - local sub = subscribers[i] - if sub[2]:eval() then - if not msg then msg = read_message("framed") end - if not send_message(sub[1], msg, 1) then - sub[1]:close() - table.remove(subscribers, i) - num_subs = num_subs - 1 - end - end - end - return 0 -end - - -function timer_event(ns, shutdown) - if shutdown then - for i,v in ipairs(subscribers) do - v[1]:close() - end - return - end - - local ready = socket.select(sockets, nil, 0) - if ready then - for _, s in ipairs(ready) do - local client = s:accept() - if client then - if ssl_ctx then - client = ssl.wrap(client, ssl_ctx) - client:dohandshake() - end - local exp, err = client:receive("*l") - local ok, mm = pcall(create_message_matcher, exp) - if ok then - num_subs = num_subs + 1 - subscribers[num_subs] = {client, mm} - else - print("bad matcher ", exp, mm) - local msg = encode_message( - { - Type = "bad matcher", - Payload = string.format("'%s' %s", tostring(exp), tostring(mm)) - }, - true) - send_message(client, msg, 1) - client:close() - end - end - end - end -end - diff --git a/sandboxes/heka/output/inject_payload.lua b/sandboxes/heka/output/inject_payload.lua deleted file mode 100644 index 9514950..0000000 --- a/sandboxes/heka/output/inject_payload.lua +++ /dev/null @@ -1,55 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "io" -require "string" - ---[[ -# Message Payload Output - -Outputs the message payload to the configured directory. The output filename is -`output_dir`/`Logger.Fields[payload_name]`.`Fields[payload_type]` -and the contents are the message `Payload`. - -## Sample Configuration -```lua -filename = "inject_payload.lua" -message_matcher = "Type == 'inject_payload'" -ticker_interval = 0 - --- location where the payload is written (e.g. make them accessible from a web --- server for external consumption) -output_dir = "/var/www/hindsight/payload" -``` ---]] - -local output_dir = read_config("output_dir") or "/tmp" - -function process_message() - local pt = read_message("Fields[payload_type]") - if type(pt) ~= "string" then return -1, "invalid payload_type" end - - local pn = read_message("Fields[payload_name]") or "" - if type(pn) ~= "string" then return -1, "invalid payload_name" end - - local logger = read_message("Logger") or "" - - pn = string.gsub(pn, "%W", "_") - pt = string.gsub(pt, "%W", "_") - logger = string.gsub(logger, "%W", "_") - - local fn = string.format("%s/%s.%s.%s", output_dir, logger, pn, pt) - local fh, err = io.open(fn, "w") - if err then return -1, err end - - local payload = read_message("Payload") or "" - fh:write(payload) - fh:close() - - return 0 -end - -function timer_event(ns) - -- no op -end diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 661552f..367bafa 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,32 +2,67 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. +if(MSVC) + add_definitions( + -DLUA_WIN + -DLUA_BUILD_AS_DLL + -D_CRT_SECURE_NO_WARNINGS + ) +else() + add_definitions( + -DLUA_USE_POSIX + -DLUA_USE_DLOPEN + -DLUA_USE_STRTODHEX + -DLUA_USE_LONGLONG + -DLUA_USE_GMTIME_R + ) +endif() + +set(LUA_SRC +lua/lapi.c +lua/lauxlib.c +lua/lbaselib.c +lua/lcode.c +lua/ldblib.c +lua/ldebug.c +lua/ldo.c +lua/ldump.c +lua/lfunc.c +lua/lgc.c +lua/linit.c +lua/liolib.c +lua/llex.c +lua/lmathlib.c +lua/lmem.c +lua/loadlib.c +lua/lobject.c +lua/lopcodes.c +lua/loslib.c +lua/lparser.c +lua/lstate.c +lua/lstring.c +lua/lstrlib.c +lua/ltable.c +lua/ltablib.c +lua/ltm.c +lua/lundump.c +lua/lvm.c +lua/lzio.c +) + set(LUA_SANDBOX_SRC luasandbox.c luasandbox_output.c luasandbox_serialize.c ) -set ( CMAKE_SKIP_BUILD_RPATH FALSE CACHE STRING "" FORCE ) -set ( CMAKE_BUILD_WITH_INSTALL_RPATH FALSE CACHE STRING "" FORCE ) - -if(MSVC) - add_definitions(-D_CRT_SECURE_NO_WARNINGS) - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core PATTERN "*.dll") -endif() - -add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) -add_dependencies(luasandbox ${LUA_PROJECT}) - -set_target_properties(luasandbox PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) -target_link_libraries(luasandbox luasandboxutil luasb ${CMAKE_DL_LIBS}) +add_library(luasandbox SHARED ${LUA_SANDBOX_SRC} ${LUA_SRC}) +set_target_properties(luasandbox PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 0) +target_link_libraries(luasandbox luasandboxutil ${CMAKE_DL_LIBS}) if(LIBM_LIBRARY) target_link_libraries(luasandbox ${LIBM_LIBRARY}) endif() - -install(TARGETS luasandbox DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) -install(DIRECTORY "${EP_BASE}/inst/${CMAKE_INSTALL_LIBDIR}/" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) -install(DIRECTORY "${EP_BASE}/inst/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} COMPONENT core) +install(TARGETS luasandbox DESTINATION ${CMAKE_INSTALL_LIBDIR}) add_subdirectory(util) add_subdirectory(heka) diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt index dbf37b3..4d580b8 100644 --- a/src/cli/CMakeLists.txt +++ b/src/cli/CMakeLists.txt @@ -9,4 +9,4 @@ if(LIBM_LIBRARY) target_link_libraries(lsb_heka_cat ${LIBM_LIBRARY}) endif() -install(TARGETS lsb_heka_cat DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT core) +install(TARGETS lsb_heka_cat DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index ed5d09a..daa8605 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -20,5 +20,5 @@ if(LIBM_LIBRARY) target_link_libraries(luasandboxheka ${LIBM_LIBRARY}) endif() -install(TARGETS luasandboxheka DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) +install(TARGETS luasandboxheka DESTINATION ${CMAKE_INSTALL_LIBDIR}) add_subdirectory(test) diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt index f0afa42..e38530e 100644 --- a/src/heka/test/CMakeLists.txt +++ b/src/heka/test/CMakeLists.txt @@ -4,7 +4,7 @@ configure_file(test.h.in test.h ESCAPE_QUOTES) include_directories(${CMAKE_CURRENT_BINARY_DIR}) -set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/heka") +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/heka") add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) @@ -15,7 +15,4 @@ add_test(NAME test_heka_sandbox COMMAND test_heka_sandbox) if(WIN32) STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) -elseif(APPLE) - STRING(REPLACE ";" ":" LIBRARY_PATHS "${LIBRARY_PATHS}") - set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua index 2e4933c..4bd2b0e 100644 --- a/src/heka/test/lua/encode_message.lua +++ b/src/heka/test/lua/encode_message.lua @@ -4,12 +4,6 @@ require "string" require "table" -require "circular_buffer" -require "rjson" - -local cb = circular_buffer.new(2,1,1) -local doc = rjson.parse([[{"foo":"bar"}]]) -assert(doc) local msgs = { { @@ -72,13 +66,6 @@ local msgs = { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = {1,2,3}, value_type = 2, representation = "widget"}}}, -- int rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\20\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\50\3\1\2\3", }, - { - msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = cb}}, -- userdata - rv = '\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\158\1\10\3\107\101\121\16\1\42\148\1\123\34\116\105\109\101\34\58\48\44\34\114\111\119\115\34\58\50\44\34\99\111\108\117\109\110\115\34\58\49\44\34\115\101\99\111\110\100\115\95\112\101\114\95\114\111\119\34\58\49\44\34\99\111\108\117\109\110\95\105\110\102\111\34\58\91\123\34\110\97\109\101\34\58\34\67\111\108\117\109\110\95\49\34\44\34\117\110\105\116\34\58\34\99\111\117\110\116\34\44\34\97\103\103\114\101\103\97\116\105\111\110\34\58\34\115\117\109\34\125\93\44\34\97\110\110\111\116\97\116\105\111\110\115\34\58\91\93\125\10\110\97\110\10\110\97\110\10' - }, - { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {json = doc:make_field()}}, - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\23\10\4\106\115\111\110\16\1\42\13\123\34\102\111\111\34\58\34\98\97\114\34\125" - }, } for i, v in ipairs(msgs) do @@ -113,6 +100,3 @@ assert("encode_message() failed: array has mixed types" == err, string.format("r ok, err = pcall(encode_message, {Fields = { {noname = "foo", value = 1}} }) assert(not ok) assert("encode_message() failed: field name must be a string" == err, string.format("received: %s", err)) - -ok, err = pcall(rjson.parse_message, "Payload") -assert("parse_message() no active message" == err, err) diff --git a/src/heka/test/lua/heka_util.lua b/src/heka/test/lua/heka_util.lua deleted file mode 100644 index 6eda8a1..0000000 --- a/src/heka/test/lua/heka_util.lua +++ /dev/null @@ -1,28 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local string = require "string" -local util = require "heka.util" - -local t = {toplevel=0, struct = { item0 = 0, item1 = 1, item2 = {nested = "n1"}}} -local fa = {} -local fb = {} - -local function table_to_fields() - util.table_to_fields(t, fa, nil) - assert(fa.toplevel == 0, fa.toplevel) - assert(fa["struct.item0"] == 0, fa["struct.item0"]) - assert(fa["struct.item1"] == 1, fa["struct.item1"]) - assert(fa["struct.item2.nested"] == "n1", fa["struct.item2.nested"]) - util.table_to_fields(t, fb, nil, "_", 2) - assert(fb.toplevel == 0, fb.toplevel) - assert(fb["struct_item0"] == 0, fb["struct_item0"]) - assert(fb["struct_item1"] == 1, fb["struct_item1"]) - assert(fb["struct_item2"] == '{"nested":"n1"}', fb["struct_item2"]) -end - -function process_message () - table_to_fields() - return 0 -end diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index 55d5948..77c2d94 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -3,18 +3,7 @@ -- file, You can obtain one at http://mozilla.org/MPL/2.0/. require "string" -require "bloom_filter"; -require "rjson" -local bf = bloom_filter.new(10, 0.01) - -local json = [[{"foo":"bar"}]] -local doc = rjson.parse(json) -assert(doc) -local doc1 = rjson.parse(json) -local doc2 = rjson.parse(json) -local large_json = string.format('{"foo":"%s"}', string.rep("x", 66000)) -local large_doc = rjson.parse(large_json) -assert(large_doc) +require "io" -- Table tests local msgs = { @@ -22,9 +11,6 @@ local msgs = { {{Timestamp = 1, Uuid = "00000000-0000-0000-0000-000000000000"}, nil}, {{Timestamp = 2, Uuid = "00000000-0000-0000-0000-000000000000", Logger = "logger", Hostname = "hostname", Type = "type", Payload = "payload", EnvVersion = "envversion", Pid = 99, Severity = 5}, 99}, {{Timestamp = 3, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}}, "foo.log:123"}, - {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc:make_field()}}, nil}, -- use the lightuserdata - {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc}}, nil}, -- use the full userdata - {{Timestamp = 4, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {json = doc2:remove()}}, nil}, -- use the removed item } local err_msgs = { @@ -42,10 +28,7 @@ local err_msgs = { {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=1}}}, err = "inject_message() failed: invalid boolean value_type: 1"}, {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, - {msg = {Timestamp = 1e9, Fields = {bf = {value=bf, representation="bf"}}}, err = "inject_message() failed: userdata object does not implement lsb_output"}, - {msg = {Timestamp = 1e9, Fields = {json = {value = doc:find()}}}, err = "inject_message() failed: a lightuserdata output must also specify a userdata value"}, - {msg = {Timestamp = 1e9, Fields = {json = {value = doc1:find(), userdata = doc}}}, err = "inject_message() failed: userdata output callback failed: 1"}, - {msg = {Timestamp = 1e9, Fields = {json = large_doc:make_field()}}, err = "inject_message() failed: userdata output callback failed: 1"}, + {msg = {Timestamp = 1e9, Fields = {bf = {value=io.stdin, representation="bf"}}}, err = "inject_message() failed: userdata object does not implement lsb_output"}, } for i, v in ipairs(msgs) do diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 7c2a9e8..66ff1ff 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -51,9 +51,6 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03iim\x4a\x02hn", .pb_len = 29, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x03iim\x4a\x02hn\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 165, .cp_numeric = NAN, .cp_string = "foo.log:123" }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x22\x03iim\x4a\x02hn\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 54, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x22\x03iim\x4a\x02hn\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 54, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04\x22\x03iim\x4a\x02hn\x52\x17\x0a\x04\x6a\x73\x6f\x6e\x10\x01\x2a\x0d\x7b\x22\x66\x6f\x6f\x22\x3a\x22\x62\x61\x72\x22\x7d", .pb_len = 54, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, }; @@ -533,8 +530,8 @@ static char* test_im_input() "Logger = 'iim'\n", &logger, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(9 == stats.im_cnt, "received %llu", stats.im_cnt); - mu_assert(494 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(332 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -567,7 +564,7 @@ static char* test_encode_message() &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(189 == stats.out_max, "received %llu", stats.out_max); + mu_assert(162 == stats.out_max, "received %llu", stats.out_max); e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -604,27 +601,29 @@ static char* test_read_message() } -static char* test_heka_util() +static char* test_get_message() { - lsb_heka_message m; - mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); - mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); - lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/heka_util.lua", NULL, - "path = [[" TEST_LUA_PATH "]]\n" - "cpath = [[" TEST_LUA_CPATH "]]\n", - &logger, aim); - mu_assert(hsb, "lsb_heka_create_analysis failed"); - int rv = lsb_heka_pm_analysis(hsb, &m, false); - mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, - lsb_heka_get_error(hsb)); + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); + mu_assert(lsb_heka_get_message(NULL) == NULL, "non NULL"); + mu_assert(lsb_heka_get_message(hsb) == NULL, "non NULL"); e = lsb_heka_destroy_sandbox(hsb); - lsb_free_heka_message(&m); return NULL; } +static char* test_get_type() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); + char t = lsb_heka_get_type(NULL); + mu_assert( t == '\0', "received %c", t); + t = lsb_heka_get_type(hsb); + mu_assert(t == 'a', "received %c", t); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + static char* benchmark_decode_message() { @@ -669,7 +668,8 @@ static char* all_tests() mu_run_test(test_encode_message); mu_run_test(test_decode_message); mu_run_test(test_read_message); - mu_run_test(test_heka_util); + mu_run_test(test_get_message); + mu_run_test(test_get_type); mu_run_test(benchmark_decode_message); return NULL; diff --git a/src/lua/lapi.c b/src/lua/lapi.c new file mode 100644 index 0000000..5d5145d --- /dev/null +++ b/src/lua/lapi.c @@ -0,0 +1,1087 @@ +/* +** $Id: lapi.c,v 2.55.1.5 2008/07/04 18:41:18 roberto Exp $ +** Lua API +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include + +#define lapi_c +#define LUA_CORE + +#include "lua.h" + +#include "lapi.h" +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lundump.h" +#include "lvm.h" + + + +const char lua_ident[] = + "$Lua: " LUA_RELEASE " " LUA_COPYRIGHT " $\n" + "$Authors: " LUA_AUTHORS " $\n" + "$URL: www.lua.org $\n"; + + + +#define api_checknelems(L, n) api_check(L, (n) <= (L->top - L->base)) + +#define api_checkvalidindex(L, i) api_check(L, (i) != luaO_nilobject) + +#define api_incr_top(L) {api_check(L, L->top < L->ci->top); L->top++;} + + + +static TValue *index2adr (lua_State *L, int idx) { + if (idx > 0) { + TValue *o = L->base + (idx - 1); + api_check(L, idx <= L->ci->top - L->base); + if (o >= L->top) return cast(TValue *, luaO_nilobject); + else return o; + } + else if (idx > LUA_REGISTRYINDEX) { + api_check(L, idx != 0 && -idx <= L->top - L->base); + return L->top + idx; + } + else switch (idx) { /* pseudo-indices */ + case LUA_REGISTRYINDEX: return registry(L); + case LUA_ENVIRONINDEX: { + Closure *func = curr_func(L); + sethvalue(L, &L->env, func->c.env); + return &L->env; + } + case LUA_GLOBALSINDEX: return gt(L); + default: { + Closure *func = curr_func(L); + idx = LUA_GLOBALSINDEX - idx; + return (idx <= func->c.nupvalues) + ? &func->c.upvalue[idx-1] + : cast(TValue *, luaO_nilobject); + } + } +} + + +static Table *getcurrenv (lua_State *L) { + if (L->ci == L->base_ci) /* no enclosing function? */ + return hvalue(gt(L)); /* use global table as environment */ + else { + Closure *func = curr_func(L); + return func->c.env; + } +} + + +void luaA_pushobject (lua_State *L, const TValue *o) { + setobj2s(L, L->top, o); + api_incr_top(L); +} + + +LUA_API int lua_checkstack (lua_State *L, int size) { + int res = 1; + lua_lock(L); + if (size > LUAI_MAXCSTACK || (L->top - L->base + size) > LUAI_MAXCSTACK) + res = 0; /* stack overflow */ + else if (size > 0) { + luaD_checkstack(L, size); + if (L->ci->top < L->top + size) + L->ci->top = L->top + size; + } + lua_unlock(L); + return res; +} + + +LUA_API void lua_xmove (lua_State *from, lua_State *to, int n) { + int i; + if (from == to) return; + lua_lock(to); + api_checknelems(from, n); + api_check(from, G(from) == G(to)); + api_check(from, to->ci->top - to->top >= n); + from->top -= n; + for (i = 0; i < n; i++) { + setobj2s(to, to->top++, from->top + i); + } + lua_unlock(to); +} + + +LUA_API void lua_setlevel (lua_State *from, lua_State *to) { + to->nCcalls = from->nCcalls; +} + + +LUA_API lua_CFunction lua_atpanic (lua_State *L, lua_CFunction panicf) { + lua_CFunction old; + lua_lock(L); + old = G(L)->panic; + G(L)->panic = panicf; + lua_unlock(L); + return old; +} + + +LUA_API lua_State *lua_newthread (lua_State *L) { + lua_State *L1; + lua_lock(L); + luaC_checkGC(L); + L1 = luaE_newthread(L); + setthvalue(L, L->top, L1); + api_incr_top(L); + lua_unlock(L); + luai_userstatethread(L, L1); + return L1; +} + + + +/* +** basic stack manipulation +*/ + + +LUA_API int lua_gettop (lua_State *L) { + return cast_int(L->top - L->base); +} + + +LUA_API void lua_settop (lua_State *L, int idx) { + lua_lock(L); + if (idx >= 0) { + api_check(L, idx <= L->stack_last - L->base); + while (L->top < L->base + idx) + setnilvalue(L->top++); + L->top = L->base + idx; + } + else { + api_check(L, -(idx+1) <= (L->top - L->base)); + L->top += idx+1; /* `subtract' index (index is negative) */ + } + lua_unlock(L); +} + + +LUA_API void lua_remove (lua_State *L, int idx) { + StkId p; + lua_lock(L); + p = index2adr(L, idx); + api_checkvalidindex(L, p); + while (++p < L->top) setobjs2s(L, p-1, p); + L->top--; + lua_unlock(L); +} + + +LUA_API void lua_insert (lua_State *L, int idx) { + StkId p; + StkId q; + lua_lock(L); + p = index2adr(L, idx); + api_checkvalidindex(L, p); + for (q = L->top; q>p; q--) setobjs2s(L, q, q-1); + setobjs2s(L, p, L->top); + lua_unlock(L); +} + + +LUA_API void lua_replace (lua_State *L, int idx) { + StkId o; + lua_lock(L); + /* explicit test for incompatible code */ + if (idx == LUA_ENVIRONINDEX && L->ci == L->base_ci) + luaG_runerror(L, "no calling environment"); + api_checknelems(L, 1); + o = index2adr(L, idx); + api_checkvalidindex(L, o); + if (idx == LUA_ENVIRONINDEX) { + Closure *func = curr_func(L); + api_check(L, ttistable(L->top - 1)); + func->c.env = hvalue(L->top - 1); + luaC_barrier(L, func, L->top - 1); + } + else { + setobj(L, o, L->top - 1); + if (idx < LUA_GLOBALSINDEX) /* function upvalue? */ + luaC_barrier(L, curr_func(L), L->top - 1); + } + L->top--; + lua_unlock(L); +} + + +LUA_API void lua_pushvalue (lua_State *L, int idx) { + lua_lock(L); + setobj2s(L, L->top, index2adr(L, idx)); + api_incr_top(L); + lua_unlock(L); +} + + + +/* +** access functions (stack -> C) +*/ + + +LUA_API int lua_type (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return (o == luaO_nilobject) ? LUA_TNONE : ttype(o); +} + + +LUA_API const char *lua_typename (lua_State *L, int t) { + UNUSED(L); + return (t == LUA_TNONE) ? "no value" : luaT_typenames[t]; +} + + +LUA_API int lua_iscfunction (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return iscfunction(o); +} + + +LUA_API int lua_isnumber (lua_State *L, int idx) { + TValue n; + const TValue *o = index2adr(L, idx); + return tonumber(o, &n); +} + + +LUA_API int lua_isstring (lua_State *L, int idx) { + int t = lua_type(L, idx); + return (t == LUA_TSTRING || t == LUA_TNUMBER); +} + + +LUA_API int lua_isuserdata (lua_State *L, int idx) { + const TValue *o = index2adr(L, idx); + return (ttisuserdata(o) || ttislightuserdata(o)); +} + + +LUA_API int lua_rawequal (lua_State *L, int index1, int index2) { + StkId o1 = index2adr(L, index1); + StkId o2 = index2adr(L, index2); + return (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 + : luaO_rawequalObj(o1, o2); +} + + +LUA_API int lua_equal (lua_State *L, int index1, int index2) { + StkId o1, o2; + int i; + lua_lock(L); /* may call tag method */ + o1 = index2adr(L, index1); + o2 = index2adr(L, index2); + i = (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 : equalobj(L, o1, o2); + lua_unlock(L); + return i; +} + + +LUA_API int lua_lessthan (lua_State *L, int index1, int index2) { + StkId o1, o2; + int i; + lua_lock(L); /* may call tag method */ + o1 = index2adr(L, index1); + o2 = index2adr(L, index2); + i = (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 + : luaV_lessthan(L, o1, o2); + lua_unlock(L); + return i; +} + + + +LUA_API lua_Number lua_tonumber (lua_State *L, int idx) { + TValue n; + const TValue *o = index2adr(L, idx); + if (tonumber(o, &n)) + return nvalue(o); + else + return 0; +} + + +LUA_API lua_Integer lua_tointeger (lua_State *L, int idx) { + TValue n; + const TValue *o = index2adr(L, idx); + if (tonumber(o, &n)) { + lua_Integer res; + lua_Number num = nvalue(o); + lua_number2integer(res, num); + return res; + } + else + return 0; +} + + +LUA_API int lua_toboolean (lua_State *L, int idx) { + const TValue *o = index2adr(L, idx); + return !l_isfalse(o); +} + + +LUA_API const char *lua_tolstring (lua_State *L, int idx, size_t *len) { + StkId o = index2adr(L, idx); + if (!ttisstring(o)) { + lua_lock(L); /* `luaV_tostring' may create a new string */ + if (!luaV_tostring(L, o)) { /* conversion failed? */ + if (len != NULL) *len = 0; + lua_unlock(L); + return NULL; + } + luaC_checkGC(L); + o = index2adr(L, idx); /* previous call may reallocate the stack */ + lua_unlock(L); + } + if (len != NULL) *len = tsvalue(o)->len; + return svalue(o); +} + + +LUA_API size_t lua_objlen (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + switch (ttype(o)) { + case LUA_TSTRING: return tsvalue(o)->len; + case LUA_TUSERDATA: return uvalue(o)->len; + case LUA_TTABLE: return luaH_getn(hvalue(o)); + case LUA_TNUMBER: { + size_t l; + lua_lock(L); /* `luaV_tostring' may create a new string */ + l = (luaV_tostring(L, o) ? tsvalue(o)->len : 0); + lua_unlock(L); + return l; + } + default: return 0; + } +} + + +LUA_API lua_CFunction lua_tocfunction (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return (!iscfunction(o)) ? NULL : clvalue(o)->c.f; +} + + +LUA_API void *lua_touserdata (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + switch (ttype(o)) { + case LUA_TUSERDATA: return (rawuvalue(o) + 1); + case LUA_TLIGHTUSERDATA: return pvalue(o); + default: return NULL; + } +} + + +LUA_API lua_State *lua_tothread (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return (!ttisthread(o)) ? NULL : thvalue(o); +} + + +LUA_API const void *lua_topointer (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + switch (ttype(o)) { + case LUA_TTABLE: return hvalue(o); + case LUA_TFUNCTION: return clvalue(o); + case LUA_TTHREAD: return thvalue(o); + case LUA_TUSERDATA: + case LUA_TLIGHTUSERDATA: + return lua_touserdata(L, idx); + default: return NULL; + } +} + + + +/* +** push functions (C -> stack) +*/ + + +LUA_API void lua_pushnil (lua_State *L) { + lua_lock(L); + setnilvalue(L->top); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushnumber (lua_State *L, lua_Number n) { + lua_lock(L); + setnvalue(L->top, n); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushinteger (lua_State *L, lua_Integer n) { + lua_lock(L); + setnvalue(L->top, cast_num(n)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushlstring (lua_State *L, const char *s, size_t len) { + lua_lock(L); + luaC_checkGC(L); + setsvalue2s(L, L->top, luaS_newlstr(L, s, len)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushstring (lua_State *L, const char *s) { + if (s == NULL) + lua_pushnil(L); + else + lua_pushlstring(L, s, strlen(s)); +} + + +LUA_API const char *lua_pushvfstring (lua_State *L, const char *fmt, + va_list argp) { + const char *ret; + lua_lock(L); + luaC_checkGC(L); + ret = luaO_pushvfstring(L, fmt, argp); + lua_unlock(L); + return ret; +} + + +LUA_API const char *lua_pushfstring (lua_State *L, const char *fmt, ...) { + const char *ret; + va_list argp; + lua_lock(L); + luaC_checkGC(L); + va_start(argp, fmt); + ret = luaO_pushvfstring(L, fmt, argp); + va_end(argp); + lua_unlock(L); + return ret; +} + + +LUA_API void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) { + Closure *cl; + lua_lock(L); + luaC_checkGC(L); + api_checknelems(L, n); + cl = luaF_newCclosure(L, n, getcurrenv(L)); + cl->c.f = fn; + L->top -= n; + while (n--) + setobj2n(L, &cl->c.upvalue[n], L->top+n); + setclvalue(L, L->top, cl); + lua_assert(iswhite(obj2gco(cl))); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushboolean (lua_State *L, int b) { + lua_lock(L); + setbvalue(L->top, (b != 0)); /* ensure that true is 1 */ + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushlightuserdata (lua_State *L, void *p) { + lua_lock(L); + setpvalue(L->top, p); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API int lua_pushthread (lua_State *L) { + lua_lock(L); + setthvalue(L, L->top, L); + api_incr_top(L); + lua_unlock(L); + return (G(L)->mainthread == L); +} + + + +/* +** get functions (Lua -> stack) +*/ + + +LUA_API void lua_gettable (lua_State *L, int idx) { + StkId t; + lua_lock(L); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + luaV_gettable(L, t, L->top - 1, L->top - 1); + lua_unlock(L); +} + + +LUA_API void lua_getfield (lua_State *L, int idx, const char *k) { + StkId t; + TValue key; + lua_lock(L); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + setsvalue(L, &key, luaS_new(L, k)); + luaV_gettable(L, t, &key, L->top); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_rawget (lua_State *L, int idx) { + StkId t; + lua_lock(L); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + setobj2s(L, L->top - 1, luaH_get(hvalue(t), L->top - 1)); + lua_unlock(L); +} + + +LUA_API void lua_rawgeti (lua_State *L, int idx, int n) { + StkId o; + lua_lock(L); + o = index2adr(L, idx); + api_check(L, ttistable(o)); + setobj2s(L, L->top, luaH_getnum(hvalue(o), n)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_createtable (lua_State *L, int narray, int nrec) { + lua_lock(L); + luaC_checkGC(L); + sethvalue(L, L->top, luaH_new(L, narray, nrec)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API int lua_getmetatable (lua_State *L, int objindex) { + const TValue *obj; + Table *mt = NULL; + int res; + lua_lock(L); + obj = index2adr(L, objindex); + switch (ttype(obj)) { + case LUA_TTABLE: + mt = hvalue(obj)->metatable; + break; + case LUA_TUSERDATA: + mt = uvalue(obj)->metatable; + break; + default: + mt = G(L)->mt[ttype(obj)]; + break; + } + if (mt == NULL) + res = 0; + else { + sethvalue(L, L->top, mt); + api_incr_top(L); + res = 1; + } + lua_unlock(L); + return res; +} + + +LUA_API void lua_getfenv (lua_State *L, int idx) { + StkId o; + lua_lock(L); + o = index2adr(L, idx); + api_checkvalidindex(L, o); + switch (ttype(o)) { + case LUA_TFUNCTION: + sethvalue(L, L->top, clvalue(o)->c.env); + break; + case LUA_TUSERDATA: + sethvalue(L, L->top, uvalue(o)->env); + break; + case LUA_TTHREAD: + setobj2s(L, L->top, gt(thvalue(o))); + break; + default: + setnilvalue(L->top); + break; + } + api_incr_top(L); + lua_unlock(L); +} + + +/* +** set functions (stack -> Lua) +*/ + + +LUA_API void lua_settable (lua_State *L, int idx) { + StkId t; + lua_lock(L); + api_checknelems(L, 2); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + luaV_settable(L, t, L->top - 2, L->top - 1); + L->top -= 2; /* pop index and value */ + lua_unlock(L); +} + + +LUA_API void lua_setfield (lua_State *L, int idx, const char *k) { + StkId t; + TValue key; + lua_lock(L); + api_checknelems(L, 1); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + setsvalue(L, &key, luaS_new(L, k)); + luaV_settable(L, t, &key, L->top - 1); + L->top--; /* pop value */ + lua_unlock(L); +} + + +LUA_API void lua_rawset (lua_State *L, int idx) { + StkId t; + lua_lock(L); + api_checknelems(L, 2); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + setobj2t(L, luaH_set(L, hvalue(t), L->top-2), L->top-1); + luaC_barriert(L, hvalue(t), L->top-1); + L->top -= 2; + lua_unlock(L); +} + + +LUA_API void lua_rawseti (lua_State *L, int idx, int n) { + StkId o; + lua_lock(L); + api_checknelems(L, 1); + o = index2adr(L, idx); + api_check(L, ttistable(o)); + setobj2t(L, luaH_setnum(L, hvalue(o), n), L->top-1); + luaC_barriert(L, hvalue(o), L->top-1); + L->top--; + lua_unlock(L); +} + + +LUA_API int lua_setmetatable (lua_State *L, int objindex) { + TValue *obj; + Table *mt; + lua_lock(L); + api_checknelems(L, 1); + obj = index2adr(L, objindex); + api_checkvalidindex(L, obj); + if (ttisnil(L->top - 1)) + mt = NULL; + else { + api_check(L, ttistable(L->top - 1)); + mt = hvalue(L->top - 1); + } + switch (ttype(obj)) { + case LUA_TTABLE: { + hvalue(obj)->metatable = mt; + if (mt) + luaC_objbarriert(L, hvalue(obj), mt); + break; + } + case LUA_TUSERDATA: { + uvalue(obj)->metatable = mt; + if (mt) + luaC_objbarrier(L, rawuvalue(obj), mt); + break; + } + default: { + G(L)->mt[ttype(obj)] = mt; + break; + } + } + L->top--; + lua_unlock(L); + return 1; +} + + +LUA_API int lua_setfenv (lua_State *L, int idx) { + StkId o; + int res = 1; + lua_lock(L); + api_checknelems(L, 1); + o = index2adr(L, idx); + api_checkvalidindex(L, o); + api_check(L, ttistable(L->top - 1)); + switch (ttype(o)) { + case LUA_TFUNCTION: + clvalue(o)->c.env = hvalue(L->top - 1); + break; + case LUA_TUSERDATA: + uvalue(o)->env = hvalue(L->top - 1); + break; + case LUA_TTHREAD: + sethvalue(L, gt(thvalue(o)), hvalue(L->top - 1)); + break; + default: + res = 0; + break; + } + if (res) luaC_objbarrier(L, gcvalue(o), hvalue(L->top - 1)); + L->top--; + lua_unlock(L); + return res; +} + + +/* +** `load' and `call' functions (run Lua code) +*/ + + +#define adjustresults(L,nres) \ + { if (nres == LUA_MULTRET && L->top >= L->ci->top) L->ci->top = L->top; } + + +#define checkresults(L,na,nr) \ + api_check(L, (nr) == LUA_MULTRET || (L->ci->top - L->top >= (nr) - (na))) + + +LUA_API void lua_call (lua_State *L, int nargs, int nresults) { + StkId func; + lua_lock(L); + api_checknelems(L, nargs+1); + checkresults(L, nargs, nresults); + func = L->top - (nargs+1); + luaD_call(L, func, nresults); + adjustresults(L, nresults); + lua_unlock(L); +} + + + +/* +** Execute a protected call. +*/ +struct CallS { /* data to `f_call' */ + StkId func; + int nresults; +}; + + +static void f_call (lua_State *L, void *ud) { + struct CallS *c = cast(struct CallS *, ud); + luaD_call(L, c->func, c->nresults); +} + + + +LUA_API int lua_pcall (lua_State *L, int nargs, int nresults, int errfunc) { + struct CallS c; + int status; + ptrdiff_t func; + lua_lock(L); + api_checknelems(L, nargs+1); + checkresults(L, nargs, nresults); + if (errfunc == 0) + func = 0; + else { + StkId o = index2adr(L, errfunc); + api_checkvalidindex(L, o); + func = savestack(L, o); + } + c.func = L->top - (nargs+1); /* function to be called */ + c.nresults = nresults; + status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func); + adjustresults(L, nresults); + lua_unlock(L); + return status; +} + + +/* +** Execute a protected C call. +*/ +struct CCallS { /* data to `f_Ccall' */ + lua_CFunction func; + void *ud; +}; + + +static void f_Ccall (lua_State *L, void *ud) { + struct CCallS *c = cast(struct CCallS *, ud); + Closure *cl; + cl = luaF_newCclosure(L, 0, getcurrenv(L)); + cl->c.f = c->func; + setclvalue(L, L->top, cl); /* push function */ + api_incr_top(L); + setpvalue(L->top, c->ud); /* push only argument */ + api_incr_top(L); + luaD_call(L, L->top - 2, 0); +} + + +LUA_API int lua_cpcall (lua_State *L, lua_CFunction func, void *ud) { + struct CCallS c; + int status; + lua_lock(L); + c.func = func; + c.ud = ud; + status = luaD_pcall(L, f_Ccall, &c, savestack(L, L->top), 0); + lua_unlock(L); + return status; +} + + +LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data, + const char *chunkname) { + ZIO z; + int status; + lua_lock(L); + if (!chunkname) chunkname = "?"; + luaZ_init(L, &z, reader, data); + status = luaD_protectedparser(L, &z, chunkname); + lua_unlock(L); + return status; +} + + +LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data) { + int status; + TValue *o; + lua_lock(L); + api_checknelems(L, 1); + o = L->top - 1; + if (isLfunction(o)) + status = luaU_dump(L, clvalue(o)->l.p, writer, data, 0); + else + status = 1; + lua_unlock(L); + return status; +} + + +LUA_API int lua_status (lua_State *L) { + return L->status; +} + + +/* +** Garbage-collection function +*/ + +LUA_API int lua_gc (lua_State *L, int what, int data) { + int res = 0; + global_State *g; + lua_lock(L); + g = G(L); + switch (what) { + case LUA_GCSTOP: { + g->GCthreshold = MAX_LUMEM; + break; + } + case LUA_GCRESTART: { + g->GCthreshold = g->totalbytes; + break; + } + case LUA_GCCOLLECT: { + luaC_fullgc(L); + break; + } + case LUA_GCCOUNT: { + /* GC values are expressed in Kbytes: #bytes/2^10 */ + res = cast_int(g->totalbytes >> 10); + break; + } + case LUA_GCCOUNTB: { + res = cast_int(g->totalbytes & 0x3ff); + break; + } + case LUA_GCSTEP: { + lu_mem a = (cast(lu_mem, data) << 10); + if (a <= g->totalbytes) + g->GCthreshold = g->totalbytes - a; + else + g->GCthreshold = 0; + while (g->GCthreshold <= g->totalbytes) { + luaC_step(L); + if (g->gcstate == GCSpause) { /* end of cycle? */ + res = 1; /* signal it */ + break; + } + } + break; + } + case LUA_GCSETPAUSE: { + res = g->gcpause; + g->gcpause = data; + break; + } + case LUA_GCSETSTEPMUL: { + res = g->gcstepmul; + g->gcstepmul = data; + break; + } + default: res = -1; /* invalid option */ + } + lua_unlock(L); + return res; +} + + + +/* +** miscellaneous functions +*/ + + +LUA_API int lua_error (lua_State *L) { + lua_lock(L); + api_checknelems(L, 1); + luaG_errormsg(L); + lua_unlock(L); + return 0; /* to avoid warnings */ +} + + +LUA_API int lua_next (lua_State *L, int idx) { + StkId t; + int more; + lua_lock(L); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + more = luaH_next(L, hvalue(t), L->top - 1); + if (more) { + api_incr_top(L); + } + else /* no more elements */ + L->top -= 1; /* remove key */ + lua_unlock(L); + return more; +} + + +LUA_API void lua_concat (lua_State *L, int n) { + lua_lock(L); + api_checknelems(L, n); + if (n >= 2) { + luaC_checkGC(L); + luaV_concat(L, n, cast_int(L->top - L->base) - 1); + L->top -= (n-1); + } + else if (n == 0) { /* push empty string */ + setsvalue2s(L, L->top, luaS_newlstr(L, "", 0)); + api_incr_top(L); + } + /* else n == 1; nothing to do */ + lua_unlock(L); +} + + +LUA_API lua_Alloc lua_getallocf (lua_State *L, void **ud) { + lua_Alloc f; + lua_lock(L); + if (ud) *ud = G(L)->ud; + f = G(L)->frealloc; + lua_unlock(L); + return f; +} + + +LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud) { + lua_lock(L); + G(L)->ud = ud; + G(L)->frealloc = f; + lua_unlock(L); +} + + +LUA_API void *lua_newuserdata (lua_State *L, size_t size) { + Udata *u; + lua_lock(L); + luaC_checkGC(L); + u = luaS_newudata(L, size, getcurrenv(L)); + setuvalue(L, L->top, u); + api_incr_top(L); + lua_unlock(L); + return u + 1; +} + + + + +static const char *aux_upvalue (StkId fi, int n, TValue **val) { + Closure *f; + if (!ttisfunction(fi)) return NULL; + f = clvalue(fi); + if (f->c.isC) { + if (!(1 <= n && n <= f->c.nupvalues)) return NULL; + *val = &f->c.upvalue[n-1]; + return ""; + } + else { + Proto *p = f->l.p; + if (!(1 <= n && n <= p->sizeupvalues)) return NULL; + *val = f->l.upvals[n-1]->v; + return getstr(p->upvalues[n-1]); + } +} + + +LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n) { + const char *name; + TValue *val; + lua_lock(L); + name = aux_upvalue(index2adr(L, funcindex), n, &val); + if (name) { + setobj2s(L, L->top, val); + api_incr_top(L); + } + lua_unlock(L); + return name; +} + + +LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n) { + const char *name; + TValue *val; + StkId fi; + lua_lock(L); + fi = index2adr(L, funcindex); + api_checknelems(L, 1); + name = aux_upvalue(fi, n, &val); + if (name) { + L->top--; + setobj(L, val, L->top); + luaC_barrier(L, clvalue(fi), L->top); + } + lua_unlock(L); + return name; +} + diff --git a/src/lua/lapi.h b/src/lua/lapi.h new file mode 100644 index 0000000..2c3fab2 --- /dev/null +++ b/src/lua/lapi.h @@ -0,0 +1,16 @@ +/* +** $Id: lapi.h,v 2.2.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions from Lua API +** See Copyright Notice in lua.h +*/ + +#ifndef lapi_h +#define lapi_h + + +#include "lobject.h" + + +LUAI_FUNC void luaA_pushobject (lua_State *L, const TValue *o); + +#endif diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c new file mode 100644 index 0000000..a1ca299 --- /dev/null +++ b/src/lua/lauxlib.c @@ -0,0 +1,653 @@ +/* +** $Id: lauxlib.c,v 1.159.1.3 2008/01/21 13:20:51 roberto Exp $ +** Auxiliary functions for building Lua libraries +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include +#include +#include + + +/* This file uses only the official API of Lua. +** Any function declared here could be written as an application function. +*/ + +#define lauxlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" + + +#define FREELIST_REF 0 /* free list of references */ + + +/* convert a stack index to positive */ +#define abs_index(L, i) ((i) > 0 || (i) <= LUA_REGISTRYINDEX ? (i) : \ + lua_gettop(L) + (i) + 1) + + +/* +** {====================================================== +** Error-report functions +** ======================================================= +*/ + + +LUALIB_API int luaL_argerror (lua_State *L, int narg, const char *extramsg) { + lua_Debug ar; + if (!lua_getstack(L, 0, &ar)) /* no stack frame? */ + return luaL_error(L, "bad argument #%d (%s)", narg, extramsg); + lua_getinfo(L, "n", &ar); + if (strcmp(ar.namewhat, "method") == 0) { + narg--; /* do not count `self' */ + if (narg == 0) /* error is in the self argument itself? */ + return luaL_error(L, "calling " LUA_QS " on bad self (%s)", + ar.name, extramsg); + } + if (ar.name == NULL) + ar.name = "?"; + return luaL_error(L, "bad argument #%d to " LUA_QS " (%s)", + narg, ar.name, extramsg); +} + + +LUALIB_API int luaL_typerror (lua_State *L, int narg, const char *tname) { + const char *msg = lua_pushfstring(L, "%s expected, got %s", + tname, luaL_typename(L, narg)); + return luaL_argerror(L, narg, msg); +} + + +static void tag_error (lua_State *L, int narg, int tag) { + luaL_typerror(L, narg, lua_typename(L, tag)); +} + + +LUALIB_API void luaL_where (lua_State *L, int level) { + lua_Debug ar; + if (lua_getstack(L, level, &ar)) { /* check function at level */ + lua_getinfo(L, "Sl", &ar); /* get info about it */ + if (ar.currentline > 0) { /* is there info? */ + lua_pushfstring(L, "%s:%d: ", ar.short_src, ar.currentline); + return; + } + } + lua_pushliteral(L, ""); /* else, no information available... */ +} + + +LUALIB_API int luaL_error (lua_State *L, const char *fmt, ...) { + va_list argp; + va_start(argp, fmt); + luaL_where(L, 1); + lua_pushvfstring(L, fmt, argp); + va_end(argp); + lua_concat(L, 2); + return lua_error(L); +} + +/* }====================================================== */ + + +LUALIB_API int luaL_checkoption (lua_State *L, int narg, const char *def, + const char *const lst[]) { + const char *name = (def) ? luaL_optstring(L, narg, def) : + luaL_checkstring(L, narg); + int i; + for (i=0; lst[i]; i++) + if (strcmp(lst[i], name) == 0) + return i; + return luaL_argerror(L, narg, + lua_pushfstring(L, "invalid option " LUA_QS, name)); +} + + +LUALIB_API int luaL_newmetatable (lua_State *L, const char *tname) { + lua_getfield(L, LUA_REGISTRYINDEX, tname); /* get registry.name */ + if (!lua_isnil(L, -1)) /* name already in use? */ + return 0; /* leave previous value on top, but return 0 */ + lua_pop(L, 1); + lua_newtable(L); /* create metatable */ + lua_pushvalue(L, -1); + lua_setfield(L, LUA_REGISTRYINDEX, tname); /* registry.name = metatable */ + return 1; +} + + +LUALIB_API void *luaL_checkudata (lua_State *L, int ud, const char *tname) { + void *p = lua_touserdata(L, ud); + if (p != NULL) { /* value is a userdata? */ + if (lua_getmetatable(L, ud)) { /* does it have a metatable? */ + lua_getfield(L, LUA_REGISTRYINDEX, tname); /* get correct metatable */ + if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */ + lua_pop(L, 2); /* remove both metatables */ + return p; + } + } + } + luaL_typerror(L, ud, tname); /* else error */ + return NULL; /* to avoid warnings */ +} + + +LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *mes) { + if (!lua_checkstack(L, space)) + luaL_error(L, "stack overflow (%s)", mes); +} + + +LUALIB_API void luaL_checktype (lua_State *L, int narg, int t) { + if (lua_type(L, narg) != t) + tag_error(L, narg, t); +} + + +LUALIB_API void luaL_checkany (lua_State *L, int narg) { + if (lua_type(L, narg) == LUA_TNONE) + luaL_argerror(L, narg, "value expected"); +} + + +LUALIB_API const char *luaL_checklstring (lua_State *L, int narg, size_t *len) { + const char *s = lua_tolstring(L, narg, len); + if (!s) tag_error(L, narg, LUA_TSTRING); + return s; +} + + +LUALIB_API const char *luaL_optlstring (lua_State *L, int narg, + const char *def, size_t *len) { + if (lua_isnoneornil(L, narg)) { + if (len) + *len = (def ? strlen(def) : 0); + return def; + } + else return luaL_checklstring(L, narg, len); +} + + +LUALIB_API lua_Number luaL_checknumber (lua_State *L, int narg) { + lua_Number d = lua_tonumber(L, narg); + if (d == 0 && !lua_isnumber(L, narg)) /* avoid extra test when d is not 0 */ + tag_error(L, narg, LUA_TNUMBER); + return d; +} + + +LUALIB_API lua_Number luaL_optnumber (lua_State *L, int narg, lua_Number def) { + return luaL_opt(L, luaL_checknumber, narg, def); +} + + +LUALIB_API lua_Integer luaL_checkinteger (lua_State *L, int narg) { + lua_Integer d = lua_tointeger(L, narg); + if (d == 0 && !lua_isnumber(L, narg)) /* avoid extra test when d is not 0 */ + tag_error(L, narg, LUA_TNUMBER); + return d; +} + + +LUALIB_API lua_Integer luaL_optinteger (lua_State *L, int narg, + lua_Integer def) { + return luaL_opt(L, luaL_checkinteger, narg, def); +} + + +LUALIB_API int luaL_getmetafield (lua_State *L, int obj, const char *event) { + if (!lua_getmetatable(L, obj)) /* no metatable? */ + return 0; + lua_pushstring(L, event); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 2); /* remove metatable and metafield */ + return 0; + } + else { + lua_remove(L, -2); /* remove only metatable */ + return 1; + } +} + + +LUALIB_API int luaL_callmeta (lua_State *L, int obj, const char *event) { + obj = abs_index(L, obj); + if (!luaL_getmetafield(L, obj, event)) /* no metafield? */ + return 0; + lua_pushvalue(L, obj); + lua_call(L, 1, 1); + return 1; +} + + +LUALIB_API void (luaL_register) (lua_State *L, const char *libname, + const luaL_Reg *l) { + luaI_openlib(L, libname, l, 0); +} + + +static int libsize (const luaL_Reg *l) { + int size = 0; + for (; l->name; l++) size++; + return size; +} + + +LUALIB_API void luaI_openlib (lua_State *L, const char *libname, + const luaL_Reg *l, int nup) { + if (libname) { + int size = libsize(l); + /* check whether lib already exists */ + luaL_findtable(L, LUA_REGISTRYINDEX, "_LOADED", 1); + lua_getfield(L, -1, libname); /* get _LOADED[libname] */ + if (!lua_istable(L, -1)) { /* not found? */ + lua_pop(L, 1); /* remove previous result */ + /* try global variable (and create one if it does not exist) */ + if (luaL_findtable(L, LUA_GLOBALSINDEX, libname, size) != NULL) + luaL_error(L, "name conflict for module " LUA_QS, libname); + lua_pushvalue(L, -1); + lua_setfield(L, -3, libname); /* _LOADED[libname] = new table */ + } + lua_remove(L, -2); /* remove _LOADED table */ + lua_insert(L, -(nup+1)); /* move library table to below upvalues */ + } + for (; l->name; l++) { + int i; + for (i=0; ifunc, nup); + lua_setfield(L, -(nup+2), l->name); + } + lua_pop(L, nup); /* remove upvalues */ +} + + + +/* +** {====================================================== +** getn-setn: size for arrays +** ======================================================= +*/ + +#if defined(LUA_COMPAT_GETN) + +static int checkint (lua_State *L, int topop) { + int n = (lua_type(L, -1) == LUA_TNUMBER) ? lua_tointeger(L, -1) : -1; + lua_pop(L, topop); + return n; +} + + +static void getsizes (lua_State *L) { + lua_getfield(L, LUA_REGISTRYINDEX, "LUA_SIZES"); + if (lua_isnil(L, -1)) { /* no `size' table? */ + lua_pop(L, 1); /* remove nil */ + lua_newtable(L); /* create it */ + lua_pushvalue(L, -1); /* `size' will be its own metatable */ + lua_setmetatable(L, -2); + lua_pushliteral(L, "kv"); + lua_setfield(L, -2, "__mode"); /* metatable(N).__mode = "kv" */ + lua_pushvalue(L, -1); + lua_setfield(L, LUA_REGISTRYINDEX, "LUA_SIZES"); /* store in register */ + } +} + + +LUALIB_API void luaL_setn (lua_State *L, int t, int n) { + t = abs_index(L, t); + lua_pushliteral(L, "n"); + lua_rawget(L, t); + if (checkint(L, 1) >= 0) { /* is there a numeric field `n'? */ + lua_pushliteral(L, "n"); /* use it */ + lua_pushinteger(L, n); + lua_rawset(L, t); + } + else { /* use `sizes' */ + getsizes(L); + lua_pushvalue(L, t); + lua_pushinteger(L, n); + lua_rawset(L, -3); /* sizes[t] = n */ + lua_pop(L, 1); /* remove `sizes' */ + } +} + + +LUALIB_API int luaL_getn (lua_State *L, int t) { + int n; + t = abs_index(L, t); + lua_pushliteral(L, "n"); /* try t.n */ + lua_rawget(L, t); + if ((n = checkint(L, 1)) >= 0) return n; + getsizes(L); /* else try sizes[t] */ + lua_pushvalue(L, t); + lua_rawget(L, -2); + if ((n = checkint(L, 2)) >= 0) return n; + return (int)lua_objlen(L, t); +} + +#endif + +/* }====================================================== */ + + + +LUALIB_API const char *luaL_gsub (lua_State *L, const char *s, const char *p, + const char *r) { + const char *wild; + size_t l = strlen(p); + luaL_Buffer b; + luaL_buffinit(L, &b); + while ((wild = strstr(s, p)) != NULL) { + luaL_addlstring(&b, s, wild - s); /* push prefix */ + luaL_addstring(&b, r); /* push replacement in place of pattern */ + s = wild + l; /* continue after `p' */ + } + luaL_addstring(&b, s); /* push last suffix */ + luaL_pushresult(&b); + return lua_tostring(L, -1); +} + + +LUALIB_API const char *luaL_findtable (lua_State *L, int idx, + const char *fname, int szhint) { + const char *e; + lua_pushvalue(L, idx); + do { + e = strchr(fname, '.'); + if (e == NULL) e = fname + strlen(fname); + lua_pushlstring(L, fname, e - fname); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { /* no such field? */ + lua_pop(L, 1); /* remove this nil */ + lua_createtable(L, 0, (*e == '.' ? 1 : szhint)); /* new table for field */ + lua_pushlstring(L, fname, e - fname); + lua_pushvalue(L, -2); + lua_settable(L, -4); /* set new table into field */ + } + else if (!lua_istable(L, -1)) { /* field has a non-table value? */ + lua_pop(L, 2); /* remove table and value */ + return fname; /* return problematic part of the name */ + } + lua_remove(L, -2); /* remove previous table */ + fname = e + 1; + } while (*e == '.'); + return NULL; +} + + + +/* +** {====================================================== +** Generic Buffer manipulation +** ======================================================= +*/ + + +#define bufflen(B) ((B)->p - (B)->buffer) +#define bufffree(B) ((size_t)(LUAL_BUFFERSIZE - bufflen(B))) + +#define LIMIT (LUA_MINSTACK/2) + + +static int emptybuffer (luaL_Buffer *B) { + size_t l = bufflen(B); + if (l == 0) return 0; /* put nothing on stack */ + else { + lua_pushlstring(B->L, B->buffer, l); + B->p = B->buffer; + B->lvl++; + return 1; + } +} + + +static void adjuststack (luaL_Buffer *B) { + if (B->lvl > 1) { + lua_State *L = B->L; + int toget = 1; /* number of levels to concat */ + size_t toplen = lua_strlen(L, -1); + do { + size_t l = lua_strlen(L, -(toget+1)); + if (B->lvl - toget + 1 >= LIMIT || toplen > l) { + toplen += l; + toget++; + } + else break; + } while (toget < B->lvl); + lua_concat(L, toget); + B->lvl = B->lvl - toget + 1; + } +} + + +LUALIB_API char *luaL_prepbuffer (luaL_Buffer *B) { + if (emptybuffer(B)) + adjuststack(B); + return B->buffer; +} + + +LUALIB_API void luaL_addlstring (luaL_Buffer *B, const char *s, size_t l) { + while (l--) + luaL_addchar(B, *s++); +} + + +LUALIB_API void luaL_addstring (luaL_Buffer *B, const char *s) { + luaL_addlstring(B, s, strlen(s)); +} + + +LUALIB_API void luaL_pushresult (luaL_Buffer *B) { + emptybuffer(B); + lua_concat(B->L, B->lvl); + B->lvl = 1; +} + + +LUALIB_API void luaL_addvalue (luaL_Buffer *B) { + lua_State *L = B->L; + size_t vl; + const char *s = lua_tolstring(L, -1, &vl); + if (vl <= bufffree(B)) { /* fit into buffer? */ + memcpy(B->p, s, vl); /* put it there */ + B->p += vl; + lua_pop(L, 1); /* remove from stack */ + } + else { + if (emptybuffer(B)) + lua_insert(L, -2); /* put buffer before new value */ + B->lvl++; /* add new value into B stack */ + adjuststack(B); + } +} + + +LUALIB_API void luaL_buffinit (lua_State *L, luaL_Buffer *B) { + B->L = L; + B->p = B->buffer; + B->lvl = 0; +} + +/* }====================================================== */ + + +LUALIB_API int luaL_ref (lua_State *L, int t) { + int ref; + t = abs_index(L, t); + if (lua_isnil(L, -1)) { + lua_pop(L, 1); /* remove from stack */ + return LUA_REFNIL; /* `nil' has a unique fixed reference */ + } + lua_rawgeti(L, t, FREELIST_REF); /* get first free element */ + ref = (int)lua_tointeger(L, -1); /* ref = t[FREELIST_REF] */ + lua_pop(L, 1); /* remove it from stack */ + if (ref != 0) { /* any free element? */ + lua_rawgeti(L, t, ref); /* remove it from list */ + lua_rawseti(L, t, FREELIST_REF); /* (t[FREELIST_REF] = t[ref]) */ + } + else { /* no free elements */ + ref = (int)lua_objlen(L, t); + ref++; /* create new reference */ + } + lua_rawseti(L, t, ref); + return ref; +} + + +LUALIB_API void luaL_unref (lua_State *L, int t, int ref) { + if (ref >= 0) { + t = abs_index(L, t); + lua_rawgeti(L, t, FREELIST_REF); + lua_rawseti(L, t, ref); /* t[ref] = t[FREELIST_REF] */ + lua_pushinteger(L, ref); + lua_rawseti(L, t, FREELIST_REF); /* t[FREELIST_REF] = ref */ + } +} + + + +/* +** {====================================================== +** Load functions +** ======================================================= +*/ + +typedef struct LoadF { + int extraline; + FILE *f; + char buff[LUAL_BUFFERSIZE]; +} LoadF; + + +static const char *getF (lua_State *L, void *ud, size_t *size) { + LoadF *lf = (LoadF *)ud; + (void)L; + if (lf->extraline) { + lf->extraline = 0; + *size = 1; + return "\n"; + } + if (feof(lf->f)) return NULL; + *size = fread(lf->buff, 1, sizeof(lf->buff), lf->f); + return (*size > 0) ? lf->buff : NULL; +} + + +static int errfile (lua_State *L, const char *what, int fnameindex) { + const char *serr = strerror(errno); + const char *filename = lua_tostring(L, fnameindex) + 1; + lua_pushfstring(L, "cannot %s %s: %s", what, filename, serr); + lua_remove(L, fnameindex); + return LUA_ERRFILE; +} + + +LUALIB_API int luaL_loadfile (lua_State *L, const char *filename) { + LoadF lf; + int status, readstatus; + int c; + int fnameindex = lua_gettop(L) + 1; /* index of filename on the stack */ + lf.extraline = 0; + if (filename == NULL) { + lua_pushliteral(L, "=stdin"); + lf.f = stdin; + } + else { + lua_pushfstring(L, "@%s", filename); + lf.f = fopen(filename, "r"); + if (lf.f == NULL) return errfile(L, "open", fnameindex); + } + c = getc(lf.f); + if (c == '#') { /* Unix exec. file? */ + lf.extraline = 1; + while ((c = getc(lf.f)) != EOF && c != '\n') ; /* skip first line */ + if (c == '\n') c = getc(lf.f); + } + if (c == LUA_SIGNATURE[0] && filename) { /* binary file? */ + lf.f = freopen(filename, "rb", lf.f); /* reopen in binary mode */ + if (lf.f == NULL) return errfile(L, "reopen", fnameindex); + /* skip eventual `#!...' */ + while ((c = getc(lf.f)) != EOF && c != LUA_SIGNATURE[0]) + ; + lf.extraline = 0; + } + ungetc(c, lf.f); + status = lua_load(L, getF, &lf, lua_tostring(L, -1)); + readstatus = ferror(lf.f); + if (filename) fclose(lf.f); /* close file (even in case of errors) */ + if (readstatus) { + lua_settop(L, fnameindex); /* ignore results from `lua_load' */ + return errfile(L, "read", fnameindex); + } + lua_remove(L, fnameindex); + return status; +} + + +typedef struct LoadS { + const char *s; + size_t size; +} LoadS; + + +static const char *getS (lua_State *L, void *ud, size_t *size) { + LoadS *ls = (LoadS *)ud; + (void)L; + if (ls->size == 0) return NULL; + *size = ls->size; + ls->size = 0; + return ls->s; +} + + +LUALIB_API int luaL_loadbuffer (lua_State *L, const char *buff, size_t size, + const char *name) { + LoadS ls; + ls.s = buff; + ls.size = size; + return lua_load(L, getS, &ls, name); +} + + +LUALIB_API int (luaL_loadstring) (lua_State *L, const char *s) { + return luaL_loadbuffer(L, s, strlen(s), s); +} + + + +/* }====================================================== */ + + +static void *l_alloc (void *ud, void *ptr, size_t osize, size_t nsize) { + (void)ud; + (void)osize; + if (nsize == 0) { + free(ptr); + return NULL; + } + else + return realloc(ptr, nsize); +} + + +static int panic (lua_State *L) { + (void)L; /* to avoid warnings */ + fprintf(stderr, "PANIC: unprotected error in call to Lua API (%s)\n", + lua_tostring(L, -1)); + return 0; +} + + +LUALIB_API lua_State *luaL_newstate (void) { + lua_State *L = lua_newstate(l_alloc, NULL); + if (L) lua_atpanic(L, &panic); + return L; +} + diff --git a/src/lua/lbaselib.c b/src/lua/lbaselib.c new file mode 100644 index 0000000..8552405 --- /dev/null +++ b/src/lua/lbaselib.c @@ -0,0 +1,657 @@ +/* +** $Id: lbaselib.c,v 1.191.1.6 2008/02/14 16:46:22 roberto Exp $ +** Basic library +** See Copyright Notice in lua.h +*/ + + + +#include +#include +#include +#include + +#define lbaselib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + + + +/* +** If your system does not support `stdout', you can just remove this function. +** If you need, you can define your own `print' function, following this +** model but changing `fputs' to put the strings at a proper place +** (a console window or a log file, for instance). +*/ +static int luaB_print (lua_State *L) { + int n = lua_gettop(L); /* number of arguments */ + int i; + lua_getglobal(L, "tostring"); + for (i=1; i<=n; i++) { + const char *s; + lua_pushvalue(L, -1); /* function to be called */ + lua_pushvalue(L, i); /* value to print */ + lua_call(L, 1, 1); + s = lua_tostring(L, -1); /* get result */ + if (s == NULL) + return luaL_error(L, LUA_QL("tostring") " must return a string to " + LUA_QL("print")); + if (i>1) fputs("\t", stdout); + fputs(s, stdout); + lua_pop(L, 1); /* pop result */ + } + fputs("\n", stdout); + return 0; +} + + +static int luaB_tonumber (lua_State *L) { + int base = luaL_optint(L, 2, 10); + if (base == 10) { /* standard conversion */ + luaL_checkany(L, 1); + if (lua_isnumber(L, 1)) { + lua_pushnumber(L, lua_tonumber(L, 1)); + return 1; + } + } + else { + const char *s1 = luaL_checkstring(L, 1); + char *s2; + unsigned long n; + luaL_argcheck(L, 2 <= base && base <= 36, 2, "base out of range"); + n = strtoul(s1, &s2, base); + if (s1 != s2) { /* at least one valid digit? */ + while (isspace((unsigned char)(*s2))) s2++; /* skip trailing spaces */ + if (*s2 == '\0') { /* no invalid trailing characters? */ + lua_pushnumber(L, (lua_Number)n); + return 1; + } + } + } + lua_pushnil(L); /* else not a number */ + return 1; +} + + +static int luaB_error (lua_State *L) { + int level = luaL_optint(L, 2, 1); + lua_settop(L, 1); + if (lua_isstring(L, 1) && level > 0) { /* add extra information? */ + luaL_where(L, level); + lua_pushvalue(L, 1); + lua_concat(L, 2); + } + return lua_error(L); +} + + +static int luaB_getmetatable (lua_State *L) { + luaL_checkany(L, 1); + if (!lua_getmetatable(L, 1)) { + lua_pushnil(L); + return 1; /* no metatable */ + } + luaL_getmetafield(L, 1, "__metatable"); + return 1; /* returns either __metatable field (if present) or metatable */ +} + + +static int luaB_setmetatable (lua_State *L) { + int t = lua_type(L, 2); + luaL_checktype(L, 1, LUA_TTABLE); + luaL_argcheck(L, t == LUA_TNIL || t == LUA_TTABLE, 2, + "nil or table expected"); + if (luaL_getmetafield(L, 1, "__metatable")) + luaL_error(L, "cannot change a protected metatable"); + lua_settop(L, 2); + lua_setmetatable(L, 1); + return 1; +} + + +static void getfunc (lua_State *L, int opt) { + if (lua_isfunction(L, 1)) lua_pushvalue(L, 1); + else { + lua_Debug ar; + int level = opt ? luaL_optint(L, 1, 1) : luaL_checkint(L, 1); + luaL_argcheck(L, level >= 0, 1, "level must be non-negative"); + if (lua_getstack(L, level, &ar) == 0) + luaL_argerror(L, 1, "invalid level"); + lua_getinfo(L, "f", &ar); + if (lua_isnil(L, -1)) + luaL_error(L, "no function environment for tail call at level %d", + level); + } +} + + +static int luaB_getfenv (lua_State *L) { + getfunc(L, 1); + if (lua_iscfunction(L, -1)) /* is a C function? */ + lua_pushvalue(L, LUA_GLOBALSINDEX); /* return the thread's global env. */ + else + lua_getfenv(L, -1); + return 1; +} + + +static int luaB_setfenv (lua_State *L) { + luaL_checktype(L, 2, LUA_TTABLE); + getfunc(L, 0); + lua_pushvalue(L, 2); + if (lua_isnumber(L, 1) && lua_tonumber(L, 1) == 0) { + /* change environment of current thread */ + lua_pushthread(L); + lua_insert(L, -2); + lua_setfenv(L, -2); + return 0; + } + else if (lua_iscfunction(L, -2) || lua_setfenv(L, -2) == 0) + luaL_error(L, + LUA_QL("setfenv") " cannot change environment of given object"); + return 1; +} + + +static int luaB_rawequal (lua_State *L) { + luaL_checkany(L, 1); + luaL_checkany(L, 2); + lua_pushboolean(L, lua_rawequal(L, 1, 2)); + return 1; +} + + +static int luaB_rawget (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + luaL_checkany(L, 2); + lua_settop(L, 2); + lua_rawget(L, 1); + return 1; +} + +static int luaB_rawset (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + luaL_checkany(L, 2); + luaL_checkany(L, 3); + lua_settop(L, 3); + lua_rawset(L, 1); + return 1; +} + + +static int luaB_gcinfo (lua_State *L) { + lua_pushinteger(L, lua_getgccount(L)); + return 1; +} + + +static int luaB_collectgarbage (lua_State *L) { + static const char *const opts[] = {"stop", "restart", "collect", + "count", "step", "setpause", "setstepmul", NULL}; + static const int optsnum[] = {LUA_GCSTOP, LUA_GCRESTART, LUA_GCCOLLECT, + LUA_GCCOUNT, LUA_GCSTEP, LUA_GCSETPAUSE, LUA_GCSETSTEPMUL}; + int o = luaL_checkoption(L, 1, "collect", opts); + int ex = luaL_optint(L, 2, 0); + int res = lua_gc(L, optsnum[o], ex); + switch (optsnum[o]) { + case LUA_GCCOUNT: { + int b = lua_gc(L, LUA_GCCOUNTB, 0); + lua_pushnumber(L, res + ((lua_Number)b/1024)); + return 1; + } + case LUA_GCSTEP: { + lua_pushboolean(L, res); + return 1; + } + default: { + lua_pushnumber(L, res); + return 1; + } + } +} + + +static int luaB_type (lua_State *L) { + luaL_checkany(L, 1); + lua_pushstring(L, luaL_typename(L, 1)); + return 1; +} + + +static int luaB_next (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_settop(L, 2); /* create a 2nd argument if there isn't one */ + if (lua_next(L, 1)) + return 2; + else { + lua_pushnil(L); + return 1; + } +} + + +static int luaB_pairs (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_pushvalue(L, lua_upvalueindex(1)); /* return generator, */ + lua_pushvalue(L, 1); /* state, */ + lua_pushnil(L); /* and initial value */ + return 3; +} + + +static int ipairsaux (lua_State *L) { + int i = luaL_checkint(L, 2); + luaL_checktype(L, 1, LUA_TTABLE); + i++; /* next value */ + lua_pushinteger(L, i); + lua_rawgeti(L, 1, i); + return (lua_isnil(L, -1)) ? 0 : 2; +} + + +static int luaB_ipairs (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_pushvalue(L, lua_upvalueindex(1)); /* return generator, */ + lua_pushvalue(L, 1); /* state, */ + lua_pushinteger(L, 0); /* and initial value */ + return 3; +} + + +static int load_aux (lua_State *L, int status) { + if (status == 0) /* OK? */ + return 1; + else { + lua_pushnil(L); + lua_insert(L, -2); /* put before error message */ + return 2; /* return nil plus error message */ + } +} + + +static int luaB_loadstring (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + const char *chunkname = luaL_optstring(L, 2, s); + return load_aux(L, luaL_loadbuffer(L, s, l, chunkname)); +} + + +static int luaB_loadfile (lua_State *L) { + const char *fname = luaL_optstring(L, 1, NULL); + return load_aux(L, luaL_loadfile(L, fname)); +} + + +/* +** Reader for generic `load' function: `lua_load' uses the +** stack for internal stuff, so the reader cannot change the +** stack top. Instead, it keeps its resulting string in a +** reserved slot inside the stack. +*/ +static const char *generic_reader (lua_State *L, void *ud, size_t *size) { + (void)ud; /* to avoid warnings */ + luaL_checkstack(L, 2, "too many nested functions"); + lua_pushvalue(L, 1); /* get function */ + lua_call(L, 0, 1); /* call it */ + if (lua_isnil(L, -1)) { + *size = 0; + return NULL; + } + else if (lua_isstring(L, -1)) { + lua_replace(L, 3); /* save string in a reserved stack slot */ + return lua_tolstring(L, 3, size); + } + else luaL_error(L, "reader function must return a string"); + return NULL; /* to avoid warnings */ +} + + +static int luaB_load (lua_State *L) { + int status; + const char *cname = luaL_optstring(L, 2, "=(load)"); + luaL_checktype(L, 1, LUA_TFUNCTION); + lua_settop(L, 3); /* function, eventual name, plus one reserved slot */ + status = lua_load(L, generic_reader, NULL, cname); + return load_aux(L, status); +} + + +static int luaB_dofile (lua_State *L) { + const char *fname = luaL_optstring(L, 1, NULL); + int n = lua_gettop(L); + if (luaL_loadfile(L, fname) != 0) lua_error(L); + lua_call(L, 0, LUA_MULTRET); + return lua_gettop(L) - n; +} + + +static int luaB_assert (lua_State *L) { + luaL_checkany(L, 1); + if (!lua_toboolean(L, 1)) + return luaL_error(L, "%s", luaL_optstring(L, 2, "assertion failed!")); + return lua_gettop(L); +} + + +static int luaB_unpack (lua_State *L) { + int i, e, n; + luaL_checktype(L, 1, LUA_TTABLE); + i = luaL_optint(L, 2, 1); + e = luaL_opt(L, luaL_checkint, 3, luaL_getn(L, 1)); + if (i > e) return 0; /* empty range */ + n = e - i + 1; /* number of elements */ + if (n <= 0 || !lua_checkstack(L, n)) /* n <= 0 means arith. overflow */ + return luaL_error(L, "too many results to unpack"); + lua_rawgeti(L, 1, i); /* push arg[i] (avoiding overflow problems) */ + while (i++ < e) /* push arg[i + 1...e] */ + lua_rawgeti(L, 1, i); + return n; +} + + +static int luaB_select (lua_State *L) { + int n = lua_gettop(L); + if (lua_type(L, 1) == LUA_TSTRING && *lua_tostring(L, 1) == '#') { + lua_pushinteger(L, n-1); + return 1; + } + else { + int i = luaL_checkint(L, 1); + if (i < 0) i = n + i; + else if (i > n) i = n; + luaL_argcheck(L, 1 <= i, 1, "index out of range"); + return n - i; + } +} + + +static int luaB_pcall (lua_State *L) { + int status; + luaL_checkany(L, 1); + status = lua_pcall(L, lua_gettop(L) - 1, LUA_MULTRET, 0); + lua_pushboolean(L, (status == 0)); + lua_insert(L, 1); + return lua_gettop(L); /* return status + all results */ +} + + +static int luaB_xpcall (lua_State *L) { + int status; + luaL_checkany(L, 2); + lua_settop(L, 2); + lua_insert(L, 1); /* put error function under function to be called */ + status = lua_pcall(L, 0, LUA_MULTRET, 1); + lua_pushboolean(L, (status == 0)); + lua_replace(L, 1); + return lua_gettop(L); /* return status + all results */ +} + + +static int luaB_tostring (lua_State *L) { + luaL_checkany(L, 1); + if (luaL_callmeta(L, 1, "__tostring")) /* is there a metafield? */ + return 1; /* use its value */ + switch (lua_type(L, 1)) { + case LUA_TNUMBER: + lua_pushstring(L, lua_tostring(L, 1)); + break; + case LUA_TSTRING: + lua_pushvalue(L, 1); + break; + case LUA_TBOOLEAN: + lua_pushstring(L, (lua_toboolean(L, 1) ? "true" : "false")); + break; + case LUA_TNIL: + lua_pushliteral(L, "nil"); + break; + default: + lua_pushfstring(L, "%s: %p", luaL_typename(L, 1), lua_topointer(L, 1)); + break; + } + return 1; +} + + +static int luaB_newproxy (lua_State *L) { + lua_settop(L, 1); + lua_newuserdata(L, 0); /* create proxy */ + if (lua_toboolean(L, 1) == 0) + return 1; /* no metatable */ + else if (lua_isboolean(L, 1)) { + lua_newtable(L); /* create a new metatable `m' ... */ + lua_pushvalue(L, -1); /* ... and mark `m' as a valid metatable */ + lua_pushboolean(L, 1); + lua_rawset(L, lua_upvalueindex(1)); /* weaktable[m] = true */ + } + else { + int validproxy = 0; /* to check if weaktable[metatable(u)] == true */ + if (lua_getmetatable(L, 1)) { + lua_rawget(L, lua_upvalueindex(1)); + validproxy = lua_toboolean(L, -1); + lua_pop(L, 1); /* remove value */ + } + luaL_argcheck(L, validproxy, 1, "boolean or proxy expected"); + lua_getmetatable(L, 1); /* metatable is valid; get it */ + } + lua_setmetatable(L, 2); + return 1; +} + + +static const luaL_Reg base_funcs[] = { + {"assert", luaB_assert}, + {"collectgarbage", luaB_collectgarbage}, + {"dofile", luaB_dofile}, + {"error", luaB_error}, + {"gcinfo", luaB_gcinfo}, + {"getfenv", luaB_getfenv}, + {"getmetatable", luaB_getmetatable}, + {"loadfile", luaB_loadfile}, + {"load", luaB_load}, + {"loadstring", luaB_loadstring}, + {"next", luaB_next}, + {"pcall", luaB_pcall}, + {"print", luaB_print}, + {"rawequal", luaB_rawequal}, + {"rawget", luaB_rawget}, + {"rawset", luaB_rawset}, + {"select", luaB_select}, + {"setfenv", luaB_setfenv}, + {"setmetatable", luaB_setmetatable}, + {"tonumber", luaB_tonumber}, + {"tostring", luaB_tostring}, + {"type", luaB_type}, + {"unpack", luaB_unpack}, + {"xpcall", luaB_xpcall}, + {NULL, NULL} +}; + + +/* +** {====================================================== +** Coroutine library +** ======================================================= +*/ + +#define CO_RUN 0 /* running */ +#define CO_SUS 1 /* suspended */ +#define CO_NOR 2 /* 'normal' (it resumed another coroutine) */ +#define CO_DEAD 3 + +static const char *const statnames[] = + {"running", "suspended", "normal", "dead"}; + +static int costatus (lua_State *L, lua_State *co) { + if (L == co) return CO_RUN; + switch (lua_status(co)) { + case LUA_YIELD: + return CO_SUS; + case 0: { + lua_Debug ar; + if (lua_getstack(co, 0, &ar) > 0) /* does it have frames? */ + return CO_NOR; /* it is running */ + else if (lua_gettop(co) == 0) + return CO_DEAD; + else + return CO_SUS; /* initial state */ + } + default: /* some error occured */ + return CO_DEAD; + } +} + + +static int luaB_costatus (lua_State *L) { + lua_State *co = lua_tothread(L, 1); + luaL_argcheck(L, co, 1, "coroutine expected"); + lua_pushstring(L, statnames[costatus(L, co)]); + return 1; +} + + +static int auxresume (lua_State *L, lua_State *co, int narg) { + int status = costatus(L, co); + if (!lua_checkstack(co, narg)) + luaL_error(L, "too many arguments to resume"); + if (status != CO_SUS) { + lua_pushfstring(L, "cannot resume %s coroutine", statnames[status]); + return -1; /* error flag */ + } + lua_xmove(L, co, narg); + lua_setlevel(L, co); + status = lua_resume(co, narg); + if (status == 0 || status == LUA_YIELD) { + int nres = lua_gettop(co); + if (!lua_checkstack(L, nres + 1)) + luaL_error(L, "too many results to resume"); + lua_xmove(co, L, nres); /* move yielded values */ + return nres; + } + else { + lua_xmove(co, L, 1); /* move error message */ + return -1; /* error flag */ + } +} + + +static int luaB_coresume (lua_State *L) { + lua_State *co = lua_tothread(L, 1); + int r; + luaL_argcheck(L, co, 1, "coroutine expected"); + r = auxresume(L, co, lua_gettop(L) - 1); + if (r < 0) { + lua_pushboolean(L, 0); + lua_insert(L, -2); + return 2; /* return false + error message */ + } + else { + lua_pushboolean(L, 1); + lua_insert(L, -(r + 1)); + return r + 1; /* return true + `resume' returns */ + } +} + + +static int luaB_auxwrap (lua_State *L) { + lua_State *co = lua_tothread(L, lua_upvalueindex(1)); + int r = auxresume(L, co, lua_gettop(L)); + if (r < 0) { + if (lua_isstring(L, -1)) { /* error object is a string? */ + luaL_where(L, 1); /* add extra info */ + lua_insert(L, -2); + lua_concat(L, 2); + } + lua_error(L); /* propagate error */ + } + return r; +} + + +static int luaB_cocreate (lua_State *L) { + lua_State *NL = lua_newthread(L); + luaL_argcheck(L, lua_isfunction(L, 1) && !lua_iscfunction(L, 1), 1, + "Lua function expected"); + lua_pushvalue(L, 1); /* move function to top */ + lua_xmove(L, NL, 1); /* move function from L to NL */ + return 1; +} + + +static int luaB_cowrap (lua_State *L) { + luaB_cocreate(L); + lua_pushcclosure(L, luaB_auxwrap, 1); + return 1; +} + + +static int luaB_yield (lua_State *L) { + return lua_yield(L, lua_gettop(L)); +} + + +static int luaB_corunning (lua_State *L) { + if (lua_pushthread(L)) + lua_pushnil(L); /* main thread is not a coroutine */ + return 1; +} + + +static const luaL_Reg co_funcs[] = { + {"create", luaB_cocreate}, + {"resume", luaB_coresume}, + {"running", luaB_corunning}, + {"status", luaB_costatus}, + {"wrap", luaB_cowrap}, + {"yield", luaB_yield}, + {NULL, NULL} +}; + +/* }====================================================== */ + + +static void auxopen (lua_State *L, const char *name, + lua_CFunction f, lua_CFunction u) { + lua_pushcfunction(L, u); + lua_pushcclosure(L, f, 1); + lua_setfield(L, -2, name); +} + + +static void base_open (lua_State *L) { + /* set global _G */ + lua_pushvalue(L, LUA_GLOBALSINDEX); + lua_setglobal(L, "_G"); + /* open lib into global table */ + luaL_register(L, "_G", base_funcs); + lua_pushliteral(L, LUA_VERSION); + lua_setglobal(L, "_VERSION"); /* set global _VERSION */ + /* `ipairs' and `pairs' need auxiliary functions as upvalues */ + auxopen(L, "ipairs", luaB_ipairs, ipairsaux); + auxopen(L, "pairs", luaB_pairs, luaB_next); + /* `newproxy' needs a weaktable as upvalue */ + lua_createtable(L, 0, 1); /* new table `w' */ + lua_pushvalue(L, -1); /* `w' will be its own metatable */ + lua_setmetatable(L, -2); + lua_pushliteral(L, "kv"); + lua_setfield(L, -2, "__mode"); /* metatable(w).__mode = "kv" */ + lua_pushcclosure(L, luaB_newproxy, 1); + lua_setglobal(L, "newproxy"); /* set global `newproxy' */ +} + + +LUALIB_API int luaopen_base (lua_State *L) { + base_open(L); + return 1; +} + +LUALIB_API int luaopen_coroutine (lua_State *L) { + luaL_register(L, LUA_COLIBNAME, co_funcs); + return 1; +} + diff --git a/src/lua/lcode.c b/src/lua/lcode.c new file mode 100644 index 0000000..679cb9c --- /dev/null +++ b/src/lua/lcode.c @@ -0,0 +1,831 @@ +/* +** $Id: lcode.c,v 2.25.1.5 2011/01/31 14:53:16 roberto Exp $ +** Code generator for Lua +** See Copyright Notice in lua.h +*/ + + +#include + +#define lcode_c +#define LUA_CORE + +#include "lua.h" + +#include "lcode.h" +#include "ldebug.h" +#include "ldo.h" +#include "lgc.h" +#include "llex.h" +#include "lmem.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" +#include "ltable.h" + + +#define hasjumps(e) ((e)->t != (e)->f) + + +static int isnumeral(expdesc *e) { + return (e->k == VKNUM && e->t == NO_JUMP && e->f == NO_JUMP); +} + + +void luaK_nil (FuncState *fs, int from, int n) { + Instruction *previous; + if (fs->pc > fs->lasttarget) { /* no jumps to current position? */ + if (fs->pc == 0) { /* function start? */ + if (from >= fs->nactvar) + return; /* positions are already clean */ + } + else { + previous = &fs->f->code[fs->pc-1]; + if (GET_OPCODE(*previous) == OP_LOADNIL) { + int pfrom = GETARG_A(*previous); + int pto = GETARG_B(*previous); + if (pfrom <= from && from <= pto+1) { /* can connect both? */ + if (from+n-1 > pto) + SETARG_B(*previous, from+n-1); + return; + } + } + } + } + luaK_codeABC(fs, OP_LOADNIL, from, from+n-1, 0); /* else no optimization */ +} + + +int luaK_jump (FuncState *fs) { + int jpc = fs->jpc; /* save list of jumps to here */ + int j; + fs->jpc = NO_JUMP; + j = luaK_codeAsBx(fs, OP_JMP, 0, NO_JUMP); + luaK_concat(fs, &j, jpc); /* keep them on hold */ + return j; +} + + +void luaK_ret (FuncState *fs, int first, int nret) { + luaK_codeABC(fs, OP_RETURN, first, nret+1, 0); +} + + +static int condjump (FuncState *fs, OpCode op, int A, int B, int C) { + luaK_codeABC(fs, op, A, B, C); + return luaK_jump(fs); +} + + +static void fixjump (FuncState *fs, int pc, int dest) { + Instruction *jmp = &fs->f->code[pc]; + int offset = dest-(pc+1); + lua_assert(dest != NO_JUMP); + if (abs(offset) > MAXARG_sBx) + luaX_syntaxerror(fs->ls, "control structure too long"); + SETARG_sBx(*jmp, offset); +} + + +/* +** returns current `pc' and marks it as a jump target (to avoid wrong +** optimizations with consecutive instructions not in the same basic block). +*/ +int luaK_getlabel (FuncState *fs) { + fs->lasttarget = fs->pc; + return fs->pc; +} + + +static int getjump (FuncState *fs, int pc) { + int offset = GETARG_sBx(fs->f->code[pc]); + if (offset == NO_JUMP) /* point to itself represents end of list */ + return NO_JUMP; /* end of list */ + else + return (pc+1)+offset; /* turn offset into absolute position */ +} + + +static Instruction *getjumpcontrol (FuncState *fs, int pc) { + Instruction *pi = &fs->f->code[pc]; + if (pc >= 1 && testTMode(GET_OPCODE(*(pi-1)))) + return pi-1; + else + return pi; +} + + +/* +** check whether list has any jump that do not produce a value +** (or produce an inverted value) +*/ +static int need_value (FuncState *fs, int list) { + for (; list != NO_JUMP; list = getjump(fs, list)) { + Instruction i = *getjumpcontrol(fs, list); + if (GET_OPCODE(i) != OP_TESTSET) return 1; + } + return 0; /* not found */ +} + + +static int patchtestreg (FuncState *fs, int node, int reg) { + Instruction *i = getjumpcontrol(fs, node); + if (GET_OPCODE(*i) != OP_TESTSET) + return 0; /* cannot patch other instructions */ + if (reg != NO_REG && reg != GETARG_B(*i)) + SETARG_A(*i, reg); + else /* no register to put value or register already has the value */ + *i = CREATE_ABC(OP_TEST, GETARG_B(*i), 0, GETARG_C(*i)); + + return 1; +} + + +static void removevalues (FuncState *fs, int list) { + for (; list != NO_JUMP; list = getjump(fs, list)) + patchtestreg(fs, list, NO_REG); +} + + +static void patchlistaux (FuncState *fs, int list, int vtarget, int reg, + int dtarget) { + while (list != NO_JUMP) { + int next = getjump(fs, list); + if (patchtestreg(fs, list, reg)) + fixjump(fs, list, vtarget); + else + fixjump(fs, list, dtarget); /* jump to default target */ + list = next; + } +} + + +static void dischargejpc (FuncState *fs) { + patchlistaux(fs, fs->jpc, fs->pc, NO_REG, fs->pc); + fs->jpc = NO_JUMP; +} + + +void luaK_patchlist (FuncState *fs, int list, int target) { + if (target == fs->pc) + luaK_patchtohere(fs, list); + else { + lua_assert(target < fs->pc); + patchlistaux(fs, list, target, NO_REG, target); + } +} + + +void luaK_patchtohere (FuncState *fs, int list) { + luaK_getlabel(fs); + luaK_concat(fs, &fs->jpc, list); +} + + +void luaK_concat (FuncState *fs, int *l1, int l2) { + if (l2 == NO_JUMP) return; + else if (*l1 == NO_JUMP) + *l1 = l2; + else { + int list = *l1; + int next; + while ((next = getjump(fs, list)) != NO_JUMP) /* find last element */ + list = next; + fixjump(fs, list, l2); + } +} + + +void luaK_checkstack (FuncState *fs, int n) { + int newstack = fs->freereg + n; + if (newstack > fs->f->maxstacksize) { + if (newstack >= MAXSTACK) + luaX_syntaxerror(fs->ls, "function or expression too complex"); + fs->f->maxstacksize = cast_byte(newstack); + } +} + + +void luaK_reserveregs (FuncState *fs, int n) { + luaK_checkstack(fs, n); + fs->freereg += n; +} + + +static void freereg (FuncState *fs, int reg) { + if (!ISK(reg) && reg >= fs->nactvar) { + fs->freereg--; + lua_assert(reg == fs->freereg); + } +} + + +static void freeexp (FuncState *fs, expdesc *e) { + if (e->k == VNONRELOC) + freereg(fs, e->u.s.info); +} + + +static int addk (FuncState *fs, TValue *k, TValue *v) { + lua_State *L = fs->L; + TValue *idx = luaH_set(L, fs->h, k); + Proto *f = fs->f; + int oldsize = f->sizek; + if (ttisnumber(idx)) { + lua_assert(luaO_rawequalObj(&fs->f->k[cast_int(nvalue(idx))], v)); + return cast_int(nvalue(idx)); + } + else { /* constant not found; create a new entry */ + setnvalue(idx, cast_num(fs->nk)); + luaM_growvector(L, f->k, fs->nk, f->sizek, TValue, + MAXARG_Bx, "constant table overflow"); + while (oldsize < f->sizek) setnilvalue(&f->k[oldsize++]); + setobj(L, &f->k[fs->nk], v); + luaC_barrier(L, f, v); + return fs->nk++; + } +} + + +int luaK_stringK (FuncState *fs, TString *s) { + TValue o; + setsvalue(fs->L, &o, s); + return addk(fs, &o, &o); +} + + +int luaK_numberK (FuncState *fs, lua_Number r) { + TValue o; + setnvalue(&o, r); + return addk(fs, &o, &o); +} + + +static int boolK (FuncState *fs, int b) { + TValue o; + setbvalue(&o, b); + return addk(fs, &o, &o); +} + + +static int nilK (FuncState *fs) { + TValue k, v; + setnilvalue(&v); + /* cannot use nil as key; instead use table itself to represent nil */ + sethvalue(fs->L, &k, fs->h); + return addk(fs, &k, &v); +} + + +void luaK_setreturns (FuncState *fs, expdesc *e, int nresults) { + if (e->k == VCALL) { /* expression is an open function call? */ + SETARG_C(getcode(fs, e), nresults+1); + } + else if (e->k == VVARARG) { + SETARG_B(getcode(fs, e), nresults+1); + SETARG_A(getcode(fs, e), fs->freereg); + luaK_reserveregs(fs, 1); + } +} + + +void luaK_setoneret (FuncState *fs, expdesc *e) { + if (e->k == VCALL) { /* expression is an open function call? */ + e->k = VNONRELOC; + e->u.s.info = GETARG_A(getcode(fs, e)); + } + else if (e->k == VVARARG) { + SETARG_B(getcode(fs, e), 2); + e->k = VRELOCABLE; /* can relocate its simple result */ + } +} + + +void luaK_dischargevars (FuncState *fs, expdesc *e) { + switch (e->k) { + case VLOCAL: { + e->k = VNONRELOC; + break; + } + case VUPVAL: { + e->u.s.info = luaK_codeABC(fs, OP_GETUPVAL, 0, e->u.s.info, 0); + e->k = VRELOCABLE; + break; + } + case VGLOBAL: { + e->u.s.info = luaK_codeABx(fs, OP_GETGLOBAL, 0, e->u.s.info); + e->k = VRELOCABLE; + break; + } + case VINDEXED: { + freereg(fs, e->u.s.aux); + freereg(fs, e->u.s.info); + e->u.s.info = luaK_codeABC(fs, OP_GETTABLE, 0, e->u.s.info, e->u.s.aux); + e->k = VRELOCABLE; + break; + } + case VVARARG: + case VCALL: { + luaK_setoneret(fs, e); + break; + } + default: break; /* there is one value available (somewhere) */ + } +} + + +static int code_label (FuncState *fs, int A, int b, int jump) { + luaK_getlabel(fs); /* those instructions may be jump targets */ + return luaK_codeABC(fs, OP_LOADBOOL, A, b, jump); +} + + +static void discharge2reg (FuncState *fs, expdesc *e, int reg) { + luaK_dischargevars(fs, e); + switch (e->k) { + case VNIL: { + luaK_nil(fs, reg, 1); + break; + } + case VFALSE: case VTRUE: { + luaK_codeABC(fs, OP_LOADBOOL, reg, e->k == VTRUE, 0); + break; + } + case VK: { + luaK_codeABx(fs, OP_LOADK, reg, e->u.s.info); + break; + } + case VKNUM: { + luaK_codeABx(fs, OP_LOADK, reg, luaK_numberK(fs, e->u.nval)); + break; + } + case VRELOCABLE: { + Instruction *pc = &getcode(fs, e); + SETARG_A(*pc, reg); + break; + } + case VNONRELOC: { + if (reg != e->u.s.info) + luaK_codeABC(fs, OP_MOVE, reg, e->u.s.info, 0); + break; + } + default: { + lua_assert(e->k == VVOID || e->k == VJMP); + return; /* nothing to do... */ + } + } + e->u.s.info = reg; + e->k = VNONRELOC; +} + + +static void discharge2anyreg (FuncState *fs, expdesc *e) { + if (e->k != VNONRELOC) { + luaK_reserveregs(fs, 1); + discharge2reg(fs, e, fs->freereg-1); + } +} + + +static void exp2reg (FuncState *fs, expdesc *e, int reg) { + discharge2reg(fs, e, reg); + if (e->k == VJMP) + luaK_concat(fs, &e->t, e->u.s.info); /* put this jump in `t' list */ + if (hasjumps(e)) { + int final; /* position after whole expression */ + int p_f = NO_JUMP; /* position of an eventual LOAD false */ + int p_t = NO_JUMP; /* position of an eventual LOAD true */ + if (need_value(fs, e->t) || need_value(fs, e->f)) { + int fj = (e->k == VJMP) ? NO_JUMP : luaK_jump(fs); + p_f = code_label(fs, reg, 0, 1); + p_t = code_label(fs, reg, 1, 0); + luaK_patchtohere(fs, fj); + } + final = luaK_getlabel(fs); + patchlistaux(fs, e->f, final, reg, p_f); + patchlistaux(fs, e->t, final, reg, p_t); + } + e->f = e->t = NO_JUMP; + e->u.s.info = reg; + e->k = VNONRELOC; +} + + +void luaK_exp2nextreg (FuncState *fs, expdesc *e) { + luaK_dischargevars(fs, e); + freeexp(fs, e); + luaK_reserveregs(fs, 1); + exp2reg(fs, e, fs->freereg - 1); +} + + +int luaK_exp2anyreg (FuncState *fs, expdesc *e) { + luaK_dischargevars(fs, e); + if (e->k == VNONRELOC) { + if (!hasjumps(e)) return e->u.s.info; /* exp is already in a register */ + if (e->u.s.info >= fs->nactvar) { /* reg. is not a local? */ + exp2reg(fs, e, e->u.s.info); /* put value on it */ + return e->u.s.info; + } + } + luaK_exp2nextreg(fs, e); /* default */ + return e->u.s.info; +} + + +void luaK_exp2val (FuncState *fs, expdesc *e) { + if (hasjumps(e)) + luaK_exp2anyreg(fs, e); + else + luaK_dischargevars(fs, e); +} + + +int luaK_exp2RK (FuncState *fs, expdesc *e) { + luaK_exp2val(fs, e); + switch (e->k) { + case VKNUM: + case VTRUE: + case VFALSE: + case VNIL: { + if (fs->nk <= MAXINDEXRK) { /* constant fit in RK operand? */ + e->u.s.info = (e->k == VNIL) ? nilK(fs) : + (e->k == VKNUM) ? luaK_numberK(fs, e->u.nval) : + boolK(fs, (e->k == VTRUE)); + e->k = VK; + return RKASK(e->u.s.info); + } + else break; + } + case VK: { + if (e->u.s.info <= MAXINDEXRK) /* constant fit in argC? */ + return RKASK(e->u.s.info); + else break; + } + default: break; + } + /* not a constant in the right range: put it in a register */ + return luaK_exp2anyreg(fs, e); +} + + +void luaK_storevar (FuncState *fs, expdesc *var, expdesc *ex) { + switch (var->k) { + case VLOCAL: { + freeexp(fs, ex); + exp2reg(fs, ex, var->u.s.info); + return; + } + case VUPVAL: { + int e = luaK_exp2anyreg(fs, ex); + luaK_codeABC(fs, OP_SETUPVAL, e, var->u.s.info, 0); + break; + } + case VGLOBAL: { + int e = luaK_exp2anyreg(fs, ex); + luaK_codeABx(fs, OP_SETGLOBAL, e, var->u.s.info); + break; + } + case VINDEXED: { + int e = luaK_exp2RK(fs, ex); + luaK_codeABC(fs, OP_SETTABLE, var->u.s.info, var->u.s.aux, e); + break; + } + default: { + lua_assert(0); /* invalid var kind to store */ + break; + } + } + freeexp(fs, ex); +} + + +void luaK_self (FuncState *fs, expdesc *e, expdesc *key) { + int func; + luaK_exp2anyreg(fs, e); + freeexp(fs, e); + func = fs->freereg; + luaK_reserveregs(fs, 2); + luaK_codeABC(fs, OP_SELF, func, e->u.s.info, luaK_exp2RK(fs, key)); + freeexp(fs, key); + e->u.s.info = func; + e->k = VNONRELOC; +} + + +static void invertjump (FuncState *fs, expdesc *e) { + Instruction *pc = getjumpcontrol(fs, e->u.s.info); + lua_assert(testTMode(GET_OPCODE(*pc)) && GET_OPCODE(*pc) != OP_TESTSET && + GET_OPCODE(*pc) != OP_TEST); + SETARG_A(*pc, !(GETARG_A(*pc))); +} + + +static int jumponcond (FuncState *fs, expdesc *e, int cond) { + if (e->k == VRELOCABLE) { + Instruction ie = getcode(fs, e); + if (GET_OPCODE(ie) == OP_NOT) { + fs->pc--; /* remove previous OP_NOT */ + return condjump(fs, OP_TEST, GETARG_B(ie), 0, !cond); + } + /* else go through */ + } + discharge2anyreg(fs, e); + freeexp(fs, e); + return condjump(fs, OP_TESTSET, NO_REG, e->u.s.info, cond); +} + + +void luaK_goiftrue (FuncState *fs, expdesc *e) { + int pc; /* pc of last jump */ + luaK_dischargevars(fs, e); + switch (e->k) { + case VK: case VKNUM: case VTRUE: { + pc = NO_JUMP; /* always true; do nothing */ + break; + } + case VJMP: { + invertjump(fs, e); + pc = e->u.s.info; + break; + } + default: { + pc = jumponcond(fs, e, 0); + break; + } + } + luaK_concat(fs, &e->f, pc); /* insert last jump in `f' list */ + luaK_patchtohere(fs, e->t); + e->t = NO_JUMP; +} + + +static void luaK_goiffalse (FuncState *fs, expdesc *e) { + int pc; /* pc of last jump */ + luaK_dischargevars(fs, e); + switch (e->k) { + case VNIL: case VFALSE: { + pc = NO_JUMP; /* always false; do nothing */ + break; + } + case VJMP: { + pc = e->u.s.info; + break; + } + default: { + pc = jumponcond(fs, e, 1); + break; + } + } + luaK_concat(fs, &e->t, pc); /* insert last jump in `t' list */ + luaK_patchtohere(fs, e->f); + e->f = NO_JUMP; +} + + +static void codenot (FuncState *fs, expdesc *e) { + luaK_dischargevars(fs, e); + switch (e->k) { + case VNIL: case VFALSE: { + e->k = VTRUE; + break; + } + case VK: case VKNUM: case VTRUE: { + e->k = VFALSE; + break; + } + case VJMP: { + invertjump(fs, e); + break; + } + case VRELOCABLE: + case VNONRELOC: { + discharge2anyreg(fs, e); + freeexp(fs, e); + e->u.s.info = luaK_codeABC(fs, OP_NOT, 0, e->u.s.info, 0); + e->k = VRELOCABLE; + break; + } + default: { + lua_assert(0); /* cannot happen */ + break; + } + } + /* interchange true and false lists */ + { int temp = e->f; e->f = e->t; e->t = temp; } + removevalues(fs, e->f); + removevalues(fs, e->t); +} + + +void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k) { + t->u.s.aux = luaK_exp2RK(fs, k); + t->k = VINDEXED; +} + + +static int constfolding (OpCode op, expdesc *e1, expdesc *e2) { + lua_Number v1, v2, r; + if (!isnumeral(e1) || !isnumeral(e2)) return 0; + v1 = e1->u.nval; + v2 = e2->u.nval; + switch (op) { + case OP_ADD: r = luai_numadd(v1, v2); break; + case OP_SUB: r = luai_numsub(v1, v2); break; + case OP_MUL: r = luai_nummul(v1, v2); break; + case OP_DIV: + if (v2 == 0) return 0; /* do not attempt to divide by 0 */ + r = luai_numdiv(v1, v2); break; + case OP_MOD: + if (v2 == 0) return 0; /* do not attempt to divide by 0 */ + r = luai_nummod(v1, v2); break; + case OP_POW: r = luai_numpow(v1, v2); break; + case OP_UNM: r = luai_numunm(v1); break; + case OP_LEN: return 0; /* no constant folding for 'len' */ + default: lua_assert(0); r = 0; break; + } + if (luai_numisnan(r)) return 0; /* do not attempt to produce NaN */ + e1->u.nval = r; + return 1; +} + + +static void codearith (FuncState *fs, OpCode op, expdesc *e1, expdesc *e2) { + if (constfolding(op, e1, e2)) + return; + else { + int o2 = (op != OP_UNM && op != OP_LEN) ? luaK_exp2RK(fs, e2) : 0; + int o1 = luaK_exp2RK(fs, e1); + if (o1 > o2) { + freeexp(fs, e1); + freeexp(fs, e2); + } + else { + freeexp(fs, e2); + freeexp(fs, e1); + } + e1->u.s.info = luaK_codeABC(fs, op, 0, o1, o2); + e1->k = VRELOCABLE; + } +} + + +static void codecomp (FuncState *fs, OpCode op, int cond, expdesc *e1, + expdesc *e2) { + int o1 = luaK_exp2RK(fs, e1); + int o2 = luaK_exp2RK(fs, e2); + freeexp(fs, e2); + freeexp(fs, e1); + if (cond == 0 && op != OP_EQ) { + int temp; /* exchange args to replace by `<' or `<=' */ + temp = o1; o1 = o2; o2 = temp; /* o1 <==> o2 */ + cond = 1; + } + e1->u.s.info = condjump(fs, op, cond, o1, o2); + e1->k = VJMP; +} + + +void luaK_prefix (FuncState *fs, UnOpr op, expdesc *e) { + expdesc e2; + e2.t = e2.f = NO_JUMP; e2.k = VKNUM; e2.u.nval = 0; + switch (op) { + case OPR_MINUS: { + if (!isnumeral(e)) + luaK_exp2anyreg(fs, e); /* cannot operate on non-numeric constants */ + codearith(fs, OP_UNM, e, &e2); + break; + } + case OPR_NOT: codenot(fs, e); break; + case OPR_LEN: { + luaK_exp2anyreg(fs, e); /* cannot operate on constants */ + codearith(fs, OP_LEN, e, &e2); + break; + } + default: lua_assert(0); + } +} + + +void luaK_infix (FuncState *fs, BinOpr op, expdesc *v) { + switch (op) { + case OPR_AND: { + luaK_goiftrue(fs, v); + break; + } + case OPR_OR: { + luaK_goiffalse(fs, v); + break; + } + case OPR_CONCAT: { + luaK_exp2nextreg(fs, v); /* operand must be on the `stack' */ + break; + } + case OPR_ADD: case OPR_SUB: case OPR_MUL: case OPR_DIV: + case OPR_MOD: case OPR_POW: { + if (!isnumeral(v)) luaK_exp2RK(fs, v); + break; + } + default: { + luaK_exp2RK(fs, v); + break; + } + } +} + + +void luaK_posfix (FuncState *fs, BinOpr op, expdesc *e1, expdesc *e2) { + switch (op) { + case OPR_AND: { + lua_assert(e1->t == NO_JUMP); /* list must be closed */ + luaK_dischargevars(fs, e2); + luaK_concat(fs, &e2->f, e1->f); + *e1 = *e2; + break; + } + case OPR_OR: { + lua_assert(e1->f == NO_JUMP); /* list must be closed */ + luaK_dischargevars(fs, e2); + luaK_concat(fs, &e2->t, e1->t); + *e1 = *e2; + break; + } + case OPR_CONCAT: { + luaK_exp2val(fs, e2); + if (e2->k == VRELOCABLE && GET_OPCODE(getcode(fs, e2)) == OP_CONCAT) { + lua_assert(e1->u.s.info == GETARG_B(getcode(fs, e2))-1); + freeexp(fs, e1); + SETARG_B(getcode(fs, e2), e1->u.s.info); + e1->k = VRELOCABLE; e1->u.s.info = e2->u.s.info; + } + else { + luaK_exp2nextreg(fs, e2); /* operand must be on the 'stack' */ + codearith(fs, OP_CONCAT, e1, e2); + } + break; + } + case OPR_ADD: codearith(fs, OP_ADD, e1, e2); break; + case OPR_SUB: codearith(fs, OP_SUB, e1, e2); break; + case OPR_MUL: codearith(fs, OP_MUL, e1, e2); break; + case OPR_DIV: codearith(fs, OP_DIV, e1, e2); break; + case OPR_MOD: codearith(fs, OP_MOD, e1, e2); break; + case OPR_POW: codearith(fs, OP_POW, e1, e2); break; + case OPR_EQ: codecomp(fs, OP_EQ, 1, e1, e2); break; + case OPR_NE: codecomp(fs, OP_EQ, 0, e1, e2); break; + case OPR_LT: codecomp(fs, OP_LT, 1, e1, e2); break; + case OPR_LE: codecomp(fs, OP_LE, 1, e1, e2); break; + case OPR_GT: codecomp(fs, OP_LT, 0, e1, e2); break; + case OPR_GE: codecomp(fs, OP_LE, 0, e1, e2); break; + default: lua_assert(0); + } +} + + +void luaK_fixline (FuncState *fs, int line) { + fs->f->lineinfo[fs->pc - 1] = line; +} + + +static int luaK_code (FuncState *fs, Instruction i, int line) { + Proto *f = fs->f; + dischargejpc(fs); /* `pc' will change */ + /* put new instruction in code array */ + luaM_growvector(fs->L, f->code, fs->pc, f->sizecode, Instruction, + MAX_INT, "code size overflow"); + f->code[fs->pc] = i; + /* save corresponding line information */ + luaM_growvector(fs->L, f->lineinfo, fs->pc, f->sizelineinfo, int, + MAX_INT, "code size overflow"); + f->lineinfo[fs->pc] = line; + return fs->pc++; +} + + +int luaK_codeABC (FuncState *fs, OpCode o, int a, int b, int c) { + lua_assert(getOpMode(o) == iABC); + lua_assert(getBMode(o) != OpArgN || b == 0); + lua_assert(getCMode(o) != OpArgN || c == 0); + return luaK_code(fs, CREATE_ABC(o, a, b, c), fs->ls->lastline); +} + + +int luaK_codeABx (FuncState *fs, OpCode o, int a, unsigned int bc) { + lua_assert(getOpMode(o) == iABx || getOpMode(o) == iAsBx); + lua_assert(getCMode(o) == OpArgN); + return luaK_code(fs, CREATE_ABx(o, a, bc), fs->ls->lastline); +} + + +void luaK_setlist (FuncState *fs, int base, int nelems, int tostore) { + int c = (nelems - 1)/LFIELDS_PER_FLUSH + 1; + int b = (tostore == LUA_MULTRET) ? 0 : tostore; + lua_assert(tostore != 0); + if (c <= MAXARG_C) + luaK_codeABC(fs, OP_SETLIST, base, b, c); + else { + luaK_codeABC(fs, OP_SETLIST, base, b, 0); + luaK_code(fs, cast(Instruction, c), fs->ls->lastline); + } + fs->freereg = base + 1; /* free registers with list values */ +} + diff --git a/src/lua/lcode.h b/src/lua/lcode.h new file mode 100644 index 0000000..b941c60 --- /dev/null +++ b/src/lua/lcode.h @@ -0,0 +1,76 @@ +/* +** $Id: lcode.h,v 1.48.1.1 2007/12/27 13:02:25 roberto Exp $ +** Code generator for Lua +** See Copyright Notice in lua.h +*/ + +#ifndef lcode_h +#define lcode_h + +#include "llex.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" + + +/* +** Marks the end of a patch list. It is an invalid value both as an absolute +** address, and as a list link (would link an element to itself). +*/ +#define NO_JUMP (-1) + + +/* +** grep "ORDER OPR" if you change these enums +*/ +typedef enum BinOpr { + OPR_ADD, OPR_SUB, OPR_MUL, OPR_DIV, OPR_MOD, OPR_POW, + OPR_CONCAT, + OPR_NE, OPR_EQ, + OPR_LT, OPR_LE, OPR_GT, OPR_GE, + OPR_AND, OPR_OR, + OPR_NOBINOPR +} BinOpr; + + +typedef enum UnOpr { OPR_MINUS, OPR_NOT, OPR_LEN, OPR_NOUNOPR } UnOpr; + + +#define getcode(fs,e) ((fs)->f->code[(e)->u.s.info]) + +#define luaK_codeAsBx(fs,o,A,sBx) luaK_codeABx(fs,o,A,(sBx)+MAXARG_sBx) + +#define luaK_setmultret(fs,e) luaK_setreturns(fs, e, LUA_MULTRET) + +LUAI_FUNC int luaK_codeABx (FuncState *fs, OpCode o, int A, unsigned int Bx); +LUAI_FUNC int luaK_codeABC (FuncState *fs, OpCode o, int A, int B, int C); +LUAI_FUNC void luaK_fixline (FuncState *fs, int line); +LUAI_FUNC void luaK_nil (FuncState *fs, int from, int n); +LUAI_FUNC void luaK_reserveregs (FuncState *fs, int n); +LUAI_FUNC void luaK_checkstack (FuncState *fs, int n); +LUAI_FUNC int luaK_stringK (FuncState *fs, TString *s); +LUAI_FUNC int luaK_numberK (FuncState *fs, lua_Number r); +LUAI_FUNC void luaK_dischargevars (FuncState *fs, expdesc *e); +LUAI_FUNC int luaK_exp2anyreg (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_exp2nextreg (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_exp2val (FuncState *fs, expdesc *e); +LUAI_FUNC int luaK_exp2RK (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_self (FuncState *fs, expdesc *e, expdesc *key); +LUAI_FUNC void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k); +LUAI_FUNC void luaK_goiftrue (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_storevar (FuncState *fs, expdesc *var, expdesc *e); +LUAI_FUNC void luaK_setreturns (FuncState *fs, expdesc *e, int nresults); +LUAI_FUNC void luaK_setoneret (FuncState *fs, expdesc *e); +LUAI_FUNC int luaK_jump (FuncState *fs); +LUAI_FUNC void luaK_ret (FuncState *fs, int first, int nret); +LUAI_FUNC void luaK_patchlist (FuncState *fs, int list, int target); +LUAI_FUNC void luaK_patchtohere (FuncState *fs, int list); +LUAI_FUNC void luaK_concat (FuncState *fs, int *l1, int l2); +LUAI_FUNC int luaK_getlabel (FuncState *fs); +LUAI_FUNC void luaK_prefix (FuncState *fs, UnOpr op, expdesc *v); +LUAI_FUNC void luaK_infix (FuncState *fs, BinOpr op, expdesc *v); +LUAI_FUNC void luaK_posfix (FuncState *fs, BinOpr op, expdesc *v1, expdesc *v2); +LUAI_FUNC void luaK_setlist (FuncState *fs, int base, int nelems, int tostore); + + +#endif diff --git a/src/lua/ldblib.c b/src/lua/ldblib.c new file mode 100644 index 0000000..2027eda --- /dev/null +++ b/src/lua/ldblib.c @@ -0,0 +1,398 @@ +/* +** $Id: ldblib.c,v 1.104.1.4 2009/08/04 18:50:18 roberto Exp $ +** Interface from Lua to its debug API +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define ldblib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + + +static int db_getregistry (lua_State *L) { + lua_pushvalue(L, LUA_REGISTRYINDEX); + return 1; +} + + +static int db_getmetatable (lua_State *L) { + luaL_checkany(L, 1); + if (!lua_getmetatable(L, 1)) { + lua_pushnil(L); /* no metatable */ + } + return 1; +} + + +static int db_setmetatable (lua_State *L) { + int t = lua_type(L, 2); + luaL_argcheck(L, t == LUA_TNIL || t == LUA_TTABLE, 2, + "nil or table expected"); + lua_settop(L, 2); + lua_pushboolean(L, lua_setmetatable(L, 1)); + return 1; +} + + +static int db_getfenv (lua_State *L) { + luaL_checkany(L, 1); + lua_getfenv(L, 1); + return 1; +} + + +static int db_setfenv (lua_State *L) { + luaL_checktype(L, 2, LUA_TTABLE); + lua_settop(L, 2); + if (lua_setfenv(L, 1) == 0) + luaL_error(L, LUA_QL("setfenv") + " cannot change environment of given object"); + return 1; +} + + +static void settabss (lua_State *L, const char *i, const char *v) { + lua_pushstring(L, v); + lua_setfield(L, -2, i); +} + + +static void settabsi (lua_State *L, const char *i, int v) { + lua_pushinteger(L, v); + lua_setfield(L, -2, i); +} + + +static lua_State *getthread (lua_State *L, int *arg) { + if (lua_isthread(L, 1)) { + *arg = 1; + return lua_tothread(L, 1); + } + else { + *arg = 0; + return L; + } +} + + +static void treatstackoption (lua_State *L, lua_State *L1, const char *fname) { + if (L == L1) { + lua_pushvalue(L, -2); + lua_remove(L, -3); + } + else + lua_xmove(L1, L, 1); + lua_setfield(L, -2, fname); +} + + +static int db_getinfo (lua_State *L) { + lua_Debug ar; + int arg; + lua_State *L1 = getthread(L, &arg); + const char *options = luaL_optstring(L, arg+2, "flnSu"); + if (lua_isnumber(L, arg+1)) { + if (!lua_getstack(L1, (int)lua_tointeger(L, arg+1), &ar)) { + lua_pushnil(L); /* level out of range */ + return 1; + } + } + else if (lua_isfunction(L, arg+1)) { + lua_pushfstring(L, ">%s", options); + options = lua_tostring(L, -1); + lua_pushvalue(L, arg+1); + lua_xmove(L, L1, 1); + } + else + return luaL_argerror(L, arg+1, "function or level expected"); + if (!lua_getinfo(L1, options, &ar)) + return luaL_argerror(L, arg+2, "invalid option"); + lua_createtable(L, 0, 2); + if (strchr(options, 'S')) { + settabss(L, "source", ar.source); + settabss(L, "short_src", ar.short_src); + settabsi(L, "linedefined", ar.linedefined); + settabsi(L, "lastlinedefined", ar.lastlinedefined); + settabss(L, "what", ar.what); + } + if (strchr(options, 'l')) + settabsi(L, "currentline", ar.currentline); + if (strchr(options, 'u')) + settabsi(L, "nups", ar.nups); + if (strchr(options, 'n')) { + settabss(L, "name", ar.name); + settabss(L, "namewhat", ar.namewhat); + } + if (strchr(options, 'L')) + treatstackoption(L, L1, "activelines"); + if (strchr(options, 'f')) + treatstackoption(L, L1, "func"); + return 1; /* return table */ +} + + +static int db_getlocal (lua_State *L) { + int arg; + lua_State *L1 = getthread(L, &arg); + lua_Debug ar; + const char *name; + if (!lua_getstack(L1, luaL_checkint(L, arg+1), &ar)) /* out of range? */ + return luaL_argerror(L, arg+1, "level out of range"); + name = lua_getlocal(L1, &ar, luaL_checkint(L, arg+2)); + if (name) { + lua_xmove(L1, L, 1); + lua_pushstring(L, name); + lua_pushvalue(L, -2); + return 2; + } + else { + lua_pushnil(L); + return 1; + } +} + + +static int db_setlocal (lua_State *L) { + int arg; + lua_State *L1 = getthread(L, &arg); + lua_Debug ar; + if (!lua_getstack(L1, luaL_checkint(L, arg+1), &ar)) /* out of range? */ + return luaL_argerror(L, arg+1, "level out of range"); + luaL_checkany(L, arg+3); + lua_settop(L, arg+3); + lua_xmove(L, L1, 1); + lua_pushstring(L, lua_setlocal(L1, &ar, luaL_checkint(L, arg+2))); + return 1; +} + + +static int auxupvalue (lua_State *L, int get) { + const char *name; + int n = luaL_checkint(L, 2); + luaL_checktype(L, 1, LUA_TFUNCTION); + if (lua_iscfunction(L, 1)) return 0; /* cannot touch C upvalues from Lua */ + name = get ? lua_getupvalue(L, 1, n) : lua_setupvalue(L, 1, n); + if (name == NULL) return 0; + lua_pushstring(L, name); + lua_insert(L, -(get+1)); + return get + 1; +} + + +static int db_getupvalue (lua_State *L) { + return auxupvalue(L, 1); +} + + +static int db_setupvalue (lua_State *L) { + luaL_checkany(L, 3); + return auxupvalue(L, 0); +} + + + +static const char KEY_HOOK = 'h'; + + +static void hookf (lua_State *L, lua_Debug *ar) { + static const char *const hooknames[] = + {"call", "return", "line", "count", "tail return"}; + lua_pushlightuserdata(L, (void *)&KEY_HOOK); + lua_rawget(L, LUA_REGISTRYINDEX); + lua_pushlightuserdata(L, L); + lua_rawget(L, -2); + if (lua_isfunction(L, -1)) { + lua_pushstring(L, hooknames[(int)ar->event]); + if (ar->currentline >= 0) + lua_pushinteger(L, ar->currentline); + else lua_pushnil(L); + lua_assert(lua_getinfo(L, "lS", ar)); + lua_call(L, 2, 0); + } +} + + +static int makemask (const char *smask, int count) { + int mask = 0; + if (strchr(smask, 'c')) mask |= LUA_MASKCALL; + if (strchr(smask, 'r')) mask |= LUA_MASKRET; + if (strchr(smask, 'l')) mask |= LUA_MASKLINE; + if (count > 0) mask |= LUA_MASKCOUNT; + return mask; +} + + +static char *unmakemask (int mask, char *smask) { + int i = 0; + if (mask & LUA_MASKCALL) smask[i++] = 'c'; + if (mask & LUA_MASKRET) smask[i++] = 'r'; + if (mask & LUA_MASKLINE) smask[i++] = 'l'; + smask[i] = '\0'; + return smask; +} + + +static void gethooktable (lua_State *L) { + lua_pushlightuserdata(L, (void *)&KEY_HOOK); + lua_rawget(L, LUA_REGISTRYINDEX); + if (!lua_istable(L, -1)) { + lua_pop(L, 1); + lua_createtable(L, 0, 1); + lua_pushlightuserdata(L, (void *)&KEY_HOOK); + lua_pushvalue(L, -2); + lua_rawset(L, LUA_REGISTRYINDEX); + } +} + + +static int db_sethook (lua_State *L) { + int arg, mask, count; + lua_Hook func; + lua_State *L1 = getthread(L, &arg); + if (lua_isnoneornil(L, arg+1)) { + lua_settop(L, arg+1); + func = NULL; mask = 0; count = 0; /* turn off hooks */ + } + else { + const char *smask = luaL_checkstring(L, arg+2); + luaL_checktype(L, arg+1, LUA_TFUNCTION); + count = luaL_optint(L, arg+3, 0); + func = hookf; mask = makemask(smask, count); + } + gethooktable(L); + lua_pushlightuserdata(L, L1); + lua_pushvalue(L, arg+1); + lua_rawset(L, -3); /* set new hook */ + lua_pop(L, 1); /* remove hook table */ + lua_sethook(L1, func, mask, count); /* set hooks */ + return 0; +} + + +static int db_gethook (lua_State *L) { + int arg; + lua_State *L1 = getthread(L, &arg); + char buff[5]; + int mask = lua_gethookmask(L1); + lua_Hook hook = lua_gethook(L1); + if (hook != NULL && hook != hookf) /* external hook? */ + lua_pushliteral(L, "external hook"); + else { + gethooktable(L); + lua_pushlightuserdata(L, L1); + lua_rawget(L, -2); /* get hook */ + lua_remove(L, -2); /* remove hook table */ + } + lua_pushstring(L, unmakemask(mask, buff)); + lua_pushinteger(L, lua_gethookcount(L1)); + return 3; +} + + +static int db_debug (lua_State *L) { + for (;;) { + char buffer[250]; + fputs("lua_debug> ", stderr); + if (fgets(buffer, sizeof(buffer), stdin) == 0 || + strcmp(buffer, "cont\n") == 0) + return 0; + if (luaL_loadbuffer(L, buffer, strlen(buffer), "=(debug command)") || + lua_pcall(L, 0, 0, 0)) { + fputs(lua_tostring(L, -1), stderr); + fputs("\n", stderr); + } + lua_settop(L, 0); /* remove eventual returns */ + } +} + + +#define LEVELS1 12 /* size of the first part of the stack */ +#define LEVELS2 10 /* size of the second part of the stack */ + +static int db_errorfb (lua_State *L) { + int level; + int firstpart = 1; /* still before eventual `...' */ + int arg; + lua_State *L1 = getthread(L, &arg); + lua_Debug ar; + if (lua_isnumber(L, arg+2)) { + level = (int)lua_tointeger(L, arg+2); + lua_pop(L, 1); + } + else + level = (L == L1) ? 1 : 0; /* level 0 may be this own function */ + if (lua_gettop(L) == arg) + lua_pushliteral(L, ""); + else if (!lua_isstring(L, arg+1)) return 1; /* message is not a string */ + else lua_pushliteral(L, "\n"); + lua_pushliteral(L, "stack traceback:"); + while (lua_getstack(L1, level++, &ar)) { + if (level > LEVELS1 && firstpart) { + /* no more than `LEVELS2' more levels? */ + if (!lua_getstack(L1, level+LEVELS2, &ar)) + level--; /* keep going */ + else { + lua_pushliteral(L, "\n\t..."); /* too many levels */ + while (lua_getstack(L1, level+LEVELS2, &ar)) /* find last levels */ + level++; + } + firstpart = 0; + continue; + } + lua_pushliteral(L, "\n\t"); + lua_getinfo(L1, "Snl", &ar); + lua_pushfstring(L, "%s:", ar.short_src); + if (ar.currentline > 0) + lua_pushfstring(L, "%d:", ar.currentline); + if (*ar.namewhat != '\0') /* is there a name? */ + lua_pushfstring(L, " in function " LUA_QS, ar.name); + else { + if (*ar.what == 'm') /* main? */ + lua_pushfstring(L, " in main chunk"); + else if (*ar.what == 'C' || *ar.what == 't') + lua_pushliteral(L, " ?"); /* C function or tail call */ + else + lua_pushfstring(L, " in function <%s:%d>", + ar.short_src, ar.linedefined); + } + lua_concat(L, lua_gettop(L) - arg); + } + lua_concat(L, lua_gettop(L) - arg); + return 1; +} + + +static const luaL_Reg dblib[] = { + {"debug", db_debug}, + {"getfenv", db_getfenv}, + {"gethook", db_gethook}, + {"getinfo", db_getinfo}, + {"getlocal", db_getlocal}, + {"getregistry", db_getregistry}, + {"getmetatable", db_getmetatable}, + {"getupvalue", db_getupvalue}, + {"setfenv", db_setfenv}, + {"sethook", db_sethook}, + {"setlocal", db_setlocal}, + {"setmetatable", db_setmetatable}, + {"setupvalue", db_setupvalue}, + {"traceback", db_errorfb}, + {NULL, NULL} +}; + + +LUALIB_API int luaopen_debug (lua_State *L) { + luaL_register(L, LUA_DBLIBNAME, dblib); + return 1; +} + diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c new file mode 100644 index 0000000..da76361 --- /dev/null +++ b/src/lua/ldebug.c @@ -0,0 +1,642 @@ +/* +** $Id: ldebug.c,v 2.29.1.6 2008/05/08 16:56:26 roberto Exp $ +** Debug Interface +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + + +#define ldebug_c +#define LUA_CORE + +#include "lua.h" + +#include "lapi.h" +#include "lcode.h" +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lvm.h" + + + +static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name); + + +static int currentpc (lua_State *L, CallInfo *ci) { + if (!isLua(ci)) return -1; /* function is not a Lua function? */ + if (ci == L->ci) + ci->savedpc = L->savedpc; + return pcRel(ci->savedpc, ci_func(ci)->l.p); +} + + +static int currentline (lua_State *L, CallInfo *ci) { + int pc = currentpc(L, ci); + if (pc < 0) + return -1; /* only active lua functions have current-line information */ + else + return getline(ci_func(ci)->l.p, pc); +} + + +/* +** this function can be called asynchronous (e.g. during a signal) +*/ +LUA_API int lua_sethook (lua_State *L, lua_Hook func, int mask, int count) { + if (func == NULL || mask == 0) { /* turn off hooks? */ + mask = 0; + func = NULL; + } + L->hook = func; + L->basehookcount = count; + resethookcount(L); + L->hookmask = cast_byte(mask); + return 1; +} + + +LUA_API lua_Hook lua_gethook (lua_State *L) { + return L->hook; +} + + +LUA_API int lua_gethookmask (lua_State *L) { + return L->hookmask; +} + + +LUA_API int lua_gethookcount (lua_State *L) { + return L->basehookcount; +} + +LUA_API int lua_gethookcountremaining (lua_State *L) { + return L->hookcount; +} + + +LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar) { + int status; + CallInfo *ci; + lua_lock(L); + for (ci = L->ci; level > 0 && ci > L->base_ci; ci--) { + level--; + if (f_isLua(ci)) /* Lua function? */ + level -= ci->tailcalls; /* skip lost tail calls */ + } + if (level == 0 && ci > L->base_ci) { /* level found? */ + status = 1; + ar->i_ci = cast_int(ci - L->base_ci); + } + else if (level < 0) { /* level is of a lost tail call? */ + status = 1; + ar->i_ci = 0; + } + else status = 0; /* no such level */ + lua_unlock(L); + return status; +} + + +static Proto *getluaproto (CallInfo *ci) { + return (isLua(ci) ? ci_func(ci)->l.p : NULL); +} + + +static const char *findlocal (lua_State *L, CallInfo *ci, int n) { + const char *name; + Proto *fp = getluaproto(ci); + if (fp && (name = luaF_getlocalname(fp, n, currentpc(L, ci))) != NULL) + return name; /* is a local variable in a Lua function */ + else { + StkId limit = (ci == L->ci) ? L->top : (ci+1)->func; + if (limit - ci->base >= n && n > 0) /* is 'n' inside 'ci' stack? */ + return "(*temporary)"; + else + return NULL; + } +} + + +LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n) { + CallInfo *ci = L->base_ci + ar->i_ci; + const char *name = findlocal(L, ci, n); + lua_lock(L); + if (name) + luaA_pushobject(L, ci->base + (n - 1)); + lua_unlock(L); + return name; +} + + +LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n) { + CallInfo *ci = L->base_ci + ar->i_ci; + const char *name = findlocal(L, ci, n); + lua_lock(L); + if (name) + setobjs2s(L, ci->base + (n - 1), L->top - 1); + L->top--; /* pop value */ + lua_unlock(L); + return name; +} + + +static void funcinfo (lua_Debug *ar, Closure *cl) { + if (cl->c.isC) { + ar->source = "=[C]"; + ar->linedefined = -1; + ar->lastlinedefined = -1; + ar->what = "C"; + } + else { + ar->source = getstr(cl->l.p->source); + ar->linedefined = cl->l.p->linedefined; + ar->lastlinedefined = cl->l.p->lastlinedefined; + ar->what = (ar->linedefined == 0) ? "main" : "Lua"; + } + luaO_chunkid(ar->short_src, ar->source, LUA_IDSIZE); +} + + +static void info_tailcall (lua_Debug *ar) { + ar->name = ar->namewhat = ""; + ar->what = "tail"; + ar->lastlinedefined = ar->linedefined = ar->currentline = -1; + ar->source = "=(tail call)"; + luaO_chunkid(ar->short_src, ar->source, LUA_IDSIZE); + ar->nups = 0; +} + + +static void collectvalidlines (lua_State *L, Closure *f) { + if (f == NULL || f->c.isC) { + setnilvalue(L->top); + } + else { + Table *t = luaH_new(L, 0, 0); + int *lineinfo = f->l.p->lineinfo; + int i; + for (i=0; il.p->sizelineinfo; i++) + setbvalue(luaH_setnum(L, t, lineinfo[i]), 1); + sethvalue(L, L->top, t); + } + incr_top(L); +} + + +static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar, + Closure *f, CallInfo *ci) { + int status = 1; + if (f == NULL) { + info_tailcall(ar); + return status; + } + for (; *what; what++) { + switch (*what) { + case 'S': { + funcinfo(ar, f); + break; + } + case 'l': { + ar->currentline = (ci) ? currentline(L, ci) : -1; + break; + } + case 'u': { + ar->nups = f->c.nupvalues; + break; + } + case 'n': { + ar->namewhat = (ci) ? getfuncname(L, ci, &ar->name) : NULL; + if (ar->namewhat == NULL) { + ar->namewhat = ""; /* not found */ + ar->name = NULL; + } + break; + } + case 'L': + case 'f': /* handled by lua_getinfo */ + break; + default: status = 0; /* invalid option */ + } + } + return status; +} + + +LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar) { + int status; + Closure *f = NULL; + CallInfo *ci = NULL; + lua_lock(L); + if (*what == '>') { + StkId func = L->top - 1; + luai_apicheck(L, ttisfunction(func)); + what++; /* skip the '>' */ + f = clvalue(func); + L->top--; /* pop function */ + } + else if (ar->i_ci != 0) { /* no tail call? */ + ci = L->base_ci + ar->i_ci; + lua_assert(ttisfunction(ci->func)); + f = clvalue(ci->func); + } + status = auxgetinfo(L, what, ar, f, ci); + if (strchr(what, 'f')) { + if (f == NULL) setnilvalue(L->top); + else setclvalue(L, L->top, f); + incr_top(L); + } + if (strchr(what, 'L')) + collectvalidlines(L, f); + lua_unlock(L); + return status; +} + + +/* +** {====================================================== +** Symbolic Execution and code checker +** ======================================================= +*/ + +#define check(x) if (!(x)) return 0; + +#define checkjump(pt,pc) check(0 <= pc && pc < pt->sizecode) + +#define checkreg(pt,reg) check((reg) < (pt)->maxstacksize) + + + +static int precheck (const Proto *pt) { + check(pt->maxstacksize <= MAXSTACK); + check(pt->numparams+(pt->is_vararg & VARARG_HASARG) <= pt->maxstacksize); + check(!(pt->is_vararg & VARARG_NEEDSARG) || + (pt->is_vararg & VARARG_HASARG)); + check(pt->sizeupvalues <= pt->nups); + check(pt->sizelineinfo == pt->sizecode || pt->sizelineinfo == 0); + check(pt->sizecode > 0 && GET_OPCODE(pt->code[pt->sizecode-1]) == OP_RETURN); + return 1; +} + + +#define checkopenop(pt,pc) luaG_checkopenop((pt)->code[(pc)+1]) + +int luaG_checkopenop (Instruction i) { + switch (GET_OPCODE(i)) { + case OP_CALL: + case OP_TAILCALL: + case OP_RETURN: + case OP_SETLIST: { + check(GETARG_B(i) == 0); + return 1; + } + default: return 0; /* invalid instruction after an open call */ + } +} + + +static int checkArgMode (const Proto *pt, int r, enum OpArgMask mode) { + switch (mode) { + case OpArgN: check(r == 0); break; + case OpArgU: break; + case OpArgR: checkreg(pt, r); break; + case OpArgK: + check(ISK(r) ? INDEXK(r) < pt->sizek : r < pt->maxstacksize); + break; + } + return 1; +} + + +static Instruction symbexec (const Proto *pt, int lastpc, int reg) { + int pc; + int last; /* stores position of last instruction that changed `reg' */ + last = pt->sizecode-1; /* points to final return (a `neutral' instruction) */ + check(precheck(pt)); + for (pc = 0; pc < lastpc; pc++) { + Instruction i = pt->code[pc]; + OpCode op = GET_OPCODE(i); + int a = GETARG_A(i); + int b = 0; + int c = 0; + check(op < NUM_OPCODES); + checkreg(pt, a); + switch (getOpMode(op)) { + case iABC: { + b = GETARG_B(i); + c = GETARG_C(i); + check(checkArgMode(pt, b, getBMode(op))); + check(checkArgMode(pt, c, getCMode(op))); + break; + } + case iABx: { + b = GETARG_Bx(i); + if (getBMode(op) == OpArgK) check(b < pt->sizek); + break; + } + case iAsBx: { + b = GETARG_sBx(i); + if (getBMode(op) == OpArgR) { + int dest = pc+1+b; + check(0 <= dest && dest < pt->sizecode); + if (dest > 0) { + int j; + /* check that it does not jump to a setlist count; this + is tricky, because the count from a previous setlist may + have the same value of an invalid setlist; so, we must + go all the way back to the first of them (if any) */ + for (j = 0; j < dest; j++) { + Instruction d = pt->code[dest-1-j]; + if (!(GET_OPCODE(d) == OP_SETLIST && GETARG_C(d) == 0)) break; + } + /* if 'j' is even, previous value is not a setlist (even if + it looks like one) */ + check((j&1) == 0); + } + } + break; + } + } + if (testAMode(op)) { + if (a == reg) last = pc; /* change register `a' */ + } + if (testTMode(op)) { + check(pc+2 < pt->sizecode); /* check skip */ + check(GET_OPCODE(pt->code[pc+1]) == OP_JMP); + } + switch (op) { + case OP_LOADBOOL: { + if (c == 1) { /* does it jump? */ + check(pc+2 < pt->sizecode); /* check its jump */ + check(GET_OPCODE(pt->code[pc+1]) != OP_SETLIST || + GETARG_C(pt->code[pc+1]) != 0); + } + break; + } + case OP_LOADNIL: { + if (a <= reg && reg <= b) + last = pc; /* set registers from `a' to `b' */ + break; + } + case OP_GETUPVAL: + case OP_SETUPVAL: { + check(b < pt->nups); + break; + } + case OP_GETGLOBAL: + case OP_SETGLOBAL: { + check(ttisstring(&pt->k[b])); + break; + } + case OP_SELF: { + checkreg(pt, a+1); + if (reg == a+1) last = pc; + break; + } + case OP_CONCAT: { + check(b < c); /* at least two operands */ + break; + } + case OP_TFORLOOP: { + check(c >= 1); /* at least one result (control variable) */ + checkreg(pt, a+2+c); /* space for results */ + if (reg >= a+2) last = pc; /* affect all regs above its base */ + break; + } + case OP_FORLOOP: + case OP_FORPREP: + checkreg(pt, a+3); + /* go through */ + case OP_JMP: { + int dest = pc+1+b; + /* not full check and jump is forward and do not skip `lastpc'? */ + if (reg != NO_REG && pc < dest && dest <= lastpc) + pc += b; /* do the jump */ + break; + } + case OP_CALL: + case OP_TAILCALL: { + if (b != 0) { + checkreg(pt, a+b-1); + } + c--; /* c = num. returns */ + if (c == LUA_MULTRET) { + check(checkopenop(pt, pc)); + } + else if (c != 0) + checkreg(pt, a+c-1); + if (reg >= a) last = pc; /* affect all registers above base */ + break; + } + case OP_RETURN: { + b--; /* b = num. returns */ + if (b > 0) checkreg(pt, a+b-1); + break; + } + case OP_SETLIST: { + if (b > 0) checkreg(pt, a + b); + if (c == 0) { + pc++; + check(pc < pt->sizecode - 1); + } + break; + } + case OP_CLOSURE: { + int nup, j; + check(b < pt->sizep); + nup = pt->p[b]->nups; + check(pc + nup < pt->sizecode); + for (j = 1; j <= nup; j++) { + OpCode op1 = GET_OPCODE(pt->code[pc + j]); + check(op1 == OP_GETUPVAL || op1 == OP_MOVE); + } + if (reg != NO_REG) /* tracing? */ + pc += nup; /* do not 'execute' these pseudo-instructions */ + break; + } + case OP_VARARG: { + check((pt->is_vararg & VARARG_ISVARARG) && + !(pt->is_vararg & VARARG_NEEDSARG)); + b--; + if (b == LUA_MULTRET) check(checkopenop(pt, pc)); + checkreg(pt, a+b-1); + break; + } + default: break; + } + } + return pt->code[last]; +} + +#undef check +#undef checkjump +#undef checkreg + +/* }====================================================== */ + + +int luaG_checkcode (const Proto *pt) { + return (symbexec(pt, pt->sizecode, NO_REG) != 0); +} + + +static const char *kname (Proto *p, int c) { + if (ISK(c) && ttisstring(&p->k[INDEXK(c)])) + return svalue(&p->k[INDEXK(c)]); + else + return "?"; +} + + +static const char *getobjname (lua_State *L, CallInfo *ci, int stackpos, + const char **name) { + if (isLua(ci)) { /* a Lua function? */ + Proto *p = ci_func(ci)->l.p; + int pc = currentpc(L, ci); + Instruction i; + *name = luaF_getlocalname(p, stackpos+1, pc); + if (*name) /* is a local? */ + return "local"; + i = symbexec(p, pc, stackpos); /* try symbolic execution */ + lua_assert(pc != -1); + switch (GET_OPCODE(i)) { + case OP_GETGLOBAL: { + int g = GETARG_Bx(i); /* global index */ + lua_assert(ttisstring(&p->k[g])); + *name = svalue(&p->k[g]); + return "global"; + } + case OP_MOVE: { + int a = GETARG_A(i); + int b = GETARG_B(i); /* move from `b' to `a' */ + if (b < a) + return getobjname(L, ci, b, name); /* get name for `b' */ + break; + } + case OP_GETTABLE: { + int k = GETARG_C(i); /* key index */ + *name = kname(p, k); + return "field"; + } + case OP_GETUPVAL: { + int u = GETARG_B(i); /* upvalue index */ + *name = p->upvalues ? getstr(p->upvalues[u]) : "?"; + return "upvalue"; + } + case OP_SELF: { + int k = GETARG_C(i); /* key index */ + *name = kname(p, k); + return "method"; + } + default: break; + } + } + return NULL; /* no useful name found */ +} + + +static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name) { + Instruction i; + if ((isLua(ci) && ci->tailcalls > 0) || !isLua(ci - 1)) + return NULL; /* calling function is not Lua (or is unknown) */ + ci--; /* calling function */ + i = ci_func(ci)->l.p->code[currentpc(L, ci)]; + if (GET_OPCODE(i) == OP_CALL || GET_OPCODE(i) == OP_TAILCALL || + GET_OPCODE(i) == OP_TFORLOOP) + return getobjname(L, ci, GETARG_A(i), name); + else + return NULL; /* no useful name can be found */ +} + + +/* only ANSI way to check whether a pointer points to an array */ +static int isinstack (CallInfo *ci, const TValue *o) { + StkId p; + for (p = ci->base; p < ci->top; p++) + if (o == p) return 1; + return 0; +} + + +void luaG_typeerror (lua_State *L, const TValue *o, const char *op) { + const char *name = NULL; + const char *t = luaT_typenames[ttype(o)]; + const char *kind = (isinstack(L->ci, o)) ? + getobjname(L, L->ci, cast_int(o - L->base), &name) : + NULL; + if (kind) + luaG_runerror(L, "attempt to %s %s " LUA_QS " (a %s value)", + op, kind, name, t); + else + luaG_runerror(L, "attempt to %s a %s value", op, t); +} + + +void luaG_concaterror (lua_State *L, StkId p1, StkId p2) { + if (ttisstring(p1) || ttisnumber(p1)) p1 = p2; + lua_assert(!ttisstring(p1) && !ttisnumber(p1)); + luaG_typeerror(L, p1, "concatenate"); +} + + +void luaG_aritherror (lua_State *L, const TValue *p1, const TValue *p2) { + TValue temp; + if (luaV_tonumber(p1, &temp) == NULL) + p2 = p1; /* first operand is wrong */ + luaG_typeerror(L, p2, "perform arithmetic on"); +} + + +int luaG_ordererror (lua_State *L, const TValue *p1, const TValue *p2) { + const char *t1 = luaT_typenames[ttype(p1)]; + const char *t2 = luaT_typenames[ttype(p2)]; + if (t1[2] == t2[2]) + luaG_runerror(L, "attempt to compare two %s values", t1); + else + luaG_runerror(L, "attempt to compare %s with %s", t1, t2); + return 0; +} + + +static void addinfo (lua_State *L, const char *msg) { + CallInfo *ci = L->ci; + if (isLua(ci)) { /* is Lua code? */ + char buff[LUA_IDSIZE]; /* add file:line information */ + int line = currentline(L, ci); + luaO_chunkid(buff, getstr(getluaproto(ci)->source), LUA_IDSIZE); + luaO_pushfstring(L, "%s:%d: %s", buff, line, msg); + } +} + + +void luaG_errormsg (lua_State *L) { + if (L->errfunc != 0) { /* is there an error handling function? */ + StkId errfunc = restorestack(L, L->errfunc); + if (!ttisfunction(errfunc)) luaD_throw(L, LUA_ERRERR); + setobjs2s(L, L->top, L->top - 1); /* move argument */ + setobjs2s(L, L->top - 1, errfunc); /* push function */ + incr_top(L); + luaD_call(L, L->top - 2, 1); /* call it */ + } + luaD_throw(L, LUA_ERRRUN); +} + + +void luaG_runerror (lua_State *L, const char *fmt, ...) { + va_list argp; + va_start(argp, fmt); + addinfo(L, luaO_pushvfstring(L, fmt, argp)); + va_end(argp); + luaG_errormsg(L); +} + diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h new file mode 100644 index 0000000..ba28a97 --- /dev/null +++ b/src/lua/ldebug.h @@ -0,0 +1,33 @@ +/* +** $Id: ldebug.h,v 2.3.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions from Debug Interface module +** See Copyright Notice in lua.h +*/ + +#ifndef ldebug_h +#define ldebug_h + + +#include "lstate.h" + + +#define pcRel(pc, p) (cast(int, (pc) - (p)->code) - 1) + +#define getline(f,pc) (((f)->lineinfo) ? (f)->lineinfo[pc] : 0) + +#define resethookcount(L) (L->hookcount = L->basehookcount) + + +LUAI_FUNC void luaG_typeerror (lua_State *L, const TValue *o, + const char *opname); +LUAI_FUNC void luaG_concaterror (lua_State *L, StkId p1, StkId p2); +LUAI_FUNC void luaG_aritherror (lua_State *L, const TValue *p1, + const TValue *p2); +LUAI_FUNC int luaG_ordererror (lua_State *L, const TValue *p1, + const TValue *p2); +LUAI_FUNC void luaG_runerror (lua_State *L, const char *fmt, ...); +LUAI_FUNC void luaG_errormsg (lua_State *L); +LUAI_FUNC int luaG_checkcode (const Proto *pt); +LUAI_FUNC int luaG_checkopenop (Instruction i); + +#endif diff --git a/src/lua/ldo.c b/src/lua/ldo.c new file mode 100644 index 0000000..d1bf786 --- /dev/null +++ b/src/lua/ldo.c @@ -0,0 +1,519 @@ +/* +** $Id: ldo.c,v 2.38.1.4 2012/01/18 02:27:10 roberto Exp $ +** Stack and Call structure of Lua +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define ldo_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lundump.h" +#include "lvm.h" +#include "lzio.h" + + + + +/* +** {====================================================== +** Error-recovery functions +** ======================================================= +*/ + + +/* chain list of long jump buffers */ +struct lua_longjmp { + struct lua_longjmp *previous; + luai_jmpbuf b; + volatile int status; /* error code */ +}; + + +void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop) { + switch (errcode) { + case LUA_ERRMEM: { + setsvalue2s(L, oldtop, luaS_newliteral(L, MEMERRMSG)); + break; + } + case LUA_ERRERR: { + setsvalue2s(L, oldtop, luaS_newliteral(L, "error in error handling")); + break; + } + case LUA_ERRSYNTAX: + case LUA_ERRRUN: { + setobjs2s(L, oldtop, L->top - 1); /* error message on current top */ + break; + } + } + L->top = oldtop + 1; +} + + +static void restore_stack_limit (lua_State *L) { + lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK - 1); + if (L->size_ci > LUAI_MAXCALLS) { /* there was an overflow? */ + int inuse = cast_int(L->ci - L->base_ci); + if (inuse + 1 < LUAI_MAXCALLS) /* can `undo' overflow? */ + luaD_reallocCI(L, LUAI_MAXCALLS); + } +} + + +static void resetstack (lua_State *L, int status) { + L->ci = L->base_ci; + L->base = L->ci->base; + luaF_close(L, L->base); /* close eventual pending closures */ + luaD_seterrorobj(L, status, L->base); + L->nCcalls = L->baseCcalls; + L->allowhook = 1; + restore_stack_limit(L); + L->errfunc = 0; + L->errorJmp = NULL; +} + + +void luaD_throw (lua_State *L, int errcode) { + if (L->errorJmp) { + L->errorJmp->status = errcode; + LUAI_THROW(L, L->errorJmp); + } + else { + L->status = cast_byte(errcode); + if (G(L)->panic) { + resetstack(L, errcode); + lua_unlock(L); + G(L)->panic(L); + } + exit(EXIT_FAILURE); + } +} + + +int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { + struct lua_longjmp lj; + lj.status = 0; + lj.previous = L->errorJmp; /* chain new error handler */ + L->errorJmp = &lj; + LUAI_TRY(L, &lj, + (*f)(L, ud); + ); + L->errorJmp = lj.previous; /* restore old error handler */ + return lj.status; +} + +/* }====================================================== */ + + +static void correctstack (lua_State *L, TValue *oldstack) { + CallInfo *ci; + GCObject *up; + L->top = (L->top - oldstack) + L->stack; + for (up = L->openupval; up != NULL; up = up->gch.next) + gco2uv(up)->v = (gco2uv(up)->v - oldstack) + L->stack; + for (ci = L->base_ci; ci <= L->ci; ci++) { + ci->top = (ci->top - oldstack) + L->stack; + ci->base = (ci->base - oldstack) + L->stack; + ci->func = (ci->func - oldstack) + L->stack; + } + L->base = (L->base - oldstack) + L->stack; +} + + +void luaD_reallocstack (lua_State *L, int newsize) { + TValue *oldstack = L->stack; + int realsize = newsize + 1 + EXTRA_STACK; + lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK - 1); + luaM_reallocvector(L, L->stack, L->stacksize, realsize, TValue); + L->stacksize = realsize; + L->stack_last = L->stack+newsize; + correctstack(L, oldstack); +} + + +void luaD_reallocCI (lua_State *L, int newsize) { + CallInfo *oldci = L->base_ci; + luaM_reallocvector(L, L->base_ci, L->size_ci, newsize, CallInfo); + L->size_ci = newsize; + L->ci = (L->ci - oldci) + L->base_ci; + L->end_ci = L->base_ci + L->size_ci - 1; +} + + +void luaD_growstack (lua_State *L, int n) { + if (n <= L->stacksize) /* double size is enough? */ + luaD_reallocstack(L, 2*L->stacksize); + else + luaD_reallocstack(L, L->stacksize + n); +} + + +static CallInfo *growCI (lua_State *L) { + if (L->size_ci > LUAI_MAXCALLS) /* overflow while handling overflow? */ + luaD_throw(L, LUA_ERRERR); + else { + luaD_reallocCI(L, 2*L->size_ci); + if (L->size_ci > LUAI_MAXCALLS) + luaG_runerror(L, "stack overflow"); + } + return ++L->ci; +} + + +void luaD_callhook (lua_State *L, int event, int line) { + lua_Hook hook = L->hook; + if (hook && L->allowhook) { + ptrdiff_t top = savestack(L, L->top); + ptrdiff_t ci_top = savestack(L, L->ci->top); + lua_Debug ar; + ar.event = event; + ar.currentline = line; + if (event == LUA_HOOKTAILRET) + ar.i_ci = 0; /* tail call; no debug information about it */ + else + ar.i_ci = cast_int(L->ci - L->base_ci); + luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */ + L->ci->top = L->top + LUA_MINSTACK; + lua_assert(L->ci->top <= L->stack_last); + L->allowhook = 0; /* cannot call hooks inside a hook */ + lua_unlock(L); + (*hook)(L, &ar); + lua_lock(L); + lua_assert(!L->allowhook); + L->allowhook = 1; + L->ci->top = restorestack(L, ci_top); + L->top = restorestack(L, top); + } +} + + +static StkId adjust_varargs (lua_State *L, Proto *p, int actual) { + int i; + int nfixargs = p->numparams; + Table *htab = NULL; + StkId base, fixed; + for (; actual < nfixargs; ++actual) + setnilvalue(L->top++); +#if defined(LUA_COMPAT_VARARG) + if (p->is_vararg & VARARG_NEEDSARG) { /* compat. with old-style vararg? */ + int nvar = actual - nfixargs; /* number of extra arguments */ + lua_assert(p->is_vararg & VARARG_HASARG); + luaC_checkGC(L); + luaD_checkstack(L, p->maxstacksize); + htab = luaH_new(L, nvar, 1); /* create `arg' table */ + for (i=0; itop - nvar + i); + /* store counter in field `n' */ + setnvalue(luaH_setstr(L, htab, luaS_newliteral(L, "n")), cast_num(nvar)); + } +#endif + /* move fixed parameters to final position */ + fixed = L->top - actual; /* first fixed argument */ + base = L->top; /* final position of first argument */ + for (i=0; itop++, fixed+i); + setnilvalue(fixed+i); + } + /* add `arg' parameter */ + if (htab) { + sethvalue(L, L->top++, htab); + lua_assert(iswhite(obj2gco(htab))); + } + return base; +} + + +static StkId tryfuncTM (lua_State *L, StkId func) { + const TValue *tm = luaT_gettmbyobj(L, func, TM_CALL); + StkId p; + ptrdiff_t funcr = savestack(L, func); + if (!ttisfunction(tm)) + luaG_typeerror(L, func, "call"); + /* Open a hole inside the stack at `func' */ + for (p = L->top; p > func; p--) setobjs2s(L, p, p-1); + incr_top(L); + func = restorestack(L, funcr); /* previous call may change stack */ + setobj2s(L, func, tm); /* tag method is the new function to be called */ + return func; +} + + + +#define inc_ci(L) \ + ((L->ci == L->end_ci) ? growCI(L) : \ + (condhardstacktests(luaD_reallocCI(L, L->size_ci)), ++L->ci)) + + +int luaD_precall (lua_State *L, StkId func, int nresults) { + LClosure *cl; + ptrdiff_t funcr; + if (!ttisfunction(func)) /* `func' is not a function? */ + func = tryfuncTM(L, func); /* check the `function' tag method */ + funcr = savestack(L, func); + cl = &clvalue(func)->l; + L->ci->savedpc = L->savedpc; + if (!cl->isC) { /* Lua function? prepare its call */ + CallInfo *ci; + StkId st, base; + Proto *p = cl->p; + luaD_checkstack(L, p->maxstacksize); + func = restorestack(L, funcr); + if (!p->is_vararg) { /* no varargs? */ + base = func + 1; + if (L->top > base + p->numparams) + L->top = base + p->numparams; + } + else { /* vararg function */ + int nargs = cast_int(L->top - func) - 1; + base = adjust_varargs(L, p, nargs); + func = restorestack(L, funcr); /* previous call may change the stack */ + } + ci = inc_ci(L); /* now `enter' new function */ + ci->func = func; + L->base = ci->base = base; + ci->top = L->base + p->maxstacksize; + lua_assert(ci->top <= L->stack_last); + L->savedpc = p->code; /* starting point */ + ci->tailcalls = 0; + ci->nresults = nresults; + for (st = L->top; st < ci->top; st++) + setnilvalue(st); + L->top = ci->top; + if (L->hookmask & LUA_MASKCALL) { + L->savedpc++; /* hooks assume 'pc' is already incremented */ + luaD_callhook(L, LUA_HOOKCALL, -1); + L->savedpc--; /* correct 'pc' */ + } + return PCRLUA; + } + else { /* if is a C function, call it */ + CallInfo *ci; + int n; + luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */ + ci = inc_ci(L); /* now `enter' new function */ + ci->func = restorestack(L, funcr); + L->base = ci->base = ci->func + 1; + ci->top = L->top + LUA_MINSTACK; + lua_assert(ci->top <= L->stack_last); + ci->nresults = nresults; + if (L->hookmask & LUA_MASKCALL) + luaD_callhook(L, LUA_HOOKCALL, -1); + lua_unlock(L); + n = (*curr_func(L)->c.f)(L); /* do the actual call */ + lua_lock(L); + if (n < 0) /* yielding? */ + return PCRYIELD; + else { + luaD_poscall(L, L->top - n); + return PCRC; + } + } +} + + +static StkId callrethooks (lua_State *L, StkId firstResult) { + ptrdiff_t fr = savestack(L, firstResult); /* next call may change stack */ + luaD_callhook(L, LUA_HOOKRET, -1); + if (f_isLua(L->ci)) { /* Lua function? */ + while ((L->hookmask & LUA_MASKRET) && L->ci->tailcalls--) /* tail calls */ + luaD_callhook(L, LUA_HOOKTAILRET, -1); + } + return restorestack(L, fr); +} + + +int luaD_poscall (lua_State *L, StkId firstResult) { + StkId res; + int wanted, i; + CallInfo *ci; + if (L->hookmask & LUA_MASKRET) + firstResult = callrethooks(L, firstResult); + ci = L->ci--; + res = ci->func; /* res == final position of 1st result */ + wanted = ci->nresults; + L->base = (ci - 1)->base; /* restore base */ + L->savedpc = (ci - 1)->savedpc; /* restore savedpc */ + /* move results to correct place */ + for (i = wanted; i != 0 && firstResult < L->top; i--) + setobjs2s(L, res++, firstResult++); + while (i-- > 0) + setnilvalue(res++); + L->top = res; + return (wanted - LUA_MULTRET); /* 0 iff wanted == LUA_MULTRET */ +} + + +/* +** Call a function (C or Lua). The function to be called is at *func. +** The arguments are on the stack, right after the function. +** When returns, all the results are on the stack, starting at the original +** function position. +*/ +void luaD_call (lua_State *L, StkId func, int nResults) { + if (++L->nCcalls >= LUAI_MAXCCALLS) { + if (L->nCcalls == LUAI_MAXCCALLS) + luaG_runerror(L, "C stack overflow"); + else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS>>3))) + luaD_throw(L, LUA_ERRERR); /* error while handing stack error */ + } + if (luaD_precall(L, func, nResults) == PCRLUA) /* is a Lua function? */ + luaV_execute(L, 1); /* call it */ + L->nCcalls--; + luaC_checkGC(L); +} + + +static void resume (lua_State *L, void *ud) { + StkId firstArg = cast(StkId, ud); + CallInfo *ci = L->ci; + if (L->status == 0) { /* start coroutine? */ + lua_assert(ci == L->base_ci && firstArg > L->base); + if (luaD_precall(L, firstArg - 1, LUA_MULTRET) != PCRLUA) + return; + } + else { /* resuming from previous yield */ + lua_assert(L->status == LUA_YIELD); + L->status = 0; + if (!f_isLua(ci)) { /* `common' yield? */ + /* finish interrupted execution of `OP_CALL' */ + lua_assert(GET_OPCODE(*((ci-1)->savedpc - 1)) == OP_CALL || + GET_OPCODE(*((ci-1)->savedpc - 1)) == OP_TAILCALL); + if (luaD_poscall(L, firstArg)) /* complete it... */ + L->top = L->ci->top; /* and correct top if not multiple results */ + } + else /* yielded inside a hook: just continue its execution */ + L->base = L->ci->base; + } + luaV_execute(L, cast_int(L->ci - L->base_ci)); +} + + +static int resume_error (lua_State *L, const char *msg) { + L->top = L->ci->base; + setsvalue2s(L, L->top, luaS_new(L, msg)); + incr_top(L); + lua_unlock(L); + return LUA_ERRRUN; +} + + +LUA_API int lua_resume (lua_State *L, int nargs) { + int status; + lua_lock(L); + if (L->status != LUA_YIELD && (L->status != 0 || L->ci != L->base_ci)) + return resume_error(L, "cannot resume non-suspended coroutine"); + if (L->nCcalls >= LUAI_MAXCCALLS) + return resume_error(L, "C stack overflow"); + luai_userstateresume(L, nargs); + lua_assert(L->errfunc == 0); + L->baseCcalls = ++L->nCcalls; + status = luaD_rawrunprotected(L, resume, L->top - nargs); + if (status != 0) { /* error? */ + L->status = cast_byte(status); /* mark thread as `dead' */ + luaD_seterrorobj(L, status, L->top); + L->ci->top = L->top; + } + else { + lua_assert(L->nCcalls == L->baseCcalls); + status = L->status; + } + --L->nCcalls; + lua_unlock(L); + return status; +} + + +LUA_API int lua_yield (lua_State *L, int nresults) { + luai_userstateyield(L, nresults); + lua_lock(L); + if (L->nCcalls > L->baseCcalls) + luaG_runerror(L, "attempt to yield across metamethod/C-call boundary"); + L->base = L->top - nresults; /* protect stack slots below */ + L->status = LUA_YIELD; + lua_unlock(L); + return -1; +} + + +int luaD_pcall (lua_State *L, Pfunc func, void *u, + ptrdiff_t old_top, ptrdiff_t ef) { + int status; + unsigned short oldnCcalls = L->nCcalls; + ptrdiff_t old_ci = saveci(L, L->ci); + lu_byte old_allowhooks = L->allowhook; + ptrdiff_t old_errfunc = L->errfunc; + L->errfunc = ef; + status = luaD_rawrunprotected(L, func, u); + if (status != 0) { /* an error occurred? */ + StkId oldtop = restorestack(L, old_top); + luaF_close(L, oldtop); /* close eventual pending closures */ + luaD_seterrorobj(L, status, oldtop); + L->nCcalls = oldnCcalls; + L->ci = restoreci(L, old_ci); + L->base = L->ci->base; + L->savedpc = L->ci->savedpc; + L->allowhook = old_allowhooks; + restore_stack_limit(L); + } + L->errfunc = old_errfunc; + return status; +} + + + +/* +** Execute a protected parser. +*/ +struct SParser { /* data to `f_parser' */ + ZIO *z; + Mbuffer buff; /* buffer to be used by the scanner */ + const char *name; +}; + +static void f_parser (lua_State *L, void *ud) { + int i; + Proto *tf; + Closure *cl; + struct SParser *p = cast(struct SParser *, ud); + int c = luaZ_lookahead(p->z); + luaC_checkGC(L); + tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z, + &p->buff, p->name); + cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L))); + cl->l.p = tf; + for (i = 0; i < tf->nups; i++) /* initialize eventual upvalues */ + cl->l.upvals[i] = luaF_newupval(L); + setclvalue(L, L->top, cl); + incr_top(L); +} + + +int luaD_protectedparser (lua_State *L, ZIO *z, const char *name) { + struct SParser p; + int status; + p.z = z; p.name = name; + luaZ_initbuffer(L, &p.buff); + status = luaD_pcall(L, f_parser, &p, savestack(L, L->top), L->errfunc); + luaZ_freebuffer(L, &p.buff); + return status; +} + + diff --git a/src/lua/ldo.h b/src/lua/ldo.h new file mode 100644 index 0000000..98fddac --- /dev/null +++ b/src/lua/ldo.h @@ -0,0 +1,57 @@ +/* +** $Id: ldo.h,v 2.7.1.1 2007/12/27 13:02:25 roberto Exp $ +** Stack and Call structure of Lua +** See Copyright Notice in lua.h +*/ + +#ifndef ldo_h +#define ldo_h + + +#include "lobject.h" +#include "lstate.h" +#include "lzio.h" + + +#define luaD_checkstack(L,n) \ + if ((char *)L->stack_last - (char *)L->top <= (n)*(int)sizeof(TValue)) \ + luaD_growstack(L, n); \ + else condhardstacktests(luaD_reallocstack(L, L->stacksize - EXTRA_STACK - 1)); + + +#define incr_top(L) {luaD_checkstack(L,1); L->top++;} + +#define savestack(L,p) ((char *)(p) - (char *)L->stack) +#define restorestack(L,n) ((TValue *)((char *)L->stack + (n))) + +#define saveci(L,p) ((char *)(p) - (char *)L->base_ci) +#define restoreci(L,n) ((CallInfo *)((char *)L->base_ci + (n))) + + +/* results from luaD_precall */ +#define PCRLUA 0 /* initiated a call to a Lua function */ +#define PCRC 1 /* did a call to a C function */ +#define PCRYIELD 2 /* C funtion yielded */ + + +/* type of protected functions, to be ran by `runprotected' */ +typedef void (*Pfunc) (lua_State *L, void *ud); + +LUAI_FUNC int luaD_protectedparser (lua_State *L, ZIO *z, const char *name); +LUAI_FUNC void luaD_callhook (lua_State *L, int event, int line); +LUAI_FUNC int luaD_precall (lua_State *L, StkId func, int nresults); +LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults); +LUAI_FUNC int luaD_pcall (lua_State *L, Pfunc func, void *u, + ptrdiff_t oldtop, ptrdiff_t ef); +LUAI_FUNC int luaD_poscall (lua_State *L, StkId firstResult); +LUAI_FUNC void luaD_reallocCI (lua_State *L, int newsize); +LUAI_FUNC void luaD_reallocstack (lua_State *L, int newsize); +LUAI_FUNC void luaD_growstack (lua_State *L, int n); + +LUAI_FUNC void luaD_throw (lua_State *L, int errcode); +LUAI_FUNC int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud); + +LUAI_FUNC void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop); + +#endif + diff --git a/src/lua/ldump.c b/src/lua/ldump.c new file mode 100644 index 0000000..c9d3d48 --- /dev/null +++ b/src/lua/ldump.c @@ -0,0 +1,164 @@ +/* +** $Id: ldump.c,v 2.8.1.1 2007/12/27 13:02:25 roberto Exp $ +** save precompiled Lua chunks +** See Copyright Notice in lua.h +*/ + +#include + +#define ldump_c +#define LUA_CORE + +#include "lua.h" + +#include "lobject.h" +#include "lstate.h" +#include "lundump.h" + +typedef struct { + lua_State* L; + lua_Writer writer; + void* data; + int strip; + int status; +} DumpState; + +#define DumpMem(b,n,size,D) DumpBlock(b,(n)*(size),D) +#define DumpVar(x,D) DumpMem(&x,1,sizeof(x),D) + +static void DumpBlock(const void* b, size_t size, DumpState* D) +{ + if (D->status==0) + { + lua_unlock(D->L); + D->status=(*D->writer)(D->L,b,size,D->data); + lua_lock(D->L); + } +} + +static void DumpChar(int y, DumpState* D) +{ + char x=(char)y; + DumpVar(x,D); +} + +static void DumpInt(int x, DumpState* D) +{ + DumpVar(x,D); +} + +static void DumpNumber(lua_Number x, DumpState* D) +{ + DumpVar(x,D); +} + +static void DumpVector(const void* b, int n, size_t size, DumpState* D) +{ + DumpInt(n,D); + DumpMem(b,n,size,D); +} + +static void DumpString(const TString* s, DumpState* D) +{ + if (s==NULL || getstr(s)==NULL) + { + size_t size=0; + DumpVar(size,D); + } + else + { + size_t size=s->tsv.len+1; /* include trailing '\0' */ + DumpVar(size,D); + DumpBlock(getstr(s),size,D); + } +} + +#define DumpCode(f,D) DumpVector(f->code,f->sizecode,sizeof(Instruction),D) + +static void DumpFunction(const Proto* f, const TString* p, DumpState* D); + +static void DumpConstants(const Proto* f, DumpState* D) +{ + int i,n=f->sizek; + DumpInt(n,D); + for (i=0; ik[i]; + DumpChar(ttype(o),D); + switch (ttype(o)) + { + case LUA_TNIL: + break; + case LUA_TBOOLEAN: + DumpChar(bvalue(o),D); + break; + case LUA_TNUMBER: + DumpNumber(nvalue(o),D); + break; + case LUA_TSTRING: + DumpString(rawtsvalue(o),D); + break; + default: + lua_assert(0); /* cannot happen */ + break; + } + } + n=f->sizep; + DumpInt(n,D); + for (i=0; ip[i],f->source,D); +} + +static void DumpDebug(const Proto* f, DumpState* D) +{ + int i,n; + n= (D->strip) ? 0 : f->sizelineinfo; + DumpVector(f->lineinfo,n,sizeof(int),D); + n= (D->strip) ? 0 : f->sizelocvars; + DumpInt(n,D); + for (i=0; ilocvars[i].varname,D); + DumpInt(f->locvars[i].startpc,D); + DumpInt(f->locvars[i].endpc,D); + } + n= (D->strip) ? 0 : f->sizeupvalues; + DumpInt(n,D); + for (i=0; iupvalues[i],D); +} + +static void DumpFunction(const Proto* f, const TString* p, DumpState* D) +{ + DumpString((f->source==p || D->strip) ? NULL : f->source,D); + DumpInt(f->linedefined,D); + DumpInt(f->lastlinedefined,D); + DumpChar(f->nups,D); + DumpChar(f->numparams,D); + DumpChar(f->is_vararg,D); + DumpChar(f->maxstacksize,D); + DumpCode(f,D); + DumpConstants(f,D); + DumpDebug(f,D); +} + +static void DumpHeader(DumpState* D) +{ + char h[LUAC_HEADERSIZE]; + luaU_header(h); + DumpBlock(h,LUAC_HEADERSIZE,D); +} + +/* +** dump Lua function as precompiled chunk +*/ +int luaU_dump (lua_State* L, const Proto* f, lua_Writer w, void* data, int strip) +{ + DumpState D; + D.L=L; + D.writer=w; + D.data=data; + D.strip=strip; + D.status=0; + DumpHeader(&D); + DumpFunction(f,NULL,&D); + return D.status; +} diff --git a/src/lua/lfunc.c b/src/lua/lfunc.c new file mode 100644 index 0000000..813e88f --- /dev/null +++ b/src/lua/lfunc.c @@ -0,0 +1,174 @@ +/* +** $Id: lfunc.c,v 2.12.1.2 2007/12/28 14:58:43 roberto Exp $ +** Auxiliary functions to manipulate prototypes and closures +** See Copyright Notice in lua.h +*/ + + +#include + +#define lfunc_c +#define LUA_CORE + +#include "lua.h" + +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" + + + +Closure *luaF_newCclosure (lua_State *L, int nelems, Table *e) { + Closure *c = cast(Closure *, luaM_malloc(L, sizeCclosure(nelems))); + luaC_link(L, obj2gco(c), LUA_TFUNCTION); + c->c.isC = 1; + c->c.env = e; + c->c.nupvalues = cast_byte(nelems); + return c; +} + + +Closure *luaF_newLclosure (lua_State *L, int nelems, Table *e) { + Closure *c = cast(Closure *, luaM_malloc(L, sizeLclosure(nelems))); + luaC_link(L, obj2gco(c), LUA_TFUNCTION); + c->l.isC = 0; + c->l.env = e; + c->l.nupvalues = cast_byte(nelems); + while (nelems--) c->l.upvals[nelems] = NULL; + return c; +} + + +UpVal *luaF_newupval (lua_State *L) { + UpVal *uv = luaM_new(L, UpVal); + luaC_link(L, obj2gco(uv), LUA_TUPVAL); + uv->v = &uv->u.value; + setnilvalue(uv->v); + return uv; +} + + +UpVal *luaF_findupval (lua_State *L, StkId level) { + global_State *g = G(L); + GCObject **pp = &L->openupval; + UpVal *p; + UpVal *uv; + while (*pp != NULL && (p = ngcotouv(*pp))->v >= level) { + lua_assert(p->v != &p->u.value); + if (p->v == level) { /* found a corresponding upvalue? */ + if (isdead(g, obj2gco(p))) /* is it dead? */ + changewhite(obj2gco(p)); /* ressurect it */ + return p; + } + pp = &p->next; + } + uv = luaM_new(L, UpVal); /* not found: create a new one */ + uv->tt = LUA_TUPVAL; + uv->marked = luaC_white(g); + uv->v = level; /* current value lives in the stack */ + uv->next = *pp; /* chain it in the proper position */ + *pp = obj2gco(uv); + uv->u.l.prev = &g->uvhead; /* double link it in `uvhead' list */ + uv->u.l.next = g->uvhead.u.l.next; + uv->u.l.next->u.l.prev = uv; + g->uvhead.u.l.next = uv; + lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); + return uv; +} + + +static void unlinkupval (UpVal *uv) { + lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); + uv->u.l.next->u.l.prev = uv->u.l.prev; /* remove from `uvhead' list */ + uv->u.l.prev->u.l.next = uv->u.l.next; +} + + +void luaF_freeupval (lua_State *L, UpVal *uv) { + if (uv->v != &uv->u.value) /* is it open? */ + unlinkupval(uv); /* remove from open list */ + luaM_free(L, uv); /* free upvalue */ +} + + +void luaF_close (lua_State *L, StkId level) { + UpVal *uv; + global_State *g = G(L); + while (L->openupval != NULL && (uv = ngcotouv(L->openupval))->v >= level) { + GCObject *o = obj2gco(uv); + lua_assert(!isblack(o) && uv->v != &uv->u.value); + L->openupval = uv->next; /* remove from `open' list */ + if (isdead(g, o)) + luaF_freeupval(L, uv); /* free upvalue */ + else { + unlinkupval(uv); + setobj(L, &uv->u.value, uv->v); + uv->v = &uv->u.value; /* now current value lives here */ + luaC_linkupval(L, uv); /* link upvalue into `gcroot' list */ + } + } +} + + +Proto *luaF_newproto (lua_State *L) { + Proto *f = luaM_new(L, Proto); + luaC_link(L, obj2gco(f), LUA_TPROTO); + f->k = NULL; + f->sizek = 0; + f->p = NULL; + f->sizep = 0; + f->code = NULL; + f->sizecode = 0; + f->sizelineinfo = 0; + f->sizeupvalues = 0; + f->nups = 0; + f->upvalues = NULL; + f->numparams = 0; + f->is_vararg = 0; + f->maxstacksize = 0; + f->lineinfo = NULL; + f->sizelocvars = 0; + f->locvars = NULL; + f->linedefined = 0; + f->lastlinedefined = 0; + f->source = NULL; + return f; +} + + +void luaF_freeproto (lua_State *L, Proto *f) { + luaM_freearray(L, f->code, f->sizecode, Instruction); + luaM_freearray(L, f->p, f->sizep, Proto *); + luaM_freearray(L, f->k, f->sizek, TValue); + luaM_freearray(L, f->lineinfo, f->sizelineinfo, int); + luaM_freearray(L, f->locvars, f->sizelocvars, struct LocVar); + luaM_freearray(L, f->upvalues, f->sizeupvalues, TString *); + luaM_free(L, f); +} + + +void luaF_freeclosure (lua_State *L, Closure *c) { + int size = (c->c.isC) ? sizeCclosure(c->c.nupvalues) : + sizeLclosure(c->l.nupvalues); + luaM_freemem(L, c, size); +} + + +/* +** Look for n-th local variable at line `line' in function `func'. +** Returns NULL if not found. +*/ +const char *luaF_getlocalname (const Proto *f, int local_number, int pc) { + int i; + for (i = 0; isizelocvars && f->locvars[i].startpc <= pc; i++) { + if (pc < f->locvars[i].endpc) { /* is variable active? */ + local_number--; + if (local_number == 0) + return getstr(f->locvars[i].varname); + } + } + return NULL; /* not found */ +} + diff --git a/src/lua/lfunc.h b/src/lua/lfunc.h new file mode 100644 index 0000000..a68cf51 --- /dev/null +++ b/src/lua/lfunc.h @@ -0,0 +1,34 @@ +/* +** $Id: lfunc.h,v 2.4.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions to manipulate prototypes and closures +** See Copyright Notice in lua.h +*/ + +#ifndef lfunc_h +#define lfunc_h + + +#include "lobject.h" + + +#define sizeCclosure(n) (cast(int, sizeof(CClosure)) + \ + cast(int, sizeof(TValue)*((n)-1))) + +#define sizeLclosure(n) (cast(int, sizeof(LClosure)) + \ + cast(int, sizeof(TValue *)*((n)-1))) + + +LUAI_FUNC Proto *luaF_newproto (lua_State *L); +LUAI_FUNC Closure *luaF_newCclosure (lua_State *L, int nelems, Table *e); +LUAI_FUNC Closure *luaF_newLclosure (lua_State *L, int nelems, Table *e); +LUAI_FUNC UpVal *luaF_newupval (lua_State *L); +LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level); +LUAI_FUNC void luaF_close (lua_State *L, StkId level); +LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f); +LUAI_FUNC void luaF_freeclosure (lua_State *L, Closure *c); +LUAI_FUNC void luaF_freeupval (lua_State *L, UpVal *uv); +LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number, + int pc); + + +#endif diff --git a/src/lua/lgc.c b/src/lua/lgc.c new file mode 100644 index 0000000..e909c79 --- /dev/null +++ b/src/lua/lgc.c @@ -0,0 +1,710 @@ +/* +** $Id: lgc.c,v 2.38.1.2 2011/03/18 18:05:38 roberto Exp $ +** Garbage Collector +** See Copyright Notice in lua.h +*/ + +#include + +#define lgc_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" + + +#define GCSTEPSIZE 1024u +#define GCSWEEPMAX 40 +#define GCSWEEPCOST 10 +#define GCFINALIZECOST 100 + + +#define maskmarks cast_byte(~(bitmask(BLACKBIT)|WHITEBITS)) + +#define makewhite(g,x) \ + ((x)->gch.marked = cast_byte(((x)->gch.marked & maskmarks) | luaC_white(g))) + +#define white2gray(x) reset2bits((x)->gch.marked, WHITE0BIT, WHITE1BIT) +#define black2gray(x) resetbit((x)->gch.marked, BLACKBIT) + +#define stringmark(s) reset2bits((s)->tsv.marked, WHITE0BIT, WHITE1BIT) + + +#define isfinalized(u) testbit((u)->marked, FINALIZEDBIT) +#define markfinalized(u) l_setbit((u)->marked, FINALIZEDBIT) + + +#define KEYWEAK bitmask(KEYWEAKBIT) +#define VALUEWEAK bitmask(VALUEWEAKBIT) + + + +#define markvalue(g,o) { checkconsistency(o); \ + if (iscollectable(o) && iswhite(gcvalue(o))) reallymarkobject(g,gcvalue(o)); } + +#define markobject(g,t) { if (iswhite(obj2gco(t))) \ + reallymarkobject(g, obj2gco(t)); } + + +#define setthreshold(g) (g->GCthreshold = (g->estimate/100) * g->gcpause) + + +static void removeentry (Node *n) { + lua_assert(ttisnil(gval(n))); + if (iscollectable(gkey(n))) + setttype(gkey(n), LUA_TDEADKEY); /* dead key; remove it */ +} + + +static void reallymarkobject (global_State *g, GCObject *o) { + lua_assert(iswhite(o) && !isdead(g, o)); + white2gray(o); + switch (o->gch.tt) { + case LUA_TSTRING: { + return; + } + case LUA_TUSERDATA: { + Table *mt = gco2u(o)->metatable; + gray2black(o); /* udata are never gray */ + if (mt) markobject(g, mt); + markobject(g, gco2u(o)->env); + return; + } + case LUA_TUPVAL: { + UpVal *uv = gco2uv(o); + markvalue(g, uv->v); + if (uv->v == &uv->u.value) /* closed? */ + gray2black(o); /* open upvalues are never black */ + return; + } + case LUA_TFUNCTION: { + gco2cl(o)->c.gclist = g->gray; + g->gray = o; + break; + } + case LUA_TTABLE: { + gco2h(o)->gclist = g->gray; + g->gray = o; + break; + } + case LUA_TTHREAD: { + gco2th(o)->gclist = g->gray; + g->gray = o; + break; + } + case LUA_TPROTO: { + gco2p(o)->gclist = g->gray; + g->gray = o; + break; + } + default: lua_assert(0); + } +} + + +static void marktmu (global_State *g) { + GCObject *u = g->tmudata; + if (u) { + do { + u = u->gch.next; + makewhite(g, u); /* may be marked, if left from previous GC */ + reallymarkobject(g, u); + } while (u != g->tmudata); + } +} + + +/* move `dead' udata that need finalization to list `tmudata' */ +size_t luaC_separateudata (lua_State *L, int all) { + global_State *g = G(L); + size_t deadmem = 0; + GCObject **p = &g->mainthread->next; + GCObject *curr; + while ((curr = *p) != NULL) { + if (!(iswhite(curr) || all) || isfinalized(gco2u(curr))) + p = &curr->gch.next; /* don't bother with them */ + else if (fasttm(L, gco2u(curr)->metatable, TM_GC) == NULL) { + markfinalized(gco2u(curr)); /* don't need finalization */ + p = &curr->gch.next; + } + else { /* must call its gc method */ + deadmem += sizeudata(gco2u(curr)); + markfinalized(gco2u(curr)); + *p = curr->gch.next; + /* link `curr' at the end of `tmudata' list */ + if (g->tmudata == NULL) /* list is empty? */ + g->tmudata = curr->gch.next = curr; /* creates a circular list */ + else { + curr->gch.next = g->tmudata->gch.next; + g->tmudata->gch.next = curr; + g->tmudata = curr; + } + } + } + return deadmem; +} + + +static int traversetable (global_State *g, Table *h) { + int i; + int weakkey = 0; + int weakvalue = 0; + const TValue *mode; + if (h->metatable) + markobject(g, h->metatable); + mode = gfasttm(g, h->metatable, TM_MODE); + if (mode && ttisstring(mode)) { /* is there a weak mode? */ + weakkey = (strchr(svalue(mode), 'k') != NULL); + weakvalue = (strchr(svalue(mode), 'v') != NULL); + if (weakkey || weakvalue) { /* is really weak? */ + h->marked &= ~(KEYWEAK | VALUEWEAK); /* clear bits */ + h->marked |= cast_byte((weakkey << KEYWEAKBIT) | + (weakvalue << VALUEWEAKBIT)); + h->gclist = g->weak; /* must be cleared after GC, ... */ + g->weak = obj2gco(h); /* ... so put in the appropriate list */ + } + } + if (weakkey && weakvalue) return 1; + if (!weakvalue) { + i = h->sizearray; + while (i--) + markvalue(g, &h->array[i]); + } + i = sizenode(h); + while (i--) { + Node *n = gnode(h, i); + lua_assert(ttype(gkey(n)) != LUA_TDEADKEY || ttisnil(gval(n))); + if (ttisnil(gval(n))) + removeentry(n); /* remove empty entries */ + else { + lua_assert(!ttisnil(gkey(n))); + if (!weakkey) markvalue(g, gkey(n)); + if (!weakvalue) markvalue(g, gval(n)); + } + } + return weakkey || weakvalue; +} + + +/* +** All marks are conditional because a GC may happen while the +** prototype is still being created +*/ +static void traverseproto (global_State *g, Proto *f) { + int i; + if (f->source) stringmark(f->source); + for (i=0; isizek; i++) /* mark literals */ + markvalue(g, &f->k[i]); + for (i=0; isizeupvalues; i++) { /* mark upvalue names */ + if (f->upvalues[i]) + stringmark(f->upvalues[i]); + } + for (i=0; isizep; i++) { /* mark nested protos */ + if (f->p[i]) + markobject(g, f->p[i]); + } + for (i=0; isizelocvars; i++) { /* mark local-variable names */ + if (f->locvars[i].varname) + stringmark(f->locvars[i].varname); + } +} + + + +static void traverseclosure (global_State *g, Closure *cl) { + markobject(g, cl->c.env); + if (cl->c.isC) { + int i; + for (i=0; ic.nupvalues; i++) /* mark its upvalues */ + markvalue(g, &cl->c.upvalue[i]); + } + else { + int i; + lua_assert(cl->l.nupvalues == cl->l.p->nups); + markobject(g, cl->l.p); + for (i=0; il.nupvalues; i++) /* mark its upvalues */ + markobject(g, cl->l.upvals[i]); + } +} + + +static void checkstacksizes (lua_State *L, StkId max) { + int ci_used = cast_int(L->ci - L->base_ci); /* number of `ci' in use */ + int s_used = cast_int(max - L->stack); /* part of stack in use */ + if (L->size_ci > LUAI_MAXCALLS) /* handling overflow? */ + return; /* do not touch the stacks */ + if (4*ci_used < L->size_ci && 2*BASIC_CI_SIZE < L->size_ci) + luaD_reallocCI(L, L->size_ci/2); /* still big enough... */ + condhardstacktests(luaD_reallocCI(L, ci_used + 1)); + if (4*s_used < L->stacksize && + 2*(BASIC_STACK_SIZE+EXTRA_STACK) < L->stacksize) + luaD_reallocstack(L, L->stacksize/2); /* still big enough... */ + condhardstacktests(luaD_reallocstack(L, s_used)); +} + + +static void traversestack (global_State *g, lua_State *l) { + StkId o, lim; + CallInfo *ci; + markvalue(g, gt(l)); + lim = l->top; + for (ci = l->base_ci; ci <= l->ci; ci++) { + lua_assert(ci->top <= l->stack_last); + if (lim < ci->top) lim = ci->top; + } + for (o = l->stack; o < l->top; o++) + markvalue(g, o); + for (; o <= lim; o++) + setnilvalue(o); + checkstacksizes(l, lim); +} + + +/* +** traverse one gray object, turning it to black. +** Returns `quantity' traversed. +*/ +static l_mem propagatemark (global_State *g) { + GCObject *o = g->gray; + lua_assert(isgray(o)); + gray2black(o); + switch (o->gch.tt) { + case LUA_TTABLE: { + Table *h = gco2h(o); + g->gray = h->gclist; + if (traversetable(g, h)) /* table is weak? */ + black2gray(o); /* keep it gray */ + return sizeof(Table) + sizeof(TValue) * h->sizearray + + sizeof(Node) * sizenode(h); + } + case LUA_TFUNCTION: { + Closure *cl = gco2cl(o); + g->gray = cl->c.gclist; + traverseclosure(g, cl); + return (cl->c.isC) ? sizeCclosure(cl->c.nupvalues) : + sizeLclosure(cl->l.nupvalues); + } + case LUA_TTHREAD: { + lua_State *th = gco2th(o); + g->gray = th->gclist; + th->gclist = g->grayagain; + g->grayagain = o; + black2gray(o); + traversestack(g, th); + return sizeof(lua_State) + sizeof(TValue) * th->stacksize + + sizeof(CallInfo) * th->size_ci; + } + case LUA_TPROTO: { + Proto *p = gco2p(o); + g->gray = p->gclist; + traverseproto(g, p); + return sizeof(Proto) + sizeof(Instruction) * p->sizecode + + sizeof(Proto *) * p->sizep + + sizeof(TValue) * p->sizek + + sizeof(int) * p->sizelineinfo + + sizeof(LocVar) * p->sizelocvars + + sizeof(TString *) * p->sizeupvalues; + } + default: lua_assert(0); return 0; + } +} + + +static size_t propagateall (global_State *g) { + size_t m = 0; + while (g->gray) m += propagatemark(g); + return m; +} + + +/* +** The next function tells whether a key or value can be cleared from +** a weak table. Non-collectable objects are never removed from weak +** tables. Strings behave as `values', so are never removed too. for +** other objects: if really collected, cannot keep them; for userdata +** being finalized, keep them in keys, but not in values +*/ +static int iscleared (const TValue *o, int iskey) { + if (!iscollectable(o)) return 0; + if (ttisstring(o)) { + stringmark(rawtsvalue(o)); /* strings are `values', so are never weak */ + return 0; + } + return iswhite(gcvalue(o)) || + (ttisuserdata(o) && (!iskey && isfinalized(uvalue(o)))); +} + + +/* +** clear collected entries from weaktables +*/ +static void cleartable (GCObject *l) { + while (l) { + Table *h = gco2h(l); + int i = h->sizearray; + lua_assert(testbit(h->marked, VALUEWEAKBIT) || + testbit(h->marked, KEYWEAKBIT)); + if (testbit(h->marked, VALUEWEAKBIT)) { + while (i--) { + TValue *o = &h->array[i]; + if (iscleared(o, 0)) /* value was collected? */ + setnilvalue(o); /* remove value */ + } + } + i = sizenode(h); + while (i--) { + Node *n = gnode(h, i); + if (!ttisnil(gval(n)) && /* non-empty entry? */ + (iscleared(key2tval(n), 1) || iscleared(gval(n), 0))) { + setnilvalue(gval(n)); /* remove value ... */ + removeentry(n); /* remove entry from table */ + } + } + l = h->gclist; + } +} + + +static void freeobj (lua_State *L, GCObject *o) { + switch (o->gch.tt) { + case LUA_TPROTO: luaF_freeproto(L, gco2p(o)); break; + case LUA_TFUNCTION: luaF_freeclosure(L, gco2cl(o)); break; + case LUA_TUPVAL: luaF_freeupval(L, gco2uv(o)); break; + case LUA_TTABLE: luaH_free(L, gco2h(o)); break; + case LUA_TTHREAD: { + lua_assert(gco2th(o) != L && gco2th(o) != G(L)->mainthread); + luaE_freethread(L, gco2th(o)); + break; + } + case LUA_TSTRING: { + G(L)->strt.nuse--; + luaM_freemem(L, o, sizestring(gco2ts(o))); + break; + } + case LUA_TUSERDATA: { + luaM_freemem(L, o, sizeudata(gco2u(o))); + break; + } + default: lua_assert(0); + } +} + + + +#define sweepwholelist(L,p) sweeplist(L,p,MAX_LUMEM) + + +static GCObject **sweeplist (lua_State *L, GCObject **p, lu_mem count) { + GCObject *curr; + global_State *g = G(L); + int deadmask = otherwhite(g); + while ((curr = *p) != NULL && count-- > 0) { + if (curr->gch.tt == LUA_TTHREAD) /* sweep open upvalues of each thread */ + sweepwholelist(L, &gco2th(curr)->openupval); + if ((curr->gch.marked ^ WHITEBITS) & deadmask) { /* not dead? */ + lua_assert(!isdead(g, curr) || testbit(curr->gch.marked, FIXEDBIT)); + makewhite(g, curr); /* make it white (for next cycle) */ + p = &curr->gch.next; + } + else { /* must erase `curr' */ + lua_assert(isdead(g, curr) || deadmask == bitmask(SFIXEDBIT)); + *p = curr->gch.next; + if (curr == g->rootgc) /* is the first element of the list? */ + g->rootgc = curr->gch.next; /* adjust first */ + freeobj(L, curr); + } + } + return p; +} + + +static void checkSizes (lua_State *L) { + global_State *g = G(L); + /* check size of string hash */ + if (g->strt.nuse < cast(lu_int32, g->strt.size/4) && + g->strt.size > MINSTRTABSIZE*2) + luaS_resize(L, g->strt.size/2); /* table is too big */ + /* check size of buffer */ + if (luaZ_sizebuffer(&g->buff) > LUA_MINBUFFER*2) { /* buffer too big? */ + size_t newsize = luaZ_sizebuffer(&g->buff) / 2; + luaZ_resizebuffer(L, &g->buff, newsize); + } +} + + +static void GCTM (lua_State *L) { + global_State *g = G(L); + GCObject *o = g->tmudata->gch.next; /* get first element */ + Udata *udata = rawgco2u(o); + const TValue *tm; + /* remove udata from `tmudata' */ + if (o == g->tmudata) /* last element? */ + g->tmudata = NULL; + else + g->tmudata->gch.next = udata->uv.next; + udata->uv.next = g->mainthread->next; /* return it to `root' list */ + g->mainthread->next = o; + makewhite(g, o); + tm = fasttm(L, udata->uv.metatable, TM_GC); + if (tm != NULL) { + lu_byte oldah = L->allowhook; + lu_mem oldt = g->GCthreshold; + L->allowhook = 0; /* stop debug hooks during GC tag method */ + g->GCthreshold = 2*g->totalbytes; /* avoid GC steps */ + setobj2s(L, L->top, tm); + setuvalue(L, L->top+1, udata); + L->top += 2; + luaD_call(L, L->top - 2, 0); + L->allowhook = oldah; /* restore hooks */ + g->GCthreshold = oldt; /* restore threshold */ + } +} + + +/* +** Call all GC tag methods +*/ +void luaC_callGCTM (lua_State *L) { + while (G(L)->tmudata) + GCTM(L); +} + + +void luaC_freeall (lua_State *L) { + global_State *g = G(L); + int i; + g->currentwhite = WHITEBITS | bitmask(SFIXEDBIT); /* mask to collect all elements */ + sweepwholelist(L, &g->rootgc); + for (i = 0; i < g->strt.size; i++) /* free all string lists */ + sweepwholelist(L, &g->strt.hash[i]); +} + + +static void markmt (global_State *g) { + int i; + for (i=0; imt[i]) markobject(g, g->mt[i]); +} + + +/* mark root set */ +static void markroot (lua_State *L) { + global_State *g = G(L); + g->gray = NULL; + g->grayagain = NULL; + g->weak = NULL; + markobject(g, g->mainthread); + /* make global table be traversed before main stack */ + markvalue(g, gt(g->mainthread)); + markvalue(g, registry(L)); + markmt(g); + g->gcstate = GCSpropagate; +} + + +static void remarkupvals (global_State *g) { + UpVal *uv; + for (uv = g->uvhead.u.l.next; uv != &g->uvhead; uv = uv->u.l.next) { + lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); + if (isgray(obj2gco(uv))) + markvalue(g, uv->v); + } +} + + +static void atomic (lua_State *L) { + global_State *g = G(L); + size_t udsize; /* total size of userdata to be finalized */ + /* remark occasional upvalues of (maybe) dead threads */ + remarkupvals(g); + /* traverse objects cautch by write barrier and by 'remarkupvals' */ + propagateall(g); + /* remark weak tables */ + g->gray = g->weak; + g->weak = NULL; + lua_assert(!iswhite(obj2gco(g->mainthread))); + markobject(g, L); /* mark running thread */ + markmt(g); /* mark basic metatables (again) */ + propagateall(g); + /* remark gray again */ + g->gray = g->grayagain; + g->grayagain = NULL; + propagateall(g); + udsize = luaC_separateudata(L, 0); /* separate userdata to be finalized */ + marktmu(g); /* mark `preserved' userdata */ + udsize += propagateall(g); /* remark, to propagate `preserveness' */ + cleartable(g->weak); /* remove collected objects from weak tables */ + /* flip current white */ + g->currentwhite = cast_byte(otherwhite(g)); + g->sweepstrgc = 0; + g->sweepgc = &g->rootgc; + g->gcstate = GCSsweepstring; + g->estimate = g->totalbytes - udsize; /* first estimate */ +} + + +static l_mem singlestep (lua_State *L) { + global_State *g = G(L); + /*lua_checkmemory(L);*/ + switch (g->gcstate) { + case GCSpause: { + markroot(L); /* start a new collection */ + return 0; + } + case GCSpropagate: { + if (g->gray) + return propagatemark(g); + else { /* no more `gray' objects */ + atomic(L); /* finish mark phase */ + return 0; + } + } + case GCSsweepstring: { + lu_mem old = g->totalbytes; + sweepwholelist(L, &g->strt.hash[g->sweepstrgc++]); + if (g->sweepstrgc >= g->strt.size) /* nothing more to sweep? */ + g->gcstate = GCSsweep; /* end sweep-string phase */ + lua_assert(old >= g->totalbytes); + g->estimate -= old - g->totalbytes; + return GCSWEEPCOST; + } + case GCSsweep: { + lu_mem old = g->totalbytes; + g->sweepgc = sweeplist(L, g->sweepgc, GCSWEEPMAX); + if (*g->sweepgc == NULL) { /* nothing more to sweep? */ + checkSizes(L); + g->gcstate = GCSfinalize; /* end sweep phase */ + } + lua_assert(old >= g->totalbytes); + g->estimate -= old - g->totalbytes; + return GCSWEEPMAX*GCSWEEPCOST; + } + case GCSfinalize: { + if (g->tmudata) { + GCTM(L); + if (g->estimate > GCFINALIZECOST) + g->estimate -= GCFINALIZECOST; + return GCFINALIZECOST; + } + else { + g->gcstate = GCSpause; /* end collection */ + g->gcdept = 0; + return 0; + } + } + default: lua_assert(0); return 0; + } +} + + +void luaC_step (lua_State *L) { + global_State *g = G(L); + l_mem lim = (GCSTEPSIZE/100) * g->gcstepmul; + if (lim == 0) + lim = (MAX_LUMEM-1)/2; /* no limit */ + g->gcdept += g->totalbytes - g->GCthreshold; + do { + lim -= singlestep(L); + if (g->gcstate == GCSpause) + break; + } while (lim > 0); + if (g->gcstate != GCSpause) { + if (g->gcdept < GCSTEPSIZE) + g->GCthreshold = g->totalbytes + GCSTEPSIZE; /* - lim/g->gcstepmul;*/ + else { + g->gcdept -= GCSTEPSIZE; + g->GCthreshold = g->totalbytes; + } + } + else { + setthreshold(g); + } +} + + +void luaC_fullgc (lua_State *L) { + global_State *g = G(L); + if (g->gcstate <= GCSpropagate) { + /* reset sweep marks to sweep all elements (returning them to white) */ + g->sweepstrgc = 0; + g->sweepgc = &g->rootgc; + /* reset other collector lists */ + g->gray = NULL; + g->grayagain = NULL; + g->weak = NULL; + g->gcstate = GCSsweepstring; + } + lua_assert(g->gcstate != GCSpause && g->gcstate != GCSpropagate); + /* finish any pending sweep phase */ + while (g->gcstate != GCSfinalize) { + lua_assert(g->gcstate == GCSsweepstring || g->gcstate == GCSsweep); + singlestep(L); + } + markroot(L); + while (g->gcstate != GCSpause) { + singlestep(L); + } + setthreshold(g); +} + + +void luaC_barrierf (lua_State *L, GCObject *o, GCObject *v) { + global_State *g = G(L); + lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o)); + lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause); + lua_assert(ttype(&o->gch) != LUA_TTABLE); + /* must keep invariant? */ + if (g->gcstate == GCSpropagate) + reallymarkobject(g, v); /* restore invariant */ + else /* don't mind */ + makewhite(g, o); /* mark as white just to avoid other barriers */ +} + + +void luaC_barrierback (lua_State *L, Table *t) { + global_State *g = G(L); + GCObject *o = obj2gco(t); + lua_assert(isblack(o) && !isdead(g, o)); + lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause); + black2gray(o); /* make table gray (again) */ + t->gclist = g->grayagain; + g->grayagain = o; +} + + +void luaC_link (lua_State *L, GCObject *o, lu_byte tt) { + global_State *g = G(L); + o->gch.next = g->rootgc; + g->rootgc = o; + o->gch.marked = luaC_white(g); + o->gch.tt = tt; +} + + +void luaC_linkupval (lua_State *L, UpVal *uv) { + global_State *g = G(L); + GCObject *o = obj2gco(uv); + o->gch.next = g->rootgc; /* link upvalue into `rootgc' list */ + g->rootgc = o; + if (isgray(o)) { + if (g->gcstate == GCSpropagate) { + gray2black(o); /* closed upvalues need barrier */ + luaC_barrier(L, uv, uv->v); + } + else { /* sweep phase: sweep it (turning it into white) */ + makewhite(g, o); + lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause); + } + } +} + diff --git a/src/lua/lgc.h b/src/lua/lgc.h new file mode 100644 index 0000000..5a8dc60 --- /dev/null +++ b/src/lua/lgc.h @@ -0,0 +1,110 @@ +/* +** $Id: lgc.h,v 2.15.1.1 2007/12/27 13:02:25 roberto Exp $ +** Garbage Collector +** See Copyright Notice in lua.h +*/ + +#ifndef lgc_h +#define lgc_h + + +#include "lobject.h" + + +/* +** Possible states of the Garbage Collector +*/ +#define GCSpause 0 +#define GCSpropagate 1 +#define GCSsweepstring 2 +#define GCSsweep 3 +#define GCSfinalize 4 + + +/* +** some userful bit tricks +*/ +#define resetbits(x,m) ((x) &= cast(lu_byte, ~(m))) +#define setbits(x,m) ((x) |= (m)) +#define testbits(x,m) ((x) & (m)) +#define bitmask(b) (1<<(b)) +#define bit2mask(b1,b2) (bitmask(b1) | bitmask(b2)) +#define l_setbit(x,b) setbits(x, bitmask(b)) +#define resetbit(x,b) resetbits(x, bitmask(b)) +#define testbit(x,b) testbits(x, bitmask(b)) +#define set2bits(x,b1,b2) setbits(x, (bit2mask(b1, b2))) +#define reset2bits(x,b1,b2) resetbits(x, (bit2mask(b1, b2))) +#define test2bits(x,b1,b2) testbits(x, (bit2mask(b1, b2))) + + + +/* +** Layout for bit use in `marked' field: +** bit 0 - object is white (type 0) +** bit 1 - object is white (type 1) +** bit 2 - object is black +** bit 3 - for userdata: has been finalized +** bit 3 - for tables: has weak keys +** bit 4 - for tables: has weak values +** bit 5 - object is fixed (should not be collected) +** bit 6 - object is "super" fixed (only the main thread) +*/ + + +#define WHITE0BIT 0 +#define WHITE1BIT 1 +#define BLACKBIT 2 +#define FINALIZEDBIT 3 +#define KEYWEAKBIT 3 +#define VALUEWEAKBIT 4 +#define FIXEDBIT 5 +#define SFIXEDBIT 6 +#define WHITEBITS bit2mask(WHITE0BIT, WHITE1BIT) + + +#define iswhite(x) test2bits((x)->gch.marked, WHITE0BIT, WHITE1BIT) +#define isblack(x) testbit((x)->gch.marked, BLACKBIT) +#define isgray(x) (!isblack(x) && !iswhite(x)) + +#define otherwhite(g) (g->currentwhite ^ WHITEBITS) +#define isdead(g,v) ((v)->gch.marked & otherwhite(g) & WHITEBITS) + +#define changewhite(x) ((x)->gch.marked ^= WHITEBITS) +#define gray2black(x) l_setbit((x)->gch.marked, BLACKBIT) + +#define valiswhite(x) (iscollectable(x) && iswhite(gcvalue(x))) + +#define luaC_white(g) cast(lu_byte, (g)->currentwhite & WHITEBITS) + + +#define luaC_checkGC(L) { \ + condhardstacktests(luaD_reallocstack(L, L->stacksize - EXTRA_STACK - 1)); \ + if (G(L)->totalbytes >= G(L)->GCthreshold) \ + luaC_step(L); } + + +#define luaC_barrier(L,p,v) { if (valiswhite(v) && isblack(obj2gco(p))) \ + luaC_barrierf(L,obj2gco(p),gcvalue(v)); } + +#define luaC_barriert(L,t,v) { if (valiswhite(v) && isblack(obj2gco(t))) \ + luaC_barrierback(L,t); } + +#define luaC_objbarrier(L,p,o) \ + { if (iswhite(obj2gco(o)) && isblack(obj2gco(p))) \ + luaC_barrierf(L,obj2gco(p),obj2gco(o)); } + +#define luaC_objbarriert(L,t,o) \ + { if (iswhite(obj2gco(o)) && isblack(obj2gco(t))) luaC_barrierback(L,t); } + +LUAI_FUNC size_t luaC_separateudata (lua_State *L, int all); +LUAI_FUNC void luaC_callGCTM (lua_State *L); +LUAI_FUNC void luaC_freeall (lua_State *L); +LUAI_FUNC void luaC_step (lua_State *L); +LUAI_FUNC void luaC_fullgc (lua_State *L); +LUAI_FUNC void luaC_link (lua_State *L, GCObject *o, lu_byte tt); +LUAI_FUNC void luaC_linkupval (lua_State *L, UpVal *uv); +LUAI_FUNC void luaC_barrierf (lua_State *L, GCObject *o, GCObject *v); +LUAI_FUNC void luaC_barrierback (lua_State *L, Table *t); + + +#endif diff --git a/src/lua/linit.c b/src/lua/linit.c new file mode 100644 index 0000000..c1f90df --- /dev/null +++ b/src/lua/linit.c @@ -0,0 +1,38 @@ +/* +** $Id: linit.c,v 1.14.1.1 2007/12/27 13:02:25 roberto Exp $ +** Initialization of libraries for lua.c +** See Copyright Notice in lua.h +*/ + + +#define linit_c +#define LUA_LIB + +#include "lua.h" + +#include "lualib.h" +#include "lauxlib.h" + + +static const luaL_Reg lualibs[] = { + {"", luaopen_base}, + {LUA_LOADLIBNAME, luaopen_package}, + {LUA_TABLIBNAME, luaopen_table}, + {LUA_IOLIBNAME, luaopen_io}, + {LUA_OSLIBNAME, luaopen_os}, + {LUA_STRLIBNAME, luaopen_string}, + {LUA_MATHLIBNAME, luaopen_math}, + {LUA_DBLIBNAME, luaopen_debug}, + {NULL, NULL} +}; + + +LUALIB_API void luaL_openlibs (lua_State *L) { + const luaL_Reg *lib = lualibs; + for (; lib->func; lib++) { + lua_pushcfunction(L, lib->func); + lua_pushstring(L, lib->name); + lua_call(L, 1, 0); + } +} + diff --git a/src/lua/liolib.c b/src/lua/liolib.c new file mode 100644 index 0000000..649f9a5 --- /dev/null +++ b/src/lua/liolib.c @@ -0,0 +1,556 @@ +/* +** $Id: liolib.c,v 2.73.1.4 2010/05/14 15:33:51 roberto Exp $ +** Standard I/O (and system) library +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include + +#define liolib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + + +#define IO_INPUT 1 +#define IO_OUTPUT 2 + + +static const char *const fnames[] = {"input", "output"}; + + +static int pushresult (lua_State *L, int i, const char *filename) { + int en = errno; /* calls to Lua API may change this value */ + if (i) { + lua_pushboolean(L, 1); + return 1; + } + else { + lua_pushnil(L); + if (filename) + lua_pushfstring(L, "%s: %s", filename, strerror(en)); + else + lua_pushfstring(L, "%s", strerror(en)); + lua_pushinteger(L, en); + return 3; + } +} + + +static void fileerror (lua_State *L, int arg, const char *filename) { + lua_pushfstring(L, "%s: %s", filename, strerror(errno)); + luaL_argerror(L, arg, lua_tostring(L, -1)); +} + + +#define tofilep(L) ((FILE **)luaL_checkudata(L, 1, LUA_FILEHANDLE)) + + +static int io_type (lua_State *L) { + void *ud; + luaL_checkany(L, 1); + ud = lua_touserdata(L, 1); + lua_getfield(L, LUA_REGISTRYINDEX, LUA_FILEHANDLE); + if (ud == NULL || !lua_getmetatable(L, 1) || !lua_rawequal(L, -2, -1)) + lua_pushnil(L); /* not a file */ + else if (*((FILE **)ud) == NULL) + lua_pushliteral(L, "closed file"); + else + lua_pushliteral(L, "file"); + return 1; +} + + +static FILE *tofile (lua_State *L) { + FILE **f = tofilep(L); + if (*f == NULL) + luaL_error(L, "attempt to use a closed file"); + return *f; +} + + + +/* +** When creating file handles, always creates a `closed' file handle +** before opening the actual file; so, if there is a memory error, the +** file is not left opened. +*/ +static FILE **newfile (lua_State *L) { + FILE **pf = (FILE **)lua_newuserdata(L, sizeof(FILE *)); + *pf = NULL; /* file handle is currently `closed' */ + luaL_getmetatable(L, LUA_FILEHANDLE); + lua_setmetatable(L, -2); + return pf; +} + + +/* +** function to (not) close the standard files stdin, stdout, and stderr +*/ +static int io_noclose (lua_State *L) { + lua_pushnil(L); + lua_pushliteral(L, "cannot close standard file"); + return 2; +} + + +/* +** function to close 'popen' files +*/ +static int io_pclose (lua_State *L) { + FILE **p = tofilep(L); + int ok = lua_pclose(L, *p); + *p = NULL; + return pushresult(L, ok, NULL); +} + + +/* +** function to close regular files +*/ +static int io_fclose (lua_State *L) { + FILE **p = tofilep(L); + int ok = (fclose(*p) == 0); + *p = NULL; + return pushresult(L, ok, NULL); +} + + +static int aux_close (lua_State *L) { + lua_getfenv(L, 1); + lua_getfield(L, -1, "__close"); + return (lua_tocfunction(L, -1))(L); +} + + +static int io_close (lua_State *L) { + if (lua_isnone(L, 1)) + lua_rawgeti(L, LUA_ENVIRONINDEX, IO_OUTPUT); + tofile(L); /* make sure argument is a file */ + return aux_close(L); +} + + +static int io_gc (lua_State *L) { + FILE *f = *tofilep(L); + /* ignore closed files */ + if (f != NULL) + aux_close(L); + return 0; +} + + +static int io_tostring (lua_State *L) { + FILE *f = *tofilep(L); + if (f == NULL) + lua_pushliteral(L, "file (closed)"); + else + lua_pushfstring(L, "file (%p)", f); + return 1; +} + + +static int io_open (lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + const char *mode = luaL_optstring(L, 2, "r"); + FILE **pf = newfile(L); + *pf = fopen(filename, mode); + return (*pf == NULL) ? pushresult(L, 0, filename) : 1; +} + + +/* +** this function has a separated environment, which defines the +** correct __close for 'popen' files +*/ +static int io_popen (lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + const char *mode = luaL_optstring(L, 2, "r"); + FILE **pf = newfile(L); + *pf = lua_popen(L, filename, mode); + return (*pf == NULL) ? pushresult(L, 0, filename) : 1; +} + + +static int io_tmpfile (lua_State *L) { + FILE **pf = newfile(L); + *pf = tmpfile(); + return (*pf == NULL) ? pushresult(L, 0, NULL) : 1; +} + + +static FILE *getiofile (lua_State *L, int findex) { + FILE *f; + lua_rawgeti(L, LUA_ENVIRONINDEX, findex); + f = *(FILE **)lua_touserdata(L, -1); + if (f == NULL) + luaL_error(L, "standard %s file is closed", fnames[findex - 1]); + return f; +} + + +static int g_iofile (lua_State *L, int f, const char *mode) { + if (!lua_isnoneornil(L, 1)) { + const char *filename = lua_tostring(L, 1); + if (filename) { + FILE **pf = newfile(L); + *pf = fopen(filename, mode); + if (*pf == NULL) + fileerror(L, 1, filename); + } + else { + tofile(L); /* check that it's a valid file handle */ + lua_pushvalue(L, 1); + } + lua_rawseti(L, LUA_ENVIRONINDEX, f); + } + /* return current value */ + lua_rawgeti(L, LUA_ENVIRONINDEX, f); + return 1; +} + + +static int io_input (lua_State *L) { + return g_iofile(L, IO_INPUT, "r"); +} + + +static int io_output (lua_State *L) { + return g_iofile(L, IO_OUTPUT, "w"); +} + + +static int io_readline (lua_State *L); + + +static void aux_lines (lua_State *L, int idx, int toclose) { + lua_pushvalue(L, idx); + lua_pushboolean(L, toclose); /* close/not close file when finished */ + lua_pushcclosure(L, io_readline, 2); +} + + +static int f_lines (lua_State *L) { + tofile(L); /* check that it's a valid file handle */ + aux_lines(L, 1, 0); + return 1; +} + + +static int io_lines (lua_State *L) { + if (lua_isnoneornil(L, 1)) { /* no arguments? */ + /* will iterate over default input */ + lua_rawgeti(L, LUA_ENVIRONINDEX, IO_INPUT); + return f_lines(L); + } + else { + const char *filename = luaL_checkstring(L, 1); + FILE **pf = newfile(L); + *pf = fopen(filename, "r"); + if (*pf == NULL) + fileerror(L, 1, filename); + aux_lines(L, lua_gettop(L), 1); + return 1; + } +} + + +/* +** {====================================================== +** READ +** ======================================================= +*/ + + +static int read_number (lua_State *L, FILE *f) { + lua_Number d; + if (fscanf(f, LUA_NUMBER_SCAN, &d) == 1) { + lua_pushnumber(L, d); + return 1; + } + else { + lua_pushnil(L); /* "result" to be removed */ + return 0; /* read fails */ + } +} + + +static int test_eof (lua_State *L, FILE *f) { + int c = getc(f); + ungetc(c, f); + lua_pushlstring(L, NULL, 0); + return (c != EOF); +} + + +static int read_line (lua_State *L, FILE *f) { + luaL_Buffer b; + luaL_buffinit(L, &b); + for (;;) { + size_t l; + char *p = luaL_prepbuffer(&b); + if (fgets(p, LUAL_BUFFERSIZE, f) == NULL) { /* eof? */ + luaL_pushresult(&b); /* close buffer */ + return (lua_objlen(L, -1) > 0); /* check whether read something */ + } + l = strlen(p); + if (l == 0 || p[l-1] != '\n') + luaL_addsize(&b, l); + else { + luaL_addsize(&b, l - 1); /* do not include `eol' */ + luaL_pushresult(&b); /* close buffer */ + return 1; /* read at least an `eol' */ + } + } +} + + +static int read_chars (lua_State *L, FILE *f, size_t n) { + size_t rlen; /* how much to read */ + size_t nr; /* number of chars actually read */ + luaL_Buffer b; + luaL_buffinit(L, &b); + rlen = LUAL_BUFFERSIZE; /* try to read that much each time */ + do { + char *p = luaL_prepbuffer(&b); + if (rlen > n) rlen = n; /* cannot read more than asked */ + nr = fread(p, sizeof(char), rlen, f); + luaL_addsize(&b, nr); + n -= nr; /* still have to read `n' chars */ + } while (n > 0 && nr == rlen); /* until end of count or eof */ + luaL_pushresult(&b); /* close buffer */ + return (n == 0 || lua_objlen(L, -1) > 0); +} + + +static int g_read (lua_State *L, FILE *f, int first) { + int nargs = lua_gettop(L) - 1; + int success; + int n; + clearerr(f); + if (nargs == 0) { /* no arguments? */ + success = read_line(L, f); + n = first+1; /* to return 1 result */ + } + else { /* ensure stack space for all results and for auxlib's buffer */ + luaL_checkstack(L, nargs+LUA_MINSTACK, "too many arguments"); + success = 1; + for (n = first; nargs-- && success; n++) { + if (lua_type(L, n) == LUA_TNUMBER) { + size_t l = (size_t)lua_tointeger(L, n); + success = (l == 0) ? test_eof(L, f) : read_chars(L, f, l); + } + else { + const char *p = lua_tostring(L, n); + luaL_argcheck(L, p && p[0] == '*', n, "invalid option"); + switch (p[1]) { + case 'n': /* number */ + success = read_number(L, f); + break; + case 'l': /* line */ + success = read_line(L, f); + break; + case 'a': /* file */ + read_chars(L, f, ~((size_t)0)); /* read MAX_SIZE_T chars */ + success = 1; /* always success */ + break; + default: + return luaL_argerror(L, n, "invalid format"); + } + } + } + } + if (ferror(f)) + return pushresult(L, 0, NULL); + if (!success) { + lua_pop(L, 1); /* remove last result */ + lua_pushnil(L); /* push nil instead */ + } + return n - first; +} + + +static int io_read (lua_State *L) { + return g_read(L, getiofile(L, IO_INPUT), 1); +} + + +static int f_read (lua_State *L) { + return g_read(L, tofile(L), 2); +} + + +static int io_readline (lua_State *L) { + FILE *f = *(FILE **)lua_touserdata(L, lua_upvalueindex(1)); + int sucess; + if (f == NULL) /* file is already closed? */ + luaL_error(L, "file is already closed"); + sucess = read_line(L, f); + if (ferror(f)) + return luaL_error(L, "%s", strerror(errno)); + if (sucess) return 1; + else { /* EOF */ + if (lua_toboolean(L, lua_upvalueindex(2))) { /* generator created file? */ + lua_settop(L, 0); + lua_pushvalue(L, lua_upvalueindex(1)); + aux_close(L); /* close it */ + } + return 0; + } +} + +/* }====================================================== */ + + +static int g_write (lua_State *L, FILE *f, int arg) { + int nargs = lua_gettop(L) - 1; + int status = 1; + for (; nargs--; arg++) { + if (lua_type(L, arg) == LUA_TNUMBER) { + /* optimization: could be done exactly as for strings */ + status = status && + fprintf(f, LUA_NUMBER_FMT, lua_tonumber(L, arg)) > 0; + } + else { + size_t l; + const char *s = luaL_checklstring(L, arg, &l); + status = status && (fwrite(s, sizeof(char), l, f) == l); + } + } + return pushresult(L, status, NULL); +} + + +static int io_write (lua_State *L) { + return g_write(L, getiofile(L, IO_OUTPUT), 1); +} + + +static int f_write (lua_State *L) { + return g_write(L, tofile(L), 2); +} + + +static int f_seek (lua_State *L) { + static const int mode[] = {SEEK_SET, SEEK_CUR, SEEK_END}; + static const char *const modenames[] = {"set", "cur", "end", NULL}; + FILE *f = tofile(L); + int op = luaL_checkoption(L, 2, "cur", modenames); + long offset = luaL_optlong(L, 3, 0); + op = fseek(f, offset, mode[op]); + if (op) + return pushresult(L, 0, NULL); /* error */ + else { + lua_pushinteger(L, ftell(f)); + return 1; + } +} + + +static int f_setvbuf (lua_State *L) { + static const int mode[] = {_IONBF, _IOFBF, _IOLBF}; + static const char *const modenames[] = {"no", "full", "line", NULL}; + FILE *f = tofile(L); + int op = luaL_checkoption(L, 2, NULL, modenames); + lua_Integer sz = luaL_optinteger(L, 3, LUAL_BUFFERSIZE); + int res = setvbuf(f, NULL, mode[op], sz); + return pushresult(L, res == 0, NULL); +} + + + +static int io_flush (lua_State *L) { + return pushresult(L, fflush(getiofile(L, IO_OUTPUT)) == 0, NULL); +} + + +static int f_flush (lua_State *L) { + return pushresult(L, fflush(tofile(L)) == 0, NULL); +} + + +static const luaL_Reg iolib[] = { + {"close", io_close}, + {"flush", io_flush}, + {"input", io_input}, + {"lines", io_lines}, + {"open", io_open}, + {"output", io_output}, + {"popen", io_popen}, + {"read", io_read}, + {"tmpfile", io_tmpfile}, + {"type", io_type}, + {"write", io_write}, + {NULL, NULL} +}; + + +static const luaL_Reg flib[] = { + {"close", io_close}, + {"flush", f_flush}, + {"lines", f_lines}, + {"read", f_read}, + {"seek", f_seek}, + {"setvbuf", f_setvbuf}, + {"write", f_write}, + {"__gc", io_gc}, + {"__tostring", io_tostring}, + {NULL, NULL} +}; + + +static void createmeta (lua_State *L) { + luaL_newmetatable(L, LUA_FILEHANDLE); /* create metatable for file handles */ + lua_pushvalue(L, -1); /* push metatable */ + lua_setfield(L, -2, "__index"); /* metatable.__index = metatable */ + luaL_register(L, NULL, flib); /* file methods */ +} + + +static void createstdfile (lua_State *L, FILE *f, int k, const char *fname) { + *newfile(L) = f; + if (k > 0) { + lua_pushvalue(L, -1); + lua_rawseti(L, LUA_ENVIRONINDEX, k); + } + lua_pushvalue(L, -2); /* copy environment */ + lua_setfenv(L, -2); /* set it */ + lua_setfield(L, -3, fname); +} + + +static void newfenv (lua_State *L, lua_CFunction cls) { + lua_createtable(L, 0, 1); + lua_pushcfunction(L, cls); + lua_setfield(L, -2, "__close"); +} + + +LUALIB_API int luaopen_io (lua_State *L) { + createmeta(L); + /* create (private) environment (with fields IO_INPUT, IO_OUTPUT, __close) */ + newfenv(L, io_fclose); + lua_replace(L, LUA_ENVIRONINDEX); + /* open library */ + luaL_register(L, LUA_IOLIBNAME, iolib); + /* create (and set) default files */ + newfenv(L, io_noclose); /* close function for default files */ + createstdfile(L, stdin, IO_INPUT, "stdin"); + createstdfile(L, stdout, IO_OUTPUT, "stdout"); + createstdfile(L, stderr, 0, "stderr"); + lua_pop(L, 1); /* pop environment for default files */ + lua_getfield(L, -1, "popen"); + newfenv(L, io_pclose); /* create environment for 'popen' */ + lua_setfenv(L, -2); /* set fenv for 'popen' */ + lua_pop(L, 1); /* pop 'popen' */ + return 1; +} + diff --git a/src/lua/llex.c b/src/lua/llex.c new file mode 100644 index 0000000..88c6790 --- /dev/null +++ b/src/lua/llex.c @@ -0,0 +1,463 @@ +/* +** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $ +** Lexical Analyzer +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define llex_c +#define LUA_CORE + +#include "lua.h" + +#include "ldo.h" +#include "llex.h" +#include "lobject.h" +#include "lparser.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "lzio.h" + + + +#define next(ls) (ls->current = zgetc(ls->z)) + + + + +#define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') + + +/* ORDER RESERVED */ +const char *const luaX_tokens [] = { + "and", "break", "do", "else", "elseif", + "end", "false", "for", "function", "if", + "in", "local", "nil", "not", "or", "repeat", + "return", "then", "true", "until", "while", + "..", "...", "==", ">=", "<=", "~=", + "", "", "", "", + NULL +}; + + +#define save_and_next(ls) (save(ls, ls->current), next(ls)) + + +static void save (LexState *ls, int c) { + Mbuffer *b = ls->buff; + if (b->n + 1 > b->buffsize) { + size_t newsize; + if (b->buffsize >= MAX_SIZET/2) + luaX_lexerror(ls, "lexical element too long", 0); + newsize = b->buffsize * 2; + luaZ_resizebuffer(ls->L, b, newsize); + } + b->buffer[b->n++] = cast(char, c); +} + + +void luaX_init (lua_State *L) { + int i; + for (i=0; itsv.reserved = cast_byte(i+1); /* reserved word */ + } +} + + +#define MAXSRC 80 + + +const char *luaX_token2str (LexState *ls, int token) { + if (token < FIRST_RESERVED) { + lua_assert(token == cast(unsigned char, token)); + return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) : + luaO_pushfstring(ls->L, "%c", token); + } + else + return luaX_tokens[token-FIRST_RESERVED]; +} + + +static const char *txtToken (LexState *ls, int token) { + switch (token) { + case TK_NAME: + case TK_STRING: + case TK_NUMBER: + save(ls, '\0'); + return luaZ_buffer(ls->buff); + default: + return luaX_token2str(ls, token); + } +} + + +void luaX_lexerror (LexState *ls, const char *msg, int token) { + char buff[MAXSRC]; + luaO_chunkid(buff, getstr(ls->source), MAXSRC); + msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg); + if (token) + luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token)); + luaD_throw(ls->L, LUA_ERRSYNTAX); +} + + +void luaX_syntaxerror (LexState *ls, const char *msg) { + luaX_lexerror(ls, msg, ls->t.token); +} + + +TString *luaX_newstring (LexState *ls, const char *str, size_t l) { + lua_State *L = ls->L; + TString *ts = luaS_newlstr(L, str, l); + TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */ + if (ttisnil(o)) { + setbvalue(o, 1); /* make sure `str' will not be collected */ + luaC_checkGC(L); + } + return ts; +} + + +static void inclinenumber (LexState *ls) { + int old = ls->current; + lua_assert(currIsNewline(ls)); + next(ls); /* skip `\n' or `\r' */ + if (currIsNewline(ls) && ls->current != old) + next(ls); /* skip `\n\r' or `\r\n' */ + if (++ls->linenumber >= MAX_INT) + luaX_syntaxerror(ls, "chunk has too many lines"); +} + + +void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) { + ls->decpoint = '.'; + ls->L = L; + ls->lookahead.token = TK_EOS; /* no look-ahead token */ + ls->z = z; + ls->fs = NULL; + ls->linenumber = 1; + ls->lastline = 1; + ls->source = source; + luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */ + next(ls); /* read first char */ +} + + + +/* +** ======================================================= +** LEXICAL ANALYZER +** ======================================================= +*/ + + + +static int check_next (LexState *ls, const char *set) { + if (!strchr(set, ls->current)) + return 0; + save_and_next(ls); + return 1; +} + + +static void buffreplace (LexState *ls, char from, char to) { + size_t n = luaZ_bufflen(ls->buff); + char *p = luaZ_buffer(ls->buff); + while (n--) + if (p[n] == from) p[n] = to; +} + + +static void trydecpoint (LexState *ls, SemInfo *seminfo) { + /* format error: try to update decimal point separator */ + struct lconv *cv = localeconv(); + char old = ls->decpoint; + ls->decpoint = (cv ? cv->decimal_point[0] : '.'); + buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */ + if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) { + /* format error with correct decimal point: no more options */ + buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */ + luaX_lexerror(ls, "malformed number", TK_NUMBER); + } +} + + +/* LUA_NUMBER */ +static void read_numeral (LexState *ls, SemInfo *seminfo) { + lua_assert(isdigit(ls->current)); + do { + save_and_next(ls); + } while (isdigit(ls->current) || ls->current == '.'); + if (check_next(ls, "Ee")) /* `E'? */ + check_next(ls, "+-"); /* optional exponent sign */ + while (isalnum(ls->current) || ls->current == '_') + save_and_next(ls); + save(ls, '\0'); + buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ + if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */ + trydecpoint(ls, seminfo); /* try to update decimal point separator */ +} + + +static int skip_sep (LexState *ls) { + int count = 0; + int s = ls->current; + lua_assert(s == '[' || s == ']'); + save_and_next(ls); + while (ls->current == '=') { + save_and_next(ls); + count++; + } + return (ls->current == s) ? count : (-count) - 1; +} + + +static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { + int cont = 0; + (void)(cont); /* avoid warnings when `cont' is not used */ + save_and_next(ls); /* skip 2nd `[' */ + if (currIsNewline(ls)) /* string starts with a newline? */ + inclinenumber(ls); /* skip it */ + for (;;) { + switch (ls->current) { + case EOZ: + luaX_lexerror(ls, (seminfo) ? "unfinished long string" : + "unfinished long comment", TK_EOS); + break; /* to avoid warnings */ +#if defined(LUA_COMPAT_LSTR) + case '[': { + if (skip_sep(ls) == sep) { + save_and_next(ls); /* skip 2nd `[' */ + cont++; +#if LUA_COMPAT_LSTR == 1 + if (sep == 0) + luaX_lexerror(ls, "nesting of [[...]] is deprecated", '['); +#endif + } + break; + } +#endif + case ']': { + if (skip_sep(ls) == sep) { + save_and_next(ls); /* skip 2nd `]' */ +#if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2 + cont--; + if (sep == 0 && cont >= 0) break; +#endif + goto endloop; + } + break; + } + case '\n': + case '\r': { + save(ls, '\n'); + inclinenumber(ls); + if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ + break; + } + default: { + if (seminfo) save_and_next(ls); + else next(ls); + } + } + } endloop: + if (seminfo) + seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), + luaZ_bufflen(ls->buff) - 2*(2 + sep)); +} + + +static void read_string (LexState *ls, int del, SemInfo *seminfo) { + save_and_next(ls); + while (ls->current != del) { + switch (ls->current) { + case EOZ: + luaX_lexerror(ls, "unfinished string", TK_EOS); + continue; /* to avoid warnings */ + case '\n': + case '\r': + luaX_lexerror(ls, "unfinished string", TK_STRING); + continue; /* to avoid warnings */ + case '\\': { + int c; + next(ls); /* do not save the `\' */ + switch (ls->current) { + case 'a': c = '\a'; break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'v': c = '\v'; break; + case '\n': /* go through */ + case '\r': save(ls, '\n'); inclinenumber(ls); continue; + case EOZ: continue; /* will raise an error next loop */ + default: { + if (!isdigit(ls->current)) + save_and_next(ls); /* handles \\, \", \', and \? */ + else { /* \xxx */ + int i = 0; + c = 0; + do { + c = 10*c + (ls->current-'0'); + next(ls); + } while (++i<3 && isdigit(ls->current)); + if (c > UCHAR_MAX) + luaX_lexerror(ls, "escape sequence too large", TK_STRING); + save(ls, c); + } + continue; + } + } + save(ls, c); + next(ls); + continue; + } + default: + save_and_next(ls); + } + } + save_and_next(ls); /* skip delimiter */ + seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, + luaZ_bufflen(ls->buff) - 2); +} + + +static int llex (LexState *ls, SemInfo *seminfo) { + luaZ_resetbuffer(ls->buff); + for (;;) { + switch (ls->current) { + case '\n': + case '\r': { + inclinenumber(ls); + continue; + } + case '-': { + next(ls); + if (ls->current != '-') return '-'; + /* else is a comment */ + next(ls); + if (ls->current == '[') { + int sep = skip_sep(ls); + luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ + if (sep >= 0) { + read_long_string(ls, NULL, sep); /* long comment */ + luaZ_resetbuffer(ls->buff); + continue; + } + } + /* else short comment */ + while (!currIsNewline(ls) && ls->current != EOZ) + next(ls); + continue; + } + case '[': { + int sep = skip_sep(ls); + if (sep >= 0) { + read_long_string(ls, seminfo, sep); + return TK_STRING; + } + else if (sep == -1) return '['; + else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING); + } + case '=': { + next(ls); + if (ls->current != '=') return '='; + else { next(ls); return TK_EQ; } + } + case '<': { + next(ls); + if (ls->current != '=') return '<'; + else { next(ls); return TK_LE; } + } + case '>': { + next(ls); + if (ls->current != '=') return '>'; + else { next(ls); return TK_GE; } + } + case '~': { + next(ls); + if (ls->current != '=') return '~'; + else { next(ls); return TK_NE; } + } + case '"': + case '\'': { + read_string(ls, ls->current, seminfo); + return TK_STRING; + } + case '.': { + save_and_next(ls); + if (check_next(ls, ".")) { + if (check_next(ls, ".")) + return TK_DOTS; /* ... */ + else return TK_CONCAT; /* .. */ + } + else if (!isdigit(ls->current)) return '.'; + else { + read_numeral(ls, seminfo); + return TK_NUMBER; + } + } + case EOZ: { + return TK_EOS; + } + default: { + if (isspace(ls->current)) { + lua_assert(!currIsNewline(ls)); + next(ls); + continue; + } + else if (isdigit(ls->current)) { + read_numeral(ls, seminfo); + return TK_NUMBER; + } + else if (isalpha(ls->current) || ls->current == '_') { + /* identifier or reserved word */ + TString *ts; + do { + save_and_next(ls); + } while (isalnum(ls->current) || ls->current == '_'); + ts = luaX_newstring(ls, luaZ_buffer(ls->buff), + luaZ_bufflen(ls->buff)); + if (ts->tsv.reserved > 0) /* reserved word? */ + return ts->tsv.reserved - 1 + FIRST_RESERVED; + else { + seminfo->ts = ts; + return TK_NAME; + } + } + else { + int c = ls->current; + next(ls); + return c; /* single-char tokens (+ - / ...) */ + } + } + } + } +} + + +void luaX_next (LexState *ls) { + ls->lastline = ls->linenumber; + if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */ + ls->t = ls->lookahead; /* use this one */ + ls->lookahead.token = TK_EOS; /* and discharge it */ + } + else + ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */ +} + + +void luaX_lookahead (LexState *ls) { + lua_assert(ls->lookahead.token == TK_EOS); + ls->lookahead.token = llex(ls, &ls->lookahead.seminfo); +} + diff --git a/src/lua/llex.h b/src/lua/llex.h new file mode 100644 index 0000000..a9201ce --- /dev/null +++ b/src/lua/llex.h @@ -0,0 +1,81 @@ +/* +** $Id: llex.h,v 1.58.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lexical Analyzer +** See Copyright Notice in lua.h +*/ + +#ifndef llex_h +#define llex_h + +#include "lobject.h" +#include "lzio.h" + + +#define FIRST_RESERVED 257 + +/* maximum length of a reserved word */ +#define TOKEN_LEN (sizeof("function")/sizeof(char)) + + +/* +* WARNING: if you change the order of this enumeration, +* grep "ORDER RESERVED" +*/ +enum RESERVED { + /* terminal symbols denoted by reserved words */ + TK_AND = FIRST_RESERVED, TK_BREAK, + TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION, + TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT, + TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE, + /* other terminal symbols */ + TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE, TK_NUMBER, + TK_NAME, TK_STRING, TK_EOS +}; + +/* number of reserved words */ +#define NUM_RESERVED (cast(int, TK_WHILE-FIRST_RESERVED+1)) + + +/* array with token `names' */ +LUAI_DATA const char *const luaX_tokens []; + + +typedef union { + lua_Number r; + TString *ts; +} SemInfo; /* semantics information */ + + +typedef struct Token { + int token; + SemInfo seminfo; +} Token; + + +typedef struct LexState { + int current; /* current character (charint) */ + int linenumber; /* input line counter */ + int lastline; /* line of last token `consumed' */ + Token t; /* current token */ + Token lookahead; /* look ahead token */ + struct FuncState *fs; /* `FuncState' is private to the parser */ + struct lua_State *L; + ZIO *z; /* input stream */ + Mbuffer *buff; /* buffer for tokens */ + TString *source; /* current source name */ + char decpoint; /* locale decimal point */ +} LexState; + + +LUAI_FUNC void luaX_init (lua_State *L); +LUAI_FUNC void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, + TString *source); +LUAI_FUNC TString *luaX_newstring (LexState *ls, const char *str, size_t l); +LUAI_FUNC void luaX_next (LexState *ls); +LUAI_FUNC void luaX_lookahead (LexState *ls); +LUAI_FUNC void luaX_lexerror (LexState *ls, const char *msg, int token); +LUAI_FUNC void luaX_syntaxerror (LexState *ls, const char *s); +LUAI_FUNC const char *luaX_token2str (LexState *ls, int token); + + +#endif diff --git a/src/lua/llimits.h b/src/lua/llimits.h new file mode 100644 index 0000000..ca8dcb7 --- /dev/null +++ b/src/lua/llimits.h @@ -0,0 +1,128 @@ +/* +** $Id: llimits.h,v 1.69.1.1 2007/12/27 13:02:25 roberto Exp $ +** Limits, basic types, and some other `installation-dependent' definitions +** See Copyright Notice in lua.h +*/ + +#ifndef llimits_h +#define llimits_h + + +#include +#include + + +#include "lua.h" + + +typedef LUAI_UINT32 lu_int32; + +typedef LUAI_UMEM lu_mem; + +typedef LUAI_MEM l_mem; + + + +/* chars used as small naturals (so that `char' is reserved for characters) */ +typedef unsigned char lu_byte; + + +#define MAX_SIZET ((size_t)(~(size_t)0)-2) + +#define MAX_LUMEM ((lu_mem)(~(lu_mem)0)-2) + + +#define MAX_INT (INT_MAX-2) /* maximum value of an int (-2 for safety) */ + +/* +** conversion of pointer to integer +** this is for hashing only; there is no problem if the integer +** cannot hold the whole pointer value +*/ +#define IntPoint(p) ((unsigned int)(lu_mem)(p)) + + + +/* type to ensure maximum alignment */ +typedef LUAI_USER_ALIGNMENT_T L_Umaxalign; + + +/* result of a `usual argument conversion' over lua_Number */ +typedef LUAI_UACNUMBER l_uacNumber; + + +/* internal assertions for in-house debugging */ +#ifdef lua_assert + +#define check_exp(c,e) (lua_assert(c), (e)) +#define api_check(l,e) lua_assert(e) + +#else + +#define lua_assert(c) ((void)0) +#define check_exp(c,e) (e) +#define api_check luai_apicheck + +#endif + + +#ifndef UNUSED +#define UNUSED(x) ((void)(x)) /* to avoid warnings */ +#endif + + +#ifndef cast +#define cast(t, exp) ((t)(exp)) +#endif + +#define cast_byte(i) cast(lu_byte, (i)) +#define cast_num(i) cast(lua_Number, (i)) +#define cast_int(i) cast(int, (i)) + + + +/* +** type for virtual-machine instructions +** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h) +*/ +typedef lu_int32 Instruction; + + + +/* maximum stack for a Lua function */ +#define MAXSTACK 250 + + + +/* minimum size for the string table (must be power of 2) */ +#ifndef MINSTRTABSIZE +#define MINSTRTABSIZE 32 +#endif + + +/* minimum size for string buffer */ +#ifndef LUA_MINBUFFER +#define LUA_MINBUFFER 32 +#endif + + +#ifndef lua_lock +#define lua_lock(L) ((void) 0) +#define lua_unlock(L) ((void) 0) +#endif + +#ifndef luai_threadyield +#define luai_threadyield(L) {lua_unlock(L); lua_lock(L);} +#endif + + +/* +** macro to control inclusion of some hard tests on stack reallocation +*/ +#ifndef HARDSTACKTESTS +#define condhardstacktests(x) ((void)0) +#else +#define condhardstacktests(x) x +#endif + +#endif diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c new file mode 100644 index 0000000..441fbf7 --- /dev/null +++ b/src/lua/lmathlib.c @@ -0,0 +1,263 @@ +/* +** $Id: lmathlib.c,v 1.67.1.1 2007/12/27 13:02:25 roberto Exp $ +** Standard mathematical library +** See Copyright Notice in lua.h +*/ + + +#include +#include + +#define lmathlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +#undef PI +#define PI (3.14159265358979323846) +#define RADIANS_PER_DEGREE (PI/180.0) + + + +static int math_abs (lua_State *L) { + lua_pushnumber(L, fabs(luaL_checknumber(L, 1))); + return 1; +} + +static int math_sin (lua_State *L) { + lua_pushnumber(L, sin(luaL_checknumber(L, 1))); + return 1; +} + +static int math_sinh (lua_State *L) { + lua_pushnumber(L, sinh(luaL_checknumber(L, 1))); + return 1; +} + +static int math_cos (lua_State *L) { + lua_pushnumber(L, cos(luaL_checknumber(L, 1))); + return 1; +} + +static int math_cosh (lua_State *L) { + lua_pushnumber(L, cosh(luaL_checknumber(L, 1))); + return 1; +} + +static int math_tan (lua_State *L) { + lua_pushnumber(L, tan(luaL_checknumber(L, 1))); + return 1; +} + +static int math_tanh (lua_State *L) { + lua_pushnumber(L, tanh(luaL_checknumber(L, 1))); + return 1; +} + +static int math_asin (lua_State *L) { + lua_pushnumber(L, asin(luaL_checknumber(L, 1))); + return 1; +} + +static int math_acos (lua_State *L) { + lua_pushnumber(L, acos(luaL_checknumber(L, 1))); + return 1; +} + +static int math_atan (lua_State *L) { + lua_pushnumber(L, atan(luaL_checknumber(L, 1))); + return 1; +} + +static int math_atan2 (lua_State *L) { + lua_pushnumber(L, atan2(luaL_checknumber(L, 1), luaL_checknumber(L, 2))); + return 1; +} + +static int math_ceil (lua_State *L) { + lua_pushnumber(L, ceil(luaL_checknumber(L, 1))); + return 1; +} + +static int math_floor (lua_State *L) { + lua_pushnumber(L, floor(luaL_checknumber(L, 1))); + return 1; +} + +static int math_fmod (lua_State *L) { + lua_pushnumber(L, fmod(luaL_checknumber(L, 1), luaL_checknumber(L, 2))); + return 1; +} + +static int math_modf (lua_State *L) { + double ip; + double fp = modf(luaL_checknumber(L, 1), &ip); + lua_pushnumber(L, ip); + lua_pushnumber(L, fp); + return 2; +} + +static int math_sqrt (lua_State *L) { + lua_pushnumber(L, sqrt(luaL_checknumber(L, 1))); + return 1; +} + +static int math_pow (lua_State *L) { + lua_pushnumber(L, pow(luaL_checknumber(L, 1), luaL_checknumber(L, 2))); + return 1; +} + +static int math_log (lua_State *L) { + lua_pushnumber(L, log(luaL_checknumber(L, 1))); + return 1; +} + +static int math_log10 (lua_State *L) { + lua_pushnumber(L, log10(luaL_checknumber(L, 1))); + return 1; +} + +static int math_exp (lua_State *L) { + lua_pushnumber(L, exp(luaL_checknumber(L, 1))); + return 1; +} + +static int math_deg (lua_State *L) { + lua_pushnumber(L, luaL_checknumber(L, 1)/RADIANS_PER_DEGREE); + return 1; +} + +static int math_rad (lua_State *L) { + lua_pushnumber(L, luaL_checknumber(L, 1)*RADIANS_PER_DEGREE); + return 1; +} + +static int math_frexp (lua_State *L) { + int e; + lua_pushnumber(L, frexp(luaL_checknumber(L, 1), &e)); + lua_pushinteger(L, e); + return 2; +} + +static int math_ldexp (lua_State *L) { + lua_pushnumber(L, ldexp(luaL_checknumber(L, 1), luaL_checkint(L, 2))); + return 1; +} + + + +static int math_min (lua_State *L) { + int n = lua_gettop(L); /* number of arguments */ + lua_Number dmin = luaL_checknumber(L, 1); + int i; + for (i=2; i<=n; i++) { + lua_Number d = luaL_checknumber(L, i); + if (d < dmin) + dmin = d; + } + lua_pushnumber(L, dmin); + return 1; +} + + +static int math_max (lua_State *L) { + int n = lua_gettop(L); /* number of arguments */ + lua_Number dmax = luaL_checknumber(L, 1); + int i; + for (i=2; i<=n; i++) { + lua_Number d = luaL_checknumber(L, i); + if (d > dmax) + dmax = d; + } + lua_pushnumber(L, dmax); + return 1; +} + + +static int math_random (lua_State *L) { + /* the `%' avoids the (rare) case of r==1, and is needed also because on + some systems (SunOS!) `rand()' may return a value larger than RAND_MAX */ + lua_Number r = (lua_Number)(rand()%RAND_MAX) / (lua_Number)RAND_MAX; + switch (lua_gettop(L)) { /* check number of arguments */ + case 0: { /* no arguments */ + lua_pushnumber(L, r); /* Number between 0 and 1 */ + break; + } + case 1: { /* only upper limit */ + int u = luaL_checkint(L, 1); + luaL_argcheck(L, 1<=u, 1, "interval is empty"); + lua_pushnumber(L, floor(r*u)+1); /* int between 1 and `u' */ + break; + } + case 2: { /* lower and upper limits */ + int l = luaL_checkint(L, 1); + int u = luaL_checkint(L, 2); + luaL_argcheck(L, l<=u, 2, "interval is empty"); + lua_pushnumber(L, floor(r*(u-l+1))+l); /* int between `l' and `u' */ + break; + } + default: return luaL_error(L, "wrong number of arguments"); + } + return 1; +} + + +static int math_randomseed (lua_State *L) { + srand(luaL_checkint(L, 1)); + return 0; +} + + +static const luaL_Reg mathlib[] = { + {"abs", math_abs}, + {"acos", math_acos}, + {"asin", math_asin}, + {"atan2", math_atan2}, + {"atan", math_atan}, + {"ceil", math_ceil}, + {"cosh", math_cosh}, + {"cos", math_cos}, + {"deg", math_deg}, + {"exp", math_exp}, + {"floor", math_floor}, + {"fmod", math_fmod}, + {"frexp", math_frexp}, + {"ldexp", math_ldexp}, + {"log10", math_log10}, + {"log", math_log}, + {"max", math_max}, + {"min", math_min}, + {"modf", math_modf}, + {"pow", math_pow}, + {"rad", math_rad}, + {"random", math_random}, + {"randomseed", math_randomseed}, + {"sinh", math_sinh}, + {"sin", math_sin}, + {"sqrt", math_sqrt}, + {"tanh", math_tanh}, + {"tan", math_tan}, + {NULL, NULL} +}; + + +/* +** Open math library +*/ +LUALIB_API int luaopen_math (lua_State *L) { + luaL_register(L, LUA_MATHLIBNAME, mathlib); + lua_pushnumber(L, PI); + lua_setfield(L, -2, "pi"); + lua_pushnumber(L, HUGE_VAL); + lua_setfield(L, -2, "huge"); +#if defined(LUA_COMPAT_MOD) + lua_getfield(L, -1, "fmod"); + lua_setfield(L, -2, "mod"); +#endif + return 1; +} + diff --git a/src/lua/lmem.c b/src/lua/lmem.c new file mode 100644 index 0000000..ae7d8c9 --- /dev/null +++ b/src/lua/lmem.c @@ -0,0 +1,86 @@ +/* +** $Id: lmem.c,v 1.70.1.1 2007/12/27 13:02:25 roberto Exp $ +** Interface to Memory Manager +** See Copyright Notice in lua.h +*/ + + +#include + +#define lmem_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" + + + +/* +** About the realloc function: +** void * frealloc (void *ud, void *ptr, size_t osize, size_t nsize); +** (`osize' is the old size, `nsize' is the new size) +** +** Lua ensures that (ptr == NULL) iff (osize == 0). +** +** * frealloc(ud, NULL, 0, x) creates a new block of size `x' +** +** * frealloc(ud, p, x, 0) frees the block `p' +** (in this specific case, frealloc must return NULL). +** particularly, frealloc(ud, NULL, 0, 0) does nothing +** (which is equivalent to free(NULL) in ANSI C) +** +** frealloc returns NULL if it cannot create or reallocate the area +** (any reallocation to an equal or smaller size cannot fail!) +*/ + + + +#define MINSIZEARRAY 4 + + +void *luaM_growaux_ (lua_State *L, void *block, int *size, size_t size_elems, + int limit, const char *errormsg) { + void *newblock; + int newsize; + if (*size >= limit/2) { /* cannot double it? */ + if (*size >= limit) /* cannot grow even a little? */ + luaG_runerror(L, errormsg); + newsize = limit; /* still have at least one free place */ + } + else { + newsize = (*size)*2; + if (newsize < MINSIZEARRAY) + newsize = MINSIZEARRAY; /* minimum size */ + } + newblock = luaM_reallocv(L, block, *size, newsize, size_elems); + *size = newsize; /* update only when everything else is OK */ + return newblock; +} + + +void *luaM_toobig (lua_State *L) { + luaG_runerror(L, "memory allocation error: block too big"); + return NULL; /* to avoid warnings */ +} + + + +/* +** generic allocation routine. +*/ +void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { + global_State *g = G(L); + lua_assert((osize == 0) == (block == NULL)); + block = (*g->frealloc)(g->ud, block, osize, nsize); + if (block == NULL && nsize > 0) + luaD_throw(L, LUA_ERRMEM); + lua_assert((nsize == 0) == (block == NULL)); + g->totalbytes = (g->totalbytes - osize) + nsize; + return block; +} + diff --git a/src/lua/lmem.h b/src/lua/lmem.h new file mode 100644 index 0000000..7c2dcb3 --- /dev/null +++ b/src/lua/lmem.h @@ -0,0 +1,49 @@ +/* +** $Id: lmem.h,v 1.31.1.1 2007/12/27 13:02:25 roberto Exp $ +** Interface to Memory Manager +** See Copyright Notice in lua.h +*/ + +#ifndef lmem_h +#define lmem_h + + +#include + +#include "llimits.h" +#include "lua.h" + +#define MEMERRMSG "not enough memory" + + +#define luaM_reallocv(L,b,on,n,e) \ + ((cast(size_t, (n)+1) <= MAX_SIZET/(e)) ? /* +1 to avoid warnings */ \ + luaM_realloc_(L, (b), (on)*(e), (n)*(e)) : \ + luaM_toobig(L)) + +#define luaM_freemem(L, b, s) luaM_realloc_(L, (b), (s), 0) +#define luaM_free(L, b) luaM_realloc_(L, (b), sizeof(*(b)), 0) +#define luaM_freearray(L, b, n, t) luaM_reallocv(L, (b), n, 0, sizeof(t)) + +#define luaM_malloc(L,t) luaM_realloc_(L, NULL, 0, (t)) +#define luaM_new(L,t) cast(t *, luaM_malloc(L, sizeof(t))) +#define luaM_newvector(L,n,t) \ + cast(t *, luaM_reallocv(L, NULL, 0, n, sizeof(t))) + +#define luaM_growvector(L,v,nelems,size,t,limit,e) \ + if ((nelems)+1 > (size)) \ + ((v)=cast(t *, luaM_growaux_(L,v,&(size),sizeof(t),limit,e))) + +#define luaM_reallocvector(L, v,oldn,n,t) \ + ((v)=cast(t *, luaM_reallocv(L, v, oldn, n, sizeof(t)))) + + +LUAI_FUNC void *luaM_realloc_ (lua_State *L, void *block, size_t oldsize, + size_t size); +LUAI_FUNC void *luaM_toobig (lua_State *L); +LUAI_FUNC void *luaM_growaux_ (lua_State *L, void *block, int *size, + size_t size_elem, int limit, + const char *errormsg); + +#endif + diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c new file mode 100644 index 0000000..43697e1 --- /dev/null +++ b/src/lua/loadlib.c @@ -0,0 +1,686 @@ +/* +** $Id: loadlib.c,v 1.52.1.4 2009/09/09 13:17:16 roberto Exp $ +** Dynamic library loader for Lua +** See Copyright Notice in lua.h +** +** This module contains an implementation of loadlib for Unix systems +** that have dlfcn, an implementation for Darwin (Mac OS X), an +** implementation for Windows, and a stub for other systems. +*/ + + +#include +#include +#include + + +#define loadlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +/* prefix for open functions in C libraries */ +#define LUA_POF "luaopen_" + +/* separator for open functions in C libraries */ +#define LUA_OFSEP "_" + + +#define LIBPREFIX "LOADLIB: " + +#define POF LUA_POF +#define LIB_FAIL "open" + + +/* error codes for ll_loadfunc */ +#define ERRLIB 1 +#define ERRFUNC 2 + + +#define LSB_CONFIG "lsb_config" + +static void ll_unloadlib (void *lib); +static void *ll_load (lua_State *L, const char *path); +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym); + + + +#if defined(LUA_DL_DLOPEN) +/* +** {======================================================================== +** This is an implementation of loadlib based on the dlfcn interface. +** The dlfcn interface is available in Linux, SunOS, Solaris, IRIX, FreeBSD, +** NetBSD, AIX 4.2, HPUX 11, and probably most other Unix flavors, at least +** as an emulation layer on top of native functions. +** ========================================================================= +*/ + +#include + +static void ll_unloadlib (void *lib) { + dlclose(lib); +} + + +static void *ll_load (lua_State *L, const char *path) { + void *lib = dlopen(path, RTLD_NOW); + if (lib == NULL) lua_pushstring(L, dlerror()); + return lib; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + lua_CFunction f; + *(void **)(&f)= dlsym(lib, sym); + if (f == NULL) lua_pushstring(L, dlerror()); + return f; +} + +/* }====================================================== */ + + + +#elif defined(LUA_DL_DLL) +/* +** {====================================================================== +** This is an implementation of loadlib for Windows using native functions. +** ======================================================================= +*/ + +#include + +static void pusherror (lua_State *L) { + int error = GetLastError(); + char buffer[128]; + if (FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, error, 0, buffer, sizeof(buffer), NULL)) + lua_pushstring(L, buffer); + else + lua_pushfstring(L, "system error %d\n", error); +} + +static void ll_unloadlib (void *lib) { + FreeLibrary((HINSTANCE)lib); +} + + +static void *ll_load (lua_State *L, const char *path) { + HINSTANCE lib = LoadLibraryA(path); + if (lib == NULL) pusherror(L); + return lib; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + lua_CFunction f = (lua_CFunction)GetProcAddress((HINSTANCE)lib, sym); + if (f == NULL) pusherror(L); + return f; +} + +/* }====================================================== */ + + + +#elif defined(LUA_DL_DYLD) +/* +** {====================================================================== +** Native Mac OS X / Darwin Implementation +** ======================================================================= +*/ + +#include + + +/* Mac appends a `_' before C function names */ +#undef POF +#define POF "_" LUA_POF + + +static void pusherror (lua_State *L) { + const char *err_str; + const char *err_file; + NSLinkEditErrors err; + int err_num; + NSLinkEditError(&err, &err_num, &err_file, &err_str); + lua_pushstring(L, err_str); +} + + +static const char *errorfromcode (NSObjectFileImageReturnCode ret) { + switch (ret) { + case NSObjectFileImageInappropriateFile: + return "file is not a bundle"; + case NSObjectFileImageArch: + return "library is for wrong CPU type"; + case NSObjectFileImageFormat: + return "bad format"; + case NSObjectFileImageAccess: + return "cannot access file"; + case NSObjectFileImageFailure: + default: + return "unable to load library"; + } +} + + +static void ll_unloadlib (void *lib) { + NSUnLinkModule((NSModule)lib, NSUNLINKMODULE_OPTION_RESET_LAZY_REFERENCES); +} + + +static void *ll_load (lua_State *L, const char *path) { + NSObjectFileImage img; + NSObjectFileImageReturnCode ret; + /* this would be a rare case, but prevents crashing if it happens */ + if(!_dyld_present()) { + lua_pushliteral(L, "dyld not present"); + return NULL; + } + ret = NSCreateObjectFileImageFromFile(path, &img); + if (ret == NSObjectFileImageSuccess) { + NSModule mod = NSLinkModule(img, path, NSLINKMODULE_OPTION_PRIVATE | + NSLINKMODULE_OPTION_RETURN_ON_ERROR); + NSDestroyObjectFileImage(img); + if (mod == NULL) pusherror(L); + return mod; + } + lua_pushstring(L, errorfromcode(ret)); + return NULL; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + NSSymbol nss = NSLookupSymbolInModule((NSModule)lib, sym); + if (nss == NULL) { + lua_pushfstring(L, "symbol " LUA_QS " not found", sym); + return NULL; + } + return (lua_CFunction)NSAddressOfSymbol(nss); +} + +/* }====================================================== */ + + + +#else +/* +** {====================================================== +** Fallback for other systems +** ======================================================= +*/ + +#undef LIB_FAIL +#define LIB_FAIL "absent" + + +#define DLMSG "dynamic libraries not enabled; check your Lua installation" + + +static void ll_unloadlib (void *lib) { + (void)lib; /* to avoid warnings */ +} + + +static void *ll_load (lua_State *L, const char *path) { + (void)path; /* to avoid warnings */ + lua_pushliteral(L, DLMSG); + return NULL; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + (void)lib; (void)sym; /* to avoid warnings */ + lua_pushliteral(L, DLMSG); + return NULL; +} + +/* }====================================================== */ +#endif + + + +static void **ll_register (lua_State *L, const char *path) { + void **plib; + lua_pushfstring(L, "%s%s", LIBPREFIX, path); + lua_gettable(L, LUA_REGISTRYINDEX); /* check library in registry? */ + if (!lua_isnil(L, -1)) /* is there an entry? */ + plib = (void **)lua_touserdata(L, -1); + else { /* no entry yet; create one */ + lua_pop(L, 1); + plib = (void **)lua_newuserdata(L, sizeof(const void *)); + *plib = NULL; + luaL_getmetatable(L, "_LOADLIB"); + lua_setmetatable(L, -2); + lua_pushfstring(L, "%s%s", LIBPREFIX, path); + lua_pushvalue(L, -2); + lua_settable(L, LUA_REGISTRYINDEX); + } + return plib; +} + + +/* +** __gc tag method: calls library's `ll_unloadlib' function with the lib +** handle +*/ +static int gctm (lua_State *L) { + void **lib = (void **)luaL_checkudata(L, 1, "_LOADLIB"); + if (*lib) ll_unloadlib(*lib); + *lib = NULL; /* mark library as closed */ + return 0; +} + + +static int ll_loadfunc (lua_State *L, const char *path, const char *sym) { + void **reg = ll_register(L, path); + if (*reg == NULL) *reg = ll_load(L, path); + if (*reg == NULL) + return ERRLIB; /* unable to load library */ + else { + lua_CFunction f = ll_sym(L, *reg, sym); + if (f == NULL) + return ERRFUNC; /* unable to find function */ + lua_pushcfunction(L, f); + return 0; /* return function */ + } +} + + +/* +** {====================================================== +** 'require' function +** ======================================================= +*/ + + +static int readable (const char *filename) { + FILE *f = fopen(filename, "r"); /* try to open file */ + if (f == NULL) return 0; /* open failed */ + fclose(f); + return 1; +} + + +static const char *pushnexttemplate (lua_State *L, const char *path) { + const char *l; + while (*path == *LUA_PATHSEP) path++; /* skip separators */ + if (*path == '\0') return NULL; /* no more templates */ + l = strchr(path, *LUA_PATHSEP); /* find next separator */ + if (l == NULL) l = path + strlen(path); + lua_pushlstring(L, path, l - path); /* template */ + return l; +} + + +static const char *findfile (lua_State *L, const char *name, + const char *pname) { + const char *path; + int i = 0; + while (name[i]) { + if (!isalnum(name[i]) && name[i] != '_' && name[i] != '.') { + luaL_error(L, "invalid module name '%s'", name); + return NULL; // never reached, just silences the compiler + } + ++i; + } + lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); + if (lua_type(L, -1) != LUA_TTABLE) { + return NULL; + } + name = luaL_gsub(L, name, ".", LUA_DIRSEP); + lua_getfield(L, -2, pname); + path = lua_tostring(L, -1); + if (path == NULL) + luaL_error(L, "no '%s' configuration was specified for the sandbox; " + "external modules have been disabled", pname); + lua_pushliteral(L, ""); /* error accumulator */ + while ((path = pushnexttemplate(L, path)) != NULL) { + const char *filename; + filename = luaL_gsub(L, lua_tostring(L, -1), LUA_PATH_MARK, name); + lua_remove(L, -2); /* remove path template */ + if (readable(filename)) /* does file exist and is readable? */ + return filename; /* return that file name */ + lua_pushfstring(L, "\n\tno file " LUA_QS, filename); + lua_remove(L, -2); /* remove file name */ + lua_concat(L, 2); /* add entry to possible error message */ + } + return NULL; /* not found */ +} + + +static void loaderror (lua_State *L, const char *filename) { + luaL_error(L, "error loading module " LUA_QS " from file " LUA_QS ":\n\t%s", + lua_tostring(L, 1), filename, lua_tostring(L, -1)); +} + + +static int loader_Lua (lua_State *L) { + const char *filename; + const char *name = luaL_checkstring(L, 1); + filename = findfile(L, name, "path"); + if (filename == NULL) return 1; /* library not found in this path */ + if (luaL_loadfile(L, filename) != 0) + loaderror(L, filename); + return 1; /* library loaded successfully */ +} + + +static const char *mkfuncname (lua_State *L, const char *modname) { + const char *funcname; + const char *mark = strchr(modname, *LUA_IGMARK); + if (mark) modname = mark + 1; + funcname = luaL_gsub(L, modname, ".", LUA_OFSEP); + funcname = lua_pushfstring(L, POF"%s", funcname); + lua_remove(L, -2); /* remove 'gsub' result */ + return funcname; +} + + +static int loader_C (lua_State *L) { + const char *funcname; + const char *name = luaL_checkstring(L, 1); + const char *filename = findfile(L, name, "cpath"); + if (filename == NULL) return 1; /* library not found in this path */ + funcname = mkfuncname(L, name); + if (ll_loadfunc(L, filename, funcname) != 0) + loaderror(L, filename); + return 1; /* library loaded successfully */ +} + + +static int loader_Croot (lua_State *L) { + const char *funcname; + const char *filename; + const char *name = luaL_checkstring(L, 1); + const char *p = strchr(name, '.'); + int stat; + if (p == NULL) return 0; /* is root */ + lua_pushlstring(L, name, p - name); + filename = findfile(L, lua_tostring(L, -1), "cpath"); + if (filename == NULL) return 1; /* root not found */ + funcname = mkfuncname(L, name); + if ((stat = ll_loadfunc(L, filename, funcname)) != 0) { + if (stat != ERRFUNC) loaderror(L, filename); /* real error */ + lua_pushfstring(L, "\n\tno module " LUA_QS " in file " LUA_QS, + name, filename); + return 1; /* function not found */ + } + return 1; +} + + +static int loader_preload (lua_State *L) { + const char *name = luaL_checkstring(L, 1); + lua_getfield(L, LUA_ENVIRONINDEX, "preload"); + if (!lua_istable(L, -1)) + luaL_error(L, LUA_QL("preload") " must be a table"); + lua_getfield(L, -1, name); + return 1; +} + +// If necessary, add an empty metatable to flag the table as non-data +// during preservation. +static void add_empty_metatable(lua_State* L) +{ + if (lua_getmetatable(L, -1) == 0) { + lua_newtable(L); + lua_setmetatable(L, -2); + } else { + lua_pop(L, 1); + } +} + + +static void remove_entries(lua_State* L, const char* name) +{ + lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 1); + return; + } + lua_getfield(L, -1, "remove_entries"); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 2); + return; + } + lua_getfield(L, -1, name); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 3); + return; + } + int n = 1; + while (1) { + lua_rawgeti(L, -1, n); + if (lua_type(L, -1) == LUA_TNIL) { + lua_pop(L, 1); + break; + } + lua_pushnil(L); + lua_settable(L, -6); + ++n; + } + lua_pop(L, 3); +} + + +static int is_disabled(lua_State* L, const char* name) +{ + int status = 0; + lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 1); + return status; + } + lua_getfield(L, -1, "disable_modules"); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 2); + return status; + } + lua_getfield(L, -1, name); + if (lua_type(L, -1) != LUA_TNIL) { + status = 1; + } + lua_pop(L, 3); + return status; +} + + +static const int sentinel_ = 0; +#define sentinel ((void *)&sentinel_) + + +static int ll_require (lua_State *L) { + const char *name = luaL_checkstring(L, 1); + if (is_disabled(L, name)) { + luaL_error(L, "module " LUA_QS " disabled", name); + } + int i; + lua_settop(L, 1); /* _LOADED table will be at index 2 */ + lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); + lua_getfield(L, 2, name); + if (lua_toboolean(L, -1)) { /* is it there? */ + if (lua_touserdata(L, -1) == sentinel) /* check loops */ + luaL_error(L, "loop or previous error loading module " LUA_QS, name); + return 1; /* package is already loaded */ + } + /* else must load it; iterate over available loaders */ + lua_getfield(L, LUA_ENVIRONINDEX, "loaders"); + if (!lua_istable(L, -1)) + luaL_error(L, LUA_QL("loaders") " must be a table"); + lua_pushliteral(L, ""); /* error message accumulator */ + for (i=1; ; i++) { + lua_rawgeti(L, -2, i); /* get a loader */ + if (lua_isnil(L, -1)) + luaL_error(L, "module " LUA_QS " not found:%s", + name, lua_tostring(L, -2)); + lua_pushstring(L, name); + lua_call(L, 1, 1); /* call it */ + if (lua_isfunction(L, -1)) /* did it find module? */ + break; /* module loaded successfully */ + else if (lua_isstring(L, -1)) /* loader returned error message? */ + lua_concat(L, 2); /* accumulate it */ + else + lua_pop(L, 1); + } + lua_pushlightuserdata(L, sentinel); + lua_setfield(L, 2, name); /* _LOADED[name] = sentinel */ + lua_pushstring(L, name); /* pass name as argument to module */ + lua_call(L, 1, 1); /* run loaded module */ + if (!lua_isnil(L, -1)) /* non-nil return? */ + lua_setfield(L, 2, name); /* _LOADED[name] = returned value */ + lua_getfield(L, 2, name); + if (lua_touserdata(L, -1) == sentinel) { /* module did not set a value? */ + lua_pushboolean(L, 1); /* use true as result */ + lua_pushvalue(L, -1); /* extra copy to be returned */ + lua_setfield(L, 2, name); /* _LOADED[name] = true */ + } + add_empty_metatable(L); + remove_entries(L, name); + return 1; +} + +/* }====================================================== */ + + + +/* +** {====================================================== +** 'module' function +** ======================================================= +*/ + + +static void setfenv (lua_State *L) { + lua_Debug ar; + if (lua_getstack(L, 1, &ar) == 0 || + lua_getinfo(L, "f", &ar) == 0 || /* get calling function */ + lua_iscfunction(L, -1)) + luaL_error(L, LUA_QL("module") " not called from a Lua function"); + lua_pushvalue(L, -2); + lua_setfenv(L, -2); + lua_pop(L, 1); +} + + +static void dooptions (lua_State *L, int n) { + int i; + for (i = 2; i <= n; i++) { + lua_pushvalue(L, i); /* get option (a function) */ + lua_pushvalue(L, -2); /* module */ + lua_call(L, 1, 0); + } +} + + +static void modinit (lua_State *L, const char *modname) { + const char *dot; + lua_pushvalue(L, -1); + lua_setfield(L, -2, "_M"); /* module._M = module */ + lua_pushstring(L, modname); + lua_setfield(L, -2, "_NAME"); + dot = strrchr(modname, '.'); /* look for last dot in module name */ + if (dot == NULL) dot = modname; + else dot++; + /* set _PACKAGE as package name (full module name minus last part) */ + lua_pushlstring(L, modname, dot - modname); + lua_setfield(L, -2, "_PACKAGE"); +} + + +static int ll_module (lua_State *L) { + const char *modname = luaL_checkstring(L, 1); + int loaded = lua_gettop(L) + 1; /* index of _LOADED table */ + lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); + lua_getfield(L, loaded, modname); /* get _LOADED[modname] */ + if (!lua_istable(L, -1)) { /* not found? */ + lua_pop(L, 1); /* remove previous result */ + /* try global variable (and create one if it does not exist) */ + if (luaL_findtable(L, LUA_GLOBALSINDEX, modname, 1) != NULL) + return luaL_error(L, "name conflict for module " LUA_QS, modname); + lua_pushvalue(L, -1); + lua_setfield(L, loaded, modname); /* _LOADED[modname] = new table */ + } + /* check whether table already has a _NAME field */ + lua_getfield(L, -1, "_NAME"); + if (!lua_isnil(L, -1)) /* is table an initialized module? */ + lua_pop(L, 1); + else { /* no; initialize it */ + lua_pop(L, 1); + modinit(L, modname); + } + lua_pushvalue(L, -1); + setfenv(L); + dooptions(L, loaded - 1); + return 0; +} + + +static int ll_seeall (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + if (!lua_getmetatable(L, 1)) { + lua_createtable(L, 0, 1); /* create new metatable */ + lua_pushvalue(L, -1); + lua_setmetatable(L, 1); + } + lua_pushvalue(L, LUA_GLOBALSINDEX); + lua_setfield(L, -2, "__index"); /* mt.__index = _G */ + return 0; +} + + +/* }====================================================== */ + + + +static const luaL_Reg pk_funcs[] = { + {"seeall", ll_seeall}, + {NULL, NULL} +}; + + +static const luaL_Reg ll_funcs[] = { + {"module", ll_module}, + {"require", ll_require}, + {NULL, NULL} +}; + + +static const lua_CFunction loaders[] = + {loader_preload, loader_Lua, loader_C, loader_Croot, NULL}; + + +LUALIB_API int luaopen_package (lua_State *L) { + int i; + /* create new type _LOADLIB */ + luaL_newmetatable(L, "_LOADLIB"); + lua_pushcfunction(L, gctm); + lua_setfield(L, -2, "__gc"); + /* create `package' environment table (not exposed to Lua) */ + lua_newtable(L); + lua_pushvalue(L, -1); + lua_replace(L, LUA_ENVIRONINDEX); + /* create `loaders' table */ + lua_createtable(L, sizeof(loaders)/sizeof(loaders[0]) - 1, 0); + /* fill it with pre-defined loaders */ + for (i=0; loaders[i] != NULL; i++) { + lua_pushcfunction(L, loaders[i]); + lua_rawseti(L, -2, i+1); + } + lua_setfield(L, -2, "loaders"); /* put it in field `loaders' */ + + /* set field `preload' */ + luaL_findtable(L, LUA_REGISTRYINDEX, "_PRELOADED", 0); + lua_setfield(L, -2, "preload"); + + /* open lib into global table */ + lua_pushvalue(L, LUA_GLOBALSINDEX); + luaL_register(L, NULL, ll_funcs); + lua_pop(L, 1); + + luaL_register(L, LUA_LOADLIBNAME, pk_funcs); + return 1; /* return 'package' table */ +} diff --git a/src/lua/lobject.c b/src/lua/lobject.c new file mode 100644 index 0000000..4ff5073 --- /dev/null +++ b/src/lua/lobject.c @@ -0,0 +1,214 @@ +/* +** $Id: lobject.c,v 2.22.1.1 2007/12/27 13:02:25 roberto Exp $ +** Some generic functions over Lua objects +** See Copyright Notice in lua.h +*/ + +#include +#include +#include +#include +#include + +#define lobject_c +#define LUA_CORE + +#include "lua.h" + +#include "ldo.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "lvm.h" + + + +const TValue luaO_nilobject_ = {{NULL}, LUA_TNIL}; + + +/* +** converts an integer to a "floating point byte", represented as +** (eeeeexxx), where the real value is (1xxx) * 2^(eeeee - 1) if +** eeeee != 0 and (xxx) otherwise. +*/ +int luaO_int2fb (unsigned int x) { + int e = 0; /* expoent */ + while (x >= 16) { + x = (x+1) >> 1; + e++; + } + if (x < 8) return x; + else return ((e+1) << 3) | (cast_int(x) - 8); +} + + +/* converts back */ +int luaO_fb2int (int x) { + int e = (x >> 3) & 31; + if (e == 0) return x; + else return ((x & 7)+8) << (e - 1); +} + + +int luaO_log2 (unsigned int x) { + static const lu_byte log_2[256] = { + 0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 + }; + int l = -1; + while (x >= 256) { l += 8; x >>= 8; } + return l + log_2[x]; + +} + + +int luaO_rawequalObj (const TValue *t1, const TValue *t2) { + if (ttype(t1) != ttype(t2)) return 0; + else switch (ttype(t1)) { + case LUA_TNIL: + return 1; + case LUA_TNUMBER: + return luai_numeq(nvalue(t1), nvalue(t2)); + case LUA_TBOOLEAN: + return bvalue(t1) == bvalue(t2); /* boolean true must be 1 !! */ + case LUA_TLIGHTUSERDATA: + return pvalue(t1) == pvalue(t2); + default: + lua_assert(iscollectable(t1)); + return gcvalue(t1) == gcvalue(t2); + } +} + + +int luaO_str2d (const char *s, lua_Number *result) { + char *endptr; + *result = lua_str2number(s, &endptr); + if (endptr == s) return 0; /* conversion failed */ + if (*endptr == 'x' || *endptr == 'X') /* maybe an hexadecimal constant? */ + *result = cast_num(strtoul(s, &endptr, 16)); + if (*endptr == '\0') return 1; /* most common case */ + while (isspace(cast(unsigned char, *endptr))) endptr++; + if (*endptr != '\0') return 0; /* invalid trailing characters? */ + return 1; +} + + + +static void pushstr (lua_State *L, const char *str) { + setsvalue2s(L, L->top, luaS_new(L, str)); + incr_top(L); +} + + +/* this function handles only `%d', `%c', %f, %p, and `%s' formats */ +const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) { + int n = 1; + pushstr(L, ""); + for (;;) { + const char *e = strchr(fmt, '%'); + if (e == NULL) break; + setsvalue2s(L, L->top, luaS_newlstr(L, fmt, e-fmt)); + incr_top(L); + switch (*(e+1)) { + case 's': { + const char *s = va_arg(argp, char *); + if (s == NULL) s = "(null)"; + pushstr(L, s); + break; + } + case 'c': { + char buff[2]; + buff[0] = cast(char, va_arg(argp, int)); + buff[1] = '\0'; + pushstr(L, buff); + break; + } + case 'd': { + setnvalue(L->top, cast_num(va_arg(argp, int))); + incr_top(L); + break; + } + case 'f': { + setnvalue(L->top, cast_num(va_arg(argp, l_uacNumber))); + incr_top(L); + break; + } + case 'p': { + char buff[4*sizeof(void *) + 8]; /* should be enough space for a `%p' */ + sprintf(buff, "%p", va_arg(argp, void *)); + pushstr(L, buff); + break; + } + case '%': { + pushstr(L, "%"); + break; + } + default: { + char buff[3]; + buff[0] = '%'; + buff[1] = *(e+1); + buff[2] = '\0'; + pushstr(L, buff); + break; + } + } + n += 2; + fmt = e+2; + } + pushstr(L, fmt); + luaV_concat(L, n+1, cast_int(L->top - L->base) - 1); + L->top -= n; + return svalue(L->top - 1); +} + + +const char *luaO_pushfstring (lua_State *L, const char *fmt, ...) { + const char *msg; + va_list argp; + va_start(argp, fmt); + msg = luaO_pushvfstring(L, fmt, argp); + va_end(argp); + return msg; +} + + +void luaO_chunkid (char *out, const char *source, size_t bufflen) { + if (*source == '=') { + strncpy(out, source+1, bufflen); /* remove first char */ + out[bufflen-1] = '\0'; /* ensures null termination */ + } + else { /* out = "source", or "...source" */ + if (*source == '@') { + size_t l; + source++; /* skip the `@' */ + bufflen -= sizeof(" '...' "); + l = strlen(source); + strcpy(out, ""); + if (l > bufflen) { + source += (l-bufflen); /* get last part of file name */ + strcat(out, "..."); + } + strcat(out, source); + } + else { /* out = [string "string"] */ + size_t len = strcspn(source, "\n\r"); /* stop at first newline */ + bufflen -= sizeof(" [string \"...\"] "); + if (len > bufflen) len = bufflen; + strcpy(out, "[string \""); + if (source[len] != '\0') { /* must truncate? */ + strncat(out, source, len); + strcat(out, "..."); + } + else + strcat(out, source); + strcat(out, "\"]"); + } + } +} diff --git a/src/lua/lobject.h b/src/lua/lobject.h new file mode 100644 index 0000000..f1e447e --- /dev/null +++ b/src/lua/lobject.h @@ -0,0 +1,381 @@ +/* +** $Id: lobject.h,v 2.20.1.2 2008/08/06 13:29:48 roberto Exp $ +** Type definitions for Lua objects +** See Copyright Notice in lua.h +*/ + + +#ifndef lobject_h +#define lobject_h + + +#include + + +#include "llimits.h" +#include "lua.h" + + +/* tags for values visible from Lua */ +#define LAST_TAG LUA_TTHREAD + +#define NUM_TAGS (LAST_TAG+1) + + +/* +** Extra tags for non-values +*/ +#define LUA_TPROTO (LAST_TAG+1) +#define LUA_TUPVAL (LAST_TAG+2) +#define LUA_TDEADKEY (LAST_TAG+3) + + +/* +** Union of all collectable objects +*/ +typedef union GCObject GCObject; + + +/* +** Common Header for all collectable objects (in macro form, to be +** included in other objects) +*/ +#define CommonHeader GCObject *next; lu_byte tt; lu_byte marked + + +/* +** Common header in struct form +*/ +typedef struct GCheader { + CommonHeader; +} GCheader; + + + + +/* +** Union of all Lua values +*/ +typedef union { + GCObject *gc; + void *p; + lua_Number n; + int b; +} Value; + + +/* +** Tagged Values +*/ + +#define TValuefields Value value; int tt + +typedef struct lua_TValue { + TValuefields; +} TValue; + + +/* Macros to test type */ +#define ttisnil(o) (ttype(o) == LUA_TNIL) +#define ttisnumber(o) (ttype(o) == LUA_TNUMBER) +#define ttisstring(o) (ttype(o) == LUA_TSTRING) +#define ttistable(o) (ttype(o) == LUA_TTABLE) +#define ttisfunction(o) (ttype(o) == LUA_TFUNCTION) +#define ttisboolean(o) (ttype(o) == LUA_TBOOLEAN) +#define ttisuserdata(o) (ttype(o) == LUA_TUSERDATA) +#define ttisthread(o) (ttype(o) == LUA_TTHREAD) +#define ttislightuserdata(o) (ttype(o) == LUA_TLIGHTUSERDATA) + +/* Macros to access values */ +#define ttype(o) ((o)->tt) +#define gcvalue(o) check_exp(iscollectable(o), (o)->value.gc) +#define pvalue(o) check_exp(ttislightuserdata(o), (o)->value.p) +#define nvalue(o) check_exp(ttisnumber(o), (o)->value.n) +#define rawtsvalue(o) check_exp(ttisstring(o), &(o)->value.gc->ts) +#define tsvalue(o) (&rawtsvalue(o)->tsv) +#define rawuvalue(o) check_exp(ttisuserdata(o), &(o)->value.gc->u) +#define uvalue(o) (&rawuvalue(o)->uv) +#define clvalue(o) check_exp(ttisfunction(o), &(o)->value.gc->cl) +#define hvalue(o) check_exp(ttistable(o), &(o)->value.gc->h) +#define bvalue(o) check_exp(ttisboolean(o), (o)->value.b) +#define thvalue(o) check_exp(ttisthread(o), &(o)->value.gc->th) + +#define l_isfalse(o) (ttisnil(o) || (ttisboolean(o) && bvalue(o) == 0)) + +/* +** for internal debug only +*/ +#define checkconsistency(obj) \ + lua_assert(!iscollectable(obj) || (ttype(obj) == (obj)->value.gc->gch.tt)) + +#define checkliveness(g,obj) \ + lua_assert(!iscollectable(obj) || \ + ((ttype(obj) == (obj)->value.gc->gch.tt) && !isdead(g, (obj)->value.gc))) + + +/* Macros to set values */ +#define setnilvalue(obj) ((obj)->tt=LUA_TNIL) + +#define setnvalue(obj,x) \ + { TValue *i_o=(obj); i_o->value.n=(x); i_o->tt=LUA_TNUMBER; } + +#define setpvalue(obj,x) \ + { TValue *i_o=(obj); i_o->value.p=(x); i_o->tt=LUA_TLIGHTUSERDATA; } + +#define setbvalue(obj,x) \ + { TValue *i_o=(obj); i_o->value.b=(x); i_o->tt=LUA_TBOOLEAN; } + +#define setsvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TSTRING; \ + checkliveness(G(L),i_o); } + +#define setuvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TUSERDATA; \ + checkliveness(G(L),i_o); } + +#define setthvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TTHREAD; \ + checkliveness(G(L),i_o); } + +#define setclvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TFUNCTION; \ + checkliveness(G(L),i_o); } + +#define sethvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TTABLE; \ + checkliveness(G(L),i_o); } + +#define setptvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TPROTO; \ + checkliveness(G(L),i_o); } + + + + +#define setobj(L,obj1,obj2) \ + { const TValue *o2=(obj2); TValue *o1=(obj1); \ + o1->value = o2->value; o1->tt=o2->tt; \ + checkliveness(G(L),o1); } + + +/* +** different types of sets, according to destination +*/ + +/* from stack to (same) stack */ +#define setobjs2s setobj +/* to stack (not from same stack) */ +#define setobj2s setobj +#define setsvalue2s setsvalue +#define sethvalue2s sethvalue +#define setptvalue2s setptvalue +/* from table to same table */ +#define setobjt2t setobj +/* to table */ +#define setobj2t setobj +/* to new object */ +#define setobj2n setobj +#define setsvalue2n setsvalue + +#define setttype(obj, tt) (ttype(obj) = (tt)) + + +#define iscollectable(o) (ttype(o) >= LUA_TSTRING) + + + +typedef TValue *StkId; /* index to stack elements */ + + +/* +** String headers for string table +*/ +typedef union TString { + L_Umaxalign dummy; /* ensures maximum alignment for strings */ + struct { + CommonHeader; + lu_byte reserved; + unsigned int hash; + size_t len; + } tsv; +} TString; + + +#define getstr(ts) cast(const char *, (ts) + 1) +#define svalue(o) getstr(rawtsvalue(o)) + + + +typedef union Udata { + L_Umaxalign dummy; /* ensures maximum alignment for `local' udata */ + struct { + CommonHeader; + struct Table *metatable; + struct Table *env; + size_t len; + } uv; +} Udata; + + + + +/* +** Function Prototypes +*/ +typedef struct Proto { + CommonHeader; + TValue *k; /* constants used by the function */ + Instruction *code; + struct Proto **p; /* functions defined inside the function */ + int *lineinfo; /* map from opcodes to source lines */ + struct LocVar *locvars; /* information about local variables */ + TString **upvalues; /* upvalue names */ + TString *source; + int sizeupvalues; + int sizek; /* size of `k' */ + int sizecode; + int sizelineinfo; + int sizep; /* size of `p' */ + int sizelocvars; + int linedefined; + int lastlinedefined; + GCObject *gclist; + lu_byte nups; /* number of upvalues */ + lu_byte numparams; + lu_byte is_vararg; + lu_byte maxstacksize; +} Proto; + + +/* masks for new-style vararg */ +#define VARARG_HASARG 1 +#define VARARG_ISVARARG 2 +#define VARARG_NEEDSARG 4 + + +typedef struct LocVar { + TString *varname; + int startpc; /* first point where variable is active */ + int endpc; /* first point where variable is dead */ +} LocVar; + + + +/* +** Upvalues +*/ + +typedef struct UpVal { + CommonHeader; + TValue *v; /* points to stack or to its own value */ + union { + TValue value; /* the value (when closed) */ + struct { /* double linked list (when open) */ + struct UpVal *prev; + struct UpVal *next; + } l; + } u; +} UpVal; + + +/* +** Closures +*/ + +#define ClosureHeader \ + CommonHeader; lu_byte isC; lu_byte nupvalues; GCObject *gclist; \ + struct Table *env + +typedef struct CClosure { + ClosureHeader; + lua_CFunction f; + TValue upvalue[1]; +} CClosure; + + +typedef struct LClosure { + ClosureHeader; + struct Proto *p; + UpVal *upvals[1]; +} LClosure; + + +typedef union Closure { + CClosure c; + LClosure l; +} Closure; + + +#define iscfunction(o) (ttype(o) == LUA_TFUNCTION && clvalue(o)->c.isC) +#define isLfunction(o) (ttype(o) == LUA_TFUNCTION && !clvalue(o)->c.isC) + + +/* +** Tables +*/ + +typedef union TKey { + struct { + TValuefields; + struct Node *next; /* for chaining */ + } nk; + TValue tvk; +} TKey; + + +typedef struct Node { + TValue i_val; + TKey i_key; +} Node; + + +typedef struct Table { + CommonHeader; + lu_byte flags; /* 1<

              lsizenode)) + + +#define luaO_nilobject (&luaO_nilobject_) + +LUAI_DATA const TValue luaO_nilobject_; + +#define ceillog2(x) (luaO_log2((x)-1) + 1) + +LUAI_FUNC int luaO_log2 (unsigned int x); +LUAI_FUNC int luaO_int2fb (unsigned int x); +LUAI_FUNC int luaO_fb2int (int x); +LUAI_FUNC int luaO_rawequalObj (const TValue *t1, const TValue *t2); +LUAI_FUNC int luaO_str2d (const char *s, lua_Number *result); +LUAI_FUNC const char *luaO_pushvfstring (lua_State *L, const char *fmt, + va_list argp); +LUAI_FUNC const char *luaO_pushfstring (lua_State *L, const char *fmt, ...); +LUAI_FUNC void luaO_chunkid (char *out, const char *source, size_t len); + + +#endif + diff --git a/src/lua/lopcodes.c b/src/lua/lopcodes.c new file mode 100644 index 0000000..4cc7452 --- /dev/null +++ b/src/lua/lopcodes.c @@ -0,0 +1,102 @@ +/* +** $Id: lopcodes.c,v 1.37.1.1 2007/12/27 13:02:25 roberto Exp $ +** See Copyright Notice in lua.h +*/ + + +#define lopcodes_c +#define LUA_CORE + + +#include "lopcodes.h" + + +/* ORDER OP */ + +const char *const luaP_opnames[NUM_OPCODES+1] = { + "MOVE", + "LOADK", + "LOADBOOL", + "LOADNIL", + "GETUPVAL", + "GETGLOBAL", + "GETTABLE", + "SETGLOBAL", + "SETUPVAL", + "SETTABLE", + "NEWTABLE", + "SELF", + "ADD", + "SUB", + "MUL", + "DIV", + "MOD", + "POW", + "UNM", + "NOT", + "LEN", + "CONCAT", + "JMP", + "EQ", + "LT", + "LE", + "TEST", + "TESTSET", + "CALL", + "TAILCALL", + "RETURN", + "FORLOOP", + "FORPREP", + "TFORLOOP", + "SETLIST", + "CLOSE", + "CLOSURE", + "VARARG", + NULL +}; + + +#define opmode(t,a,b,c,m) (((t)<<7) | ((a)<<6) | ((b)<<4) | ((c)<<2) | (m)) + +const lu_byte luaP_opmodes[NUM_OPCODES] = { +/* T A B C mode opcode */ + opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_MOVE */ + ,opmode(0, 1, OpArgK, OpArgN, iABx) /* OP_LOADK */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_LOADBOOL */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_LOADNIL */ + ,opmode(0, 1, OpArgU, OpArgN, iABC) /* OP_GETUPVAL */ + ,opmode(0, 1, OpArgK, OpArgN, iABx) /* OP_GETGLOBAL */ + ,opmode(0, 1, OpArgR, OpArgK, iABC) /* OP_GETTABLE */ + ,opmode(0, 0, OpArgK, OpArgN, iABx) /* OP_SETGLOBAL */ + ,opmode(0, 0, OpArgU, OpArgN, iABC) /* OP_SETUPVAL */ + ,opmode(0, 0, OpArgK, OpArgK, iABC) /* OP_SETTABLE */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_NEWTABLE */ + ,opmode(0, 1, OpArgR, OpArgK, iABC) /* OP_SELF */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_ADD */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_SUB */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_MUL */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_DIV */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_MOD */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_POW */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_UNM */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_NOT */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_LEN */ + ,opmode(0, 1, OpArgR, OpArgR, iABC) /* OP_CONCAT */ + ,opmode(0, 0, OpArgR, OpArgN, iAsBx) /* OP_JMP */ + ,opmode(1, 0, OpArgK, OpArgK, iABC) /* OP_EQ */ + ,opmode(1, 0, OpArgK, OpArgK, iABC) /* OP_LT */ + ,opmode(1, 0, OpArgK, OpArgK, iABC) /* OP_LE */ + ,opmode(1, 1, OpArgR, OpArgU, iABC) /* OP_TEST */ + ,opmode(1, 1, OpArgR, OpArgU, iABC) /* OP_TESTSET */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_CALL */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_TAILCALL */ + ,opmode(0, 0, OpArgU, OpArgN, iABC) /* OP_RETURN */ + ,opmode(0, 1, OpArgR, OpArgN, iAsBx) /* OP_FORLOOP */ + ,opmode(0, 1, OpArgR, OpArgN, iAsBx) /* OP_FORPREP */ + ,opmode(1, 0, OpArgN, OpArgU, iABC) /* OP_TFORLOOP */ + ,opmode(0, 0, OpArgU, OpArgU, iABC) /* OP_SETLIST */ + ,opmode(0, 0, OpArgN, OpArgN, iABC) /* OP_CLOSE */ + ,opmode(0, 1, OpArgU, OpArgN, iABx) /* OP_CLOSURE */ + ,opmode(0, 1, OpArgU, OpArgN, iABC) /* OP_VARARG */ +}; + diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h new file mode 100644 index 0000000..41224d6 --- /dev/null +++ b/src/lua/lopcodes.h @@ -0,0 +1,268 @@ +/* +** $Id: lopcodes.h,v 1.125.1.1 2007/12/27 13:02:25 roberto Exp $ +** Opcodes for Lua virtual machine +** See Copyright Notice in lua.h +*/ + +#ifndef lopcodes_h +#define lopcodes_h + +#include "llimits.h" + + +/*=========================================================================== + We assume that instructions are unsigned numbers. + All instructions have an opcode in the first 6 bits. + Instructions can have the following fields: + `A' : 8 bits + `B' : 9 bits + `C' : 9 bits + `Bx' : 18 bits (`B' and `C' together) + `sBx' : signed Bx + + A signed argument is represented in excess K; that is, the number + value is the unsigned value minus K. K is exactly the maximum value + for that argument (so that -max is represented by 0, and +max is + represented by 2*max), which is half the maximum for the corresponding + unsigned argument. +===========================================================================*/ + + +enum OpMode {iABC, iABx, iAsBx}; /* basic instruction format */ + + +/* +** size and position of opcode arguments. +*/ +#define SIZE_C 9 +#define SIZE_B 9 +#define SIZE_Bx (SIZE_C + SIZE_B) +#define SIZE_A 8 + +#define SIZE_OP 6 + +#define POS_OP 0 +#define POS_A (POS_OP + SIZE_OP) +#define POS_C (POS_A + SIZE_A) +#define POS_B (POS_C + SIZE_C) +#define POS_Bx POS_C + + +/* +** limits for opcode arguments. +** we use (signed) int to manipulate most arguments, +** so they must fit in LUAI_BITSINT-1 bits (-1 for sign) +*/ +#if SIZE_Bx < LUAI_BITSINT-1 +#define MAXARG_Bx ((1<>1) /* `sBx' is signed */ +#else +#define MAXARG_Bx MAX_INT +#define MAXARG_sBx MAX_INT +#endif + + +#define MAXARG_A ((1<>POS_OP) & MASK1(SIZE_OP,0))) +#define SET_OPCODE(i,o) ((i) = (((i)&MASK0(SIZE_OP,POS_OP)) | \ + ((cast(Instruction, o)<>POS_A) & MASK1(SIZE_A,0))) +#define SETARG_A(i,u) ((i) = (((i)&MASK0(SIZE_A,POS_A)) | \ + ((cast(Instruction, u)<>POS_B) & MASK1(SIZE_B,0))) +#define SETARG_B(i,b) ((i) = (((i)&MASK0(SIZE_B,POS_B)) | \ + ((cast(Instruction, b)<>POS_C) & MASK1(SIZE_C,0))) +#define SETARG_C(i,b) ((i) = (((i)&MASK0(SIZE_C,POS_C)) | \ + ((cast(Instruction, b)<>POS_Bx) & MASK1(SIZE_Bx,0))) +#define SETARG_Bx(i,b) ((i) = (((i)&MASK0(SIZE_Bx,POS_Bx)) | \ + ((cast(Instruction, b)< C) then pc++ */ +OP_TESTSET,/* A B C if (R(B) <=> C) then R(A) := R(B) else pc++ */ + +OP_CALL,/* A B C R(A), ... ,R(A+C-2) := R(A)(R(A+1), ... ,R(A+B-1)) */ +OP_TAILCALL,/* A B C return R(A)(R(A+1), ... ,R(A+B-1)) */ +OP_RETURN,/* A B return R(A), ... ,R(A+B-2) (see note) */ + +OP_FORLOOP,/* A sBx R(A)+=R(A+2); + if R(A) =) R(A)*/ +OP_CLOSURE,/* A Bx R(A) := closure(KPROTO[Bx], R(A), ... ,R(A+n)) */ + +OP_VARARG/* A B R(A), R(A+1), ..., R(A+B-1) = vararg */ +} OpCode; + + +#define NUM_OPCODES (cast(int, OP_VARARG) + 1) + + + +/*=========================================================================== + Notes: + (*) In OP_CALL, if (B == 0) then B = top. C is the number of returns - 1, + and can be 0: OP_CALL then sets `top' to last_result+1, so + next open instruction (OP_CALL, OP_RETURN, OP_SETLIST) may use `top'. + + (*) In OP_VARARG, if (B == 0) then use actual number of varargs and + set top (like in OP_CALL with C == 0). + + (*) In OP_RETURN, if (B == 0) then return up to `top' + + (*) In OP_SETLIST, if (B == 0) then B = `top'; + if (C == 0) then next `instruction' is real C + + (*) For comparisons, A specifies what condition the test should accept + (true or false). + + (*) All `skips' (pc++) assume that next instruction is a jump +===========================================================================*/ + + +/* +** masks for instruction properties. The format is: +** bits 0-1: op mode +** bits 2-3: C arg mode +** bits 4-5: B arg mode +** bit 6: instruction set register A +** bit 7: operator is a test +*/ + +enum OpArgMask { + OpArgN, /* argument is not used */ + OpArgU, /* argument is used */ + OpArgR, /* argument is a register or a jump offset */ + OpArgK /* argument is a constant or register/constant */ +}; + +LUAI_DATA const lu_byte luaP_opmodes[NUM_OPCODES]; + +#define getOpMode(m) (cast(enum OpMode, luaP_opmodes[m] & 3)) +#define getBMode(m) (cast(enum OpArgMask, (luaP_opmodes[m] >> 4) & 3)) +#define getCMode(m) (cast(enum OpArgMask, (luaP_opmodes[m] >> 2) & 3)) +#define testAMode(m) (luaP_opmodes[m] & (1 << 6)) +#define testTMode(m) (luaP_opmodes[m] & (1 << 7)) + + +LUAI_DATA const char *const luaP_opnames[NUM_OPCODES+1]; /* opcode names */ + + +/* number of list items to accumulate before a SETLIST instruction */ +#define LFIELDS_PER_FLUSH 50 + + +#endif diff --git a/src/lua/loslib.c b/src/lua/loslib.c new file mode 100644 index 0000000..b822e2d --- /dev/null +++ b/src/lua/loslib.c @@ -0,0 +1,322 @@ +/* +** $Id: loslib.c,v 1.19.1.3 2008/01/18 16:38:18 roberto Exp $ +** Standard Operating System library +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include +#include + +#define loslib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +static int os_pushresult (lua_State *L, int i, const char *filename) { + int en = errno; /* calls to Lua API may change this value */ + if (i) { + lua_pushboolean(L, 1); + return 1; + } + else { + lua_pushnil(L); + lua_pushfstring(L, "%s: %s", filename, strerror(en)); + lua_pushinteger(L, en); + return 3; + } +} +/* +** list of valid conversion specifiers for the 'strftime' function +*/ +#if !defined(LUA_STRFTIMEOPTIONS) + +#if !defined(LUA_USE_POSIX) +#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYzZ%", "" } +#else +#define LUA_STRFTIMEOPTIONS \ + { "aAbBcCdDeFgGhHIjklmMnprRSstTuUVwWxXyYzZ%", "" \ + "", "E", "cCxXyY", \ + "O", "deHImMSuUVwWy" } +#endif + +#endif + + + +/* +** By default, Lua uses tmpnam except when POSIX is available, where it +** uses mkstemp. +*/ +#if defined(LUA_USE_MKSTEMP) +#include +#define LUA_TMPNAMBUFSIZE 32 +#define lua_tmpnam(b,e) { \ + strcpy(b, "/tmp/lua_XXXXXX"); \ + e = mkstemp(b); \ + if (e != -1) close(e); \ + e = (e == -1); } + +#elif !defined(lua_tmpnam) + +#define LUA_TMPNAMBUFSIZE L_tmpnam +#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } + +#endif + + +/* +** By default, Lua uses gmtime/localtime, except when POSIX is available, +** where it uses gmtime_r/localtime_r +*/ +#if defined(LUA_USE_GMTIME_R) + +#define l_gmtime(t,r) gmtime_r(t,r) +#define l_localtime(t,r) localtime_r(t,r) + +#elif !defined(l_gmtime) + +#define l_gmtime(t,r) ((void)r, gmtime(t)) +#define l_localtime(t,r) ((void)r, localtime(t)) + +#endif + + + +static int os_execute (lua_State *L) { + lua_pushinteger(L, system(luaL_optstring(L, 1, NULL))); + return 1; +} + + +static int os_remove (lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + return os_pushresult(L, remove(filename) == 0, filename); +} + + +static int os_rename (lua_State *L) { + const char *fromname = luaL_checkstring(L, 1); + const char *toname = luaL_checkstring(L, 2); + return os_pushresult(L, rename(fromname, toname) == 0, fromname); +} + + +static int os_tmpname (lua_State *L) { + char buff[LUA_TMPNAMBUFSIZE]; + int err; + lua_tmpnam(buff, err); + if (err) + return luaL_error(L, "unable to generate a unique filename"); + lua_pushstring(L, buff); + return 1; +} + + +static int os_getenv (lua_State *L) { + lua_pushstring(L, getenv(luaL_checkstring(L, 1))); /* if NULL push nil */ + return 1; +} + + +static int os_clock (lua_State *L) { + lua_pushnumber(L, ((lua_Number)clock())/(lua_Number)CLOCKS_PER_SEC); + return 1; +} + + +/* +** {====================================================== +** Time/Date operations +** { year=%Y, month=%m, day=%d, hour=%H, min=%M, sec=%S, +** wday=%w+1, yday=%j, isdst=? } +** ======================================================= +*/ + +static void setfield (lua_State *L, const char *key, int value) { + lua_pushinteger(L, value); + lua_setfield(L, -2, key); +} + +static void setboolfield (lua_State *L, const char *key, int value) { + if (value < 0) /* undefined? */ + return; /* does not set field */ + lua_pushboolean(L, value); + lua_setfield(L, -2, key); +} + +static int getboolfield (lua_State *L, const char *key) { + int res; + lua_getfield(L, -1, key); + res = lua_isnil(L, -1) ? -1 : lua_toboolean(L, -1); + lua_pop(L, 1); + return res; +} + + +static int getfield (lua_State *L, const char *key, int d) { + int res; + lua_getfield(L, -1, key); + if (lua_isnumber(L, -1)) + res = (int)lua_tointeger(L, -1); + else { + if (d < 0) + return luaL_error(L, "field " LUA_QS " missing in date table", key); + res = d; + } + lua_pop(L, 1); + return res; +} + + +static const char *checkoption (lua_State *L, const char *conv, char *buff) { + static const char *const options[] = LUA_STRFTIMEOPTIONS; + unsigned int i; + for (i = 0; i < sizeof(options)/sizeof(options[0]); i += 2) { + if (*conv != '\0' && strchr(options[i], *conv) != NULL) { + buff[1] = *conv; + if (*options[i + 1] == '\0') { /* one-char conversion specifier? */ + buff[2] = '\0'; /* end buffer */ + return conv + 1; + } + else if (*(conv + 1) != '\0' && + strchr(options[i + 1], *(conv + 1)) != NULL) { + buff[2] = *(conv + 1); /* valid two-char conversion specifier */ + buff[3] = '\0'; /* end buffer */ + return conv + 2; + } + } + } + luaL_argerror(L, 1, + lua_pushfstring(L, "invalid conversion specifier '%%%s'", conv)); + return conv; /* to avoid warnings */ +} + + +static int os_date (lua_State *L) { + const char *s = luaL_optstring(L, 1, "%c"); + time_t t = luaL_opt(L, (time_t)luaL_checknumber, 2, time(NULL)); + struct tm tmr, *stm; + if (*s == '!') { /* UTC? */ + stm = l_gmtime(&t, &tmr); + s++; /* skip `!' */ + } + else + stm = l_localtime(&t, &tmr); + if (stm == NULL) /* invalid date? */ + lua_pushnil(L); + else if (strcmp(s, "*t") == 0) { + lua_createtable(L, 0, 9); /* 9 = number of fields */ + setfield(L, "sec", stm->tm_sec); + setfield(L, "min", stm->tm_min); + setfield(L, "hour", stm->tm_hour); + setfield(L, "day", stm->tm_mday); + setfield(L, "month", stm->tm_mon+1); + setfield(L, "year", stm->tm_year+1900); + setfield(L, "wday", stm->tm_wday+1); + setfield(L, "yday", stm->tm_yday+1); + setboolfield(L, "isdst", stm->tm_isdst); + } + else { + char cc[4]; + luaL_Buffer b; + cc[0] = '%'; + luaL_buffinit(L, &b); + while (*s) { + if (*s != '%') /* no conversion specifier? */ + luaL_addchar(&b, *s++); + else { + size_t reslen; + char buff[200]; /* should be big enough for any conversion result */ + s = checkoption(L, s + 1, cc); + reslen = strftime(buff, sizeof(buff), cc, stm); + luaL_addlstring(&b, buff, reslen); + } + } + luaL_pushresult(&b); + } + return 1; +} + + +static int os_time (lua_State *L) { + time_t t; + if (lua_isnoneornil(L, 1)) /* called without args? */ + t = time(NULL); /* get current time */ + else { + struct tm ts; + luaL_checktype(L, 1, LUA_TTABLE); + lua_settop(L, 1); /* make sure table is at the top */ + ts.tm_sec = getfield(L, "sec", 0); + ts.tm_min = getfield(L, "min", 0); + ts.tm_hour = getfield(L, "hour", 12); + ts.tm_mday = getfield(L, "day", -1); + ts.tm_mon = getfield(L, "month", -1) - 1; + ts.tm_year = getfield(L, "year", -1) - 1900; + ts.tm_isdst = getboolfield(L, "isdst"); + t = mktime(&ts); + } + if (t == (time_t)(-1)) + lua_pushnil(L); + else + lua_pushnumber(L, (lua_Number)t); + return 1; +} + + +static int os_difftime (lua_State *L) { + lua_pushnumber(L, difftime((time_t)(luaL_checknumber(L, 1)), + (time_t)(luaL_optnumber(L, 2, 0)))); + return 1; +} + +/* }====================================================== */ + + +static int os_setlocale (lua_State *L) { + static const int cat[] = {LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, + LC_NUMERIC, LC_TIME}; + static const char *const catnames[] = {"all", "collate", "ctype", "monetary", + "numeric", "time", NULL}; + const char *l = luaL_optstring(L, 1, NULL); + int op = luaL_checkoption(L, 2, "all", catnames); + lua_pushstring(L, setlocale(cat[op], l)); + return 1; +} + + +static int os_exit (lua_State *L) { + exit(luaL_optint(L, 1, EXIT_SUCCESS)); +} + +static const luaL_Reg syslib[] = { + {"clock", os_clock}, + {"date", os_date}, + {"difftime", os_difftime}, + {"execute", os_execute}, + {"exit", os_exit}, + {"getenv", os_getenv}, + {"remove", os_remove}, + {"rename", os_rename}, + {"setlocale", os_setlocale}, + {"time", os_time}, + {"tmpname", os_tmpname}, + {NULL, NULL} +}; + +/* }====================================================== */ + + + +LUALIB_API int luaopen_os (lua_State *L) { + luaL_register(L, LUA_OSLIBNAME, syslib); + return 1; +} + diff --git a/src/lua/lparser.c b/src/lua/lparser.c new file mode 100644 index 0000000..dda7488 --- /dev/null +++ b/src/lua/lparser.c @@ -0,0 +1,1339 @@ +/* +** $Id: lparser.c,v 2.42.1.4 2011/10/21 19:31:42 roberto Exp $ +** Lua Parser +** See Copyright Notice in lua.h +*/ + + +#include + +#define lparser_c +#define LUA_CORE + +#include "lua.h" + +#include "lcode.h" +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "llex.h" +#include "lmem.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" + + + +#define hasmultret(k) ((k) == VCALL || (k) == VVARARG) + +#define getlocvar(fs, i) ((fs)->f->locvars[(fs)->actvar[i]]) + +#define luaY_checklimit(fs,v,l,m) if ((v)>(l)) errorlimit(fs,l,m) + + +/* +** nodes for block list (list of active blocks) +*/ +typedef struct BlockCnt { + struct BlockCnt *previous; /* chain */ + int breaklist; /* list of jumps out of this loop */ + lu_byte nactvar; /* # active locals outside the breakable structure */ + lu_byte upval; /* true if some variable in the block is an upvalue */ + lu_byte isbreakable; /* true if `block' is a loop */ +} BlockCnt; + + + +/* +** prototypes for recursive non-terminal functions +*/ +static void chunk (LexState *ls); +static void expr (LexState *ls, expdesc *v); + + +static void anchor_token (LexState *ls) { + if (ls->t.token == TK_NAME || ls->t.token == TK_STRING) { + TString *ts = ls->t.seminfo.ts; + luaX_newstring(ls, getstr(ts), ts->tsv.len); + } +} + + +static void error_expected (LexState *ls, int token) { + luaX_syntaxerror(ls, + luaO_pushfstring(ls->L, LUA_QS " expected", luaX_token2str(ls, token))); +} + + +static void errorlimit (FuncState *fs, int limit, const char *what) { + const char *msg = (fs->f->linedefined == 0) ? + luaO_pushfstring(fs->L, "main function has more than %d %s", limit, what) : + luaO_pushfstring(fs->L, "function at line %d has more than %d %s", + fs->f->linedefined, limit, what); + luaX_lexerror(fs->ls, msg, 0); +} + + +static int testnext (LexState *ls, int c) { + if (ls->t.token == c) { + luaX_next(ls); + return 1; + } + else return 0; +} + + +static void check (LexState *ls, int c) { + if (ls->t.token != c) + error_expected(ls, c); +} + +static void checknext (LexState *ls, int c) { + check(ls, c); + luaX_next(ls); +} + + +#define check_condition(ls,c,msg) { if (!(c)) luaX_syntaxerror(ls, msg); } + + + +static void check_match (LexState *ls, int what, int who, int where) { + if (!testnext(ls, what)) { + if (where == ls->linenumber) + error_expected(ls, what); + else { + luaX_syntaxerror(ls, luaO_pushfstring(ls->L, + LUA_QS " expected (to close " LUA_QS " at line %d)", + luaX_token2str(ls, what), luaX_token2str(ls, who), where)); + } + } +} + + +static TString *str_checkname (LexState *ls) { + TString *ts; + check(ls, TK_NAME); + ts = ls->t.seminfo.ts; + luaX_next(ls); + return ts; +} + + +static void init_exp (expdesc *e, expkind k, int i) { + e->f = e->t = NO_JUMP; + e->k = k; + e->u.s.info = i; +} + + +static void codestring (LexState *ls, expdesc *e, TString *s) { + init_exp(e, VK, luaK_stringK(ls->fs, s)); +} + + +static void checkname(LexState *ls, expdesc *e) { + codestring(ls, e, str_checkname(ls)); +} + + +static int registerlocalvar (LexState *ls, TString *varname) { + FuncState *fs = ls->fs; + Proto *f = fs->f; + int oldsize = f->sizelocvars; + luaM_growvector(ls->L, f->locvars, fs->nlocvars, f->sizelocvars, + LocVar, SHRT_MAX, "too many local variables"); + while (oldsize < f->sizelocvars) f->locvars[oldsize++].varname = NULL; + f->locvars[fs->nlocvars].varname = varname; + luaC_objbarrier(ls->L, f, varname); + return fs->nlocvars++; +} + + +#define new_localvarliteral(ls,v,n) \ + new_localvar(ls, luaX_newstring(ls, "" v, (sizeof(v)/sizeof(char))-1), n) + + +static void new_localvar (LexState *ls, TString *name, int n) { + FuncState *fs = ls->fs; + luaY_checklimit(fs, fs->nactvar+n+1, LUAI_MAXVARS, "local variables"); + fs->actvar[fs->nactvar+n] = cast(unsigned short, registerlocalvar(ls, name)); +} + + +static void adjustlocalvars (LexState *ls, int nvars) { + FuncState *fs = ls->fs; + fs->nactvar = cast_byte(fs->nactvar + nvars); + for (; nvars; nvars--) { + getlocvar(fs, fs->nactvar - nvars).startpc = fs->pc; + } +} + + +static void removevars (LexState *ls, int tolevel) { + FuncState *fs = ls->fs; + while (fs->nactvar > tolevel) + getlocvar(fs, --fs->nactvar).endpc = fs->pc; +} + + +static int indexupvalue (FuncState *fs, TString *name, expdesc *v) { + int i; + Proto *f = fs->f; + int oldsize = f->sizeupvalues; + for (i=0; inups; i++) { + if (fs->upvalues[i].k == v->k && fs->upvalues[i].info == v->u.s.info) { + lua_assert(f->upvalues[i] == name); + return i; + } + } + /* new one */ + luaY_checklimit(fs, f->nups + 1, LUAI_MAXUPVALUES, "upvalues"); + luaM_growvector(fs->L, f->upvalues, f->nups, f->sizeupvalues, + TString *, MAX_INT, ""); + while (oldsize < f->sizeupvalues) f->upvalues[oldsize++] = NULL; + f->upvalues[f->nups] = name; + luaC_objbarrier(fs->L, f, name); + lua_assert(v->k == VLOCAL || v->k == VUPVAL); + fs->upvalues[f->nups].k = cast_byte(v->k); + fs->upvalues[f->nups].info = cast_byte(v->u.s.info); + return f->nups++; +} + + +static int searchvar (FuncState *fs, TString *n) { + int i; + for (i=fs->nactvar-1; i >= 0; i--) { + if (n == getlocvar(fs, i).varname) + return i; + } + return -1; /* not found */ +} + + +static void markupval (FuncState *fs, int level) { + BlockCnt *bl = fs->bl; + while (bl && bl->nactvar > level) bl = bl->previous; + if (bl) bl->upval = 1; +} + + +static int singlevaraux (FuncState *fs, TString *n, expdesc *var, int base) { + if (fs == NULL) { /* no more levels? */ + init_exp(var, VGLOBAL, NO_REG); /* default is global variable */ + return VGLOBAL; + } + else { + int v = searchvar(fs, n); /* look up at current level */ + if (v >= 0) { + init_exp(var, VLOCAL, v); + if (!base) + markupval(fs, v); /* local will be used as an upval */ + return VLOCAL; + } + else { /* not found at current level; try upper one */ + if (singlevaraux(fs->prev, n, var, 0) == VGLOBAL) + return VGLOBAL; + var->u.s.info = indexupvalue(fs, n, var); /* else was LOCAL or UPVAL */ + var->k = VUPVAL; /* upvalue in this level */ + return VUPVAL; + } + } +} + + +static void singlevar (LexState *ls, expdesc *var) { + TString *varname = str_checkname(ls); + FuncState *fs = ls->fs; + if (singlevaraux(fs, varname, var, 1) == VGLOBAL) + var->u.s.info = luaK_stringK(fs, varname); /* info points to global name */ +} + + +static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) { + FuncState *fs = ls->fs; + int extra = nvars - nexps; + if (hasmultret(e->k)) { + extra++; /* includes call itself */ + if (extra < 0) extra = 0; + luaK_setreturns(fs, e, extra); /* last exp. provides the difference */ + if (extra > 1) luaK_reserveregs(fs, extra-1); + } + else { + if (e->k != VVOID) luaK_exp2nextreg(fs, e); /* close last expression */ + if (extra > 0) { + int reg = fs->freereg; + luaK_reserveregs(fs, extra); + luaK_nil(fs, reg, extra); + } + } +} + + +static void enterlevel (LexState *ls) { + if (++ls->L->nCcalls > LUAI_MAXCCALLS) + luaX_lexerror(ls, "chunk has too many syntax levels", 0); +} + + +#define leavelevel(ls) ((ls)->L->nCcalls--) + + +static void enterblock (FuncState *fs, BlockCnt *bl, lu_byte isbreakable) { + bl->breaklist = NO_JUMP; + bl->isbreakable = isbreakable; + bl->nactvar = fs->nactvar; + bl->upval = 0; + bl->previous = fs->bl; + fs->bl = bl; + lua_assert(fs->freereg == fs->nactvar); +} + + +static void leaveblock (FuncState *fs) { + BlockCnt *bl = fs->bl; + fs->bl = bl->previous; + removevars(fs->ls, bl->nactvar); + if (bl->upval) + luaK_codeABC(fs, OP_CLOSE, bl->nactvar, 0, 0); + /* a block either controls scope or breaks (never both) */ + lua_assert(!bl->isbreakable || !bl->upval); + lua_assert(bl->nactvar == fs->nactvar); + fs->freereg = fs->nactvar; /* free registers */ + luaK_patchtohere(fs, bl->breaklist); +} + + +static void pushclosure (LexState *ls, FuncState *func, expdesc *v) { + FuncState *fs = ls->fs; + Proto *f = fs->f; + int oldsize = f->sizep; + int i; + luaM_growvector(ls->L, f->p, fs->np, f->sizep, Proto *, + MAXARG_Bx, "constant table overflow"); + while (oldsize < f->sizep) f->p[oldsize++] = NULL; + f->p[fs->np++] = func->f; + luaC_objbarrier(ls->L, f, func->f); + init_exp(v, VRELOCABLE, luaK_codeABx(fs, OP_CLOSURE, 0, fs->np-1)); + for (i=0; if->nups; i++) { + OpCode o = (func->upvalues[i].k == VLOCAL) ? OP_MOVE : OP_GETUPVAL; + luaK_codeABC(fs, o, 0, func->upvalues[i].info, 0); + } +} + + +static void open_func (LexState *ls, FuncState *fs) { + lua_State *L = ls->L; + Proto *f = luaF_newproto(L); + fs->f = f; + fs->prev = ls->fs; /* linked list of funcstates */ + fs->ls = ls; + fs->L = L; + ls->fs = fs; + fs->pc = 0; + fs->lasttarget = -1; + fs->jpc = NO_JUMP; + fs->freereg = 0; + fs->nk = 0; + fs->np = 0; + fs->nlocvars = 0; + fs->nactvar = 0; + fs->bl = NULL; + f->source = ls->source; + f->maxstacksize = 2; /* registers 0/1 are always valid */ + fs->h = luaH_new(L, 0, 0); + /* anchor table of constants and prototype (to avoid being collected) */ + sethvalue2s(L, L->top, fs->h); + incr_top(L); + setptvalue2s(L, L->top, f); + incr_top(L); +} + + +static void close_func (LexState *ls) { + lua_State *L = ls->L; + FuncState *fs = ls->fs; + Proto *f = fs->f; + removevars(ls, 0); + luaK_ret(fs, 0, 0); /* final return */ + luaM_reallocvector(L, f->code, f->sizecode, fs->pc, Instruction); + f->sizecode = fs->pc; + luaM_reallocvector(L, f->lineinfo, f->sizelineinfo, fs->pc, int); + f->sizelineinfo = fs->pc; + luaM_reallocvector(L, f->k, f->sizek, fs->nk, TValue); + f->sizek = fs->nk; + luaM_reallocvector(L, f->p, f->sizep, fs->np, Proto *); + f->sizep = fs->np; + luaM_reallocvector(L, f->locvars, f->sizelocvars, fs->nlocvars, LocVar); + f->sizelocvars = fs->nlocvars; + luaM_reallocvector(L, f->upvalues, f->sizeupvalues, f->nups, TString *); + f->sizeupvalues = f->nups; + lua_assert(luaG_checkcode(f)); + lua_assert(fs->bl == NULL); + ls->fs = fs->prev; + /* last token read was anchored in defunct function; must reanchor it */ + if (fs) anchor_token(ls); + L->top -= 2; /* remove table and prototype from the stack */ +} + + +Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, const char *name) { + struct LexState lexstate; + struct FuncState funcstate; + lexstate.buff = buff; + luaX_setinput(L, &lexstate, z, luaS_new(L, name)); + open_func(&lexstate, &funcstate); + funcstate.f->is_vararg = VARARG_ISVARARG; /* main func. is always vararg */ + luaX_next(&lexstate); /* read first token */ + chunk(&lexstate); + check(&lexstate, TK_EOS); + close_func(&lexstate); + lua_assert(funcstate.prev == NULL); + lua_assert(funcstate.f->nups == 0); + lua_assert(lexstate.fs == NULL); + return funcstate.f; +} + + + +/*============================================================*/ +/* GRAMMAR RULES */ +/*============================================================*/ + + +static void field (LexState *ls, expdesc *v) { + /* field -> ['.' | ':'] NAME */ + FuncState *fs = ls->fs; + expdesc key; + luaK_exp2anyreg(fs, v); + luaX_next(ls); /* skip the dot or colon */ + checkname(ls, &key); + luaK_indexed(fs, v, &key); +} + + +static void yindex (LexState *ls, expdesc *v) { + /* index -> '[' expr ']' */ + luaX_next(ls); /* skip the '[' */ + expr(ls, v); + luaK_exp2val(ls->fs, v); + checknext(ls, ']'); +} + + +/* +** {====================================================================== +** Rules for Constructors +** ======================================================================= +*/ + + +struct ConsControl { + expdesc v; /* last list item read */ + expdesc *t; /* table descriptor */ + int nh; /* total number of `record' elements */ + int na; /* total number of array elements */ + int tostore; /* number of array elements pending to be stored */ +}; + + +static void recfield (LexState *ls, struct ConsControl *cc) { + /* recfield -> (NAME | `['exp1`]') = exp1 */ + FuncState *fs = ls->fs; + int reg = ls->fs->freereg; + expdesc key, val; + int rkkey; + if (ls->t.token == TK_NAME) { + luaY_checklimit(fs, cc->nh, MAX_INT, "items in a constructor"); + checkname(ls, &key); + } + else /* ls->t.token == '[' */ + yindex(ls, &key); + cc->nh++; + checknext(ls, '='); + rkkey = luaK_exp2RK(fs, &key); + expr(ls, &val); + luaK_codeABC(fs, OP_SETTABLE, cc->t->u.s.info, rkkey, luaK_exp2RK(fs, &val)); + fs->freereg = reg; /* free registers */ +} + + +static void closelistfield (FuncState *fs, struct ConsControl *cc) { + if (cc->v.k == VVOID) return; /* there is no list item */ + luaK_exp2nextreg(fs, &cc->v); + cc->v.k = VVOID; + if (cc->tostore == LFIELDS_PER_FLUSH) { + luaK_setlist(fs, cc->t->u.s.info, cc->na, cc->tostore); /* flush */ + cc->tostore = 0; /* no more items pending */ + } +} + + +static void lastlistfield (FuncState *fs, struct ConsControl *cc) { + if (cc->tostore == 0) return; + if (hasmultret(cc->v.k)) { + luaK_setmultret(fs, &cc->v); + luaK_setlist(fs, cc->t->u.s.info, cc->na, LUA_MULTRET); + cc->na--; /* do not count last expression (unknown number of elements) */ + } + else { + if (cc->v.k != VVOID) + luaK_exp2nextreg(fs, &cc->v); + luaK_setlist(fs, cc->t->u.s.info, cc->na, cc->tostore); + } +} + + +static void listfield (LexState *ls, struct ConsControl *cc) { + expr(ls, &cc->v); + luaY_checklimit(ls->fs, cc->na, MAX_INT, "items in a constructor"); + cc->na++; + cc->tostore++; +} + + +static void constructor (LexState *ls, expdesc *t) { + /* constructor -> ?? */ + FuncState *fs = ls->fs; + int line = ls->linenumber; + int pc = luaK_codeABC(fs, OP_NEWTABLE, 0, 0, 0); + struct ConsControl cc; + cc.na = cc.nh = cc.tostore = 0; + cc.t = t; + init_exp(t, VRELOCABLE, pc); + init_exp(&cc.v, VVOID, 0); /* no value (yet) */ + luaK_exp2nextreg(ls->fs, t); /* fix it at stack top (for gc) */ + checknext(ls, '{'); + do { + lua_assert(cc.v.k == VVOID || cc.tostore > 0); + if (ls->t.token == '}') break; + closelistfield(fs, &cc); + switch(ls->t.token) { + case TK_NAME: { /* may be listfields or recfields */ + luaX_lookahead(ls); + if (ls->lookahead.token != '=') /* expression? */ + listfield(ls, &cc); + else + recfield(ls, &cc); + break; + } + case '[': { /* constructor_item -> recfield */ + recfield(ls, &cc); + break; + } + default: { /* constructor_part -> listfield */ + listfield(ls, &cc); + break; + } + } + } while (testnext(ls, ',') || testnext(ls, ';')); + check_match(ls, '}', '{', line); + lastlistfield(fs, &cc); + SETARG_B(fs->f->code[pc], luaO_int2fb(cc.na)); /* set initial array size */ + SETARG_C(fs->f->code[pc], luaO_int2fb(cc.nh)); /* set initial table size */ +} + +/* }====================================================================== */ + + + +static void parlist (LexState *ls) { + /* parlist -> [ param { `,' param } ] */ + FuncState *fs = ls->fs; + Proto *f = fs->f; + int nparams = 0; + f->is_vararg = 0; + if (ls->t.token != ')') { /* is `parlist' not empty? */ + do { + switch (ls->t.token) { + case TK_NAME: { /* param -> NAME */ + new_localvar(ls, str_checkname(ls), nparams++); + break; + } + case TK_DOTS: { /* param -> `...' */ + luaX_next(ls); +#if defined(LUA_COMPAT_VARARG) + /* use `arg' as default name */ + new_localvarliteral(ls, "arg", nparams++); + f->is_vararg = VARARG_HASARG | VARARG_NEEDSARG; +#endif + f->is_vararg |= VARARG_ISVARARG; + break; + } + default: luaX_syntaxerror(ls, " or " LUA_QL("...") " expected"); + } + } while (!f->is_vararg && testnext(ls, ',')); + } + adjustlocalvars(ls, nparams); + f->numparams = cast_byte(fs->nactvar - (f->is_vararg & VARARG_HASARG)); + luaK_reserveregs(fs, fs->nactvar); /* reserve register for parameters */ +} + + +static void body (LexState *ls, expdesc *e, int needself, int line) { + /* body -> `(' parlist `)' chunk END */ + FuncState new_fs; + open_func(ls, &new_fs); + new_fs.f->linedefined = line; + checknext(ls, '('); + if (needself) { + new_localvarliteral(ls, "self", 0); + adjustlocalvars(ls, 1); + } + parlist(ls); + checknext(ls, ')'); + chunk(ls); + new_fs.f->lastlinedefined = ls->linenumber; + check_match(ls, TK_END, TK_FUNCTION, line); + close_func(ls); + pushclosure(ls, &new_fs, e); +} + + +static int explist1 (LexState *ls, expdesc *v) { + /* explist1 -> expr { `,' expr } */ + int n = 1; /* at least one expression */ + expr(ls, v); + while (testnext(ls, ',')) { + luaK_exp2nextreg(ls->fs, v); + expr(ls, v); + n++; + } + return n; +} + + +static void funcargs (LexState *ls, expdesc *f) { + FuncState *fs = ls->fs; + expdesc args; + int base, nparams; + int line = ls->linenumber; + switch (ls->t.token) { + case '(': { /* funcargs -> `(' [ explist1 ] `)' */ + if (line != ls->lastline) + luaX_syntaxerror(ls,"ambiguous syntax (function call x new statement)"); + luaX_next(ls); + if (ls->t.token == ')') /* arg list is empty? */ + args.k = VVOID; + else { + explist1(ls, &args); + luaK_setmultret(fs, &args); + } + check_match(ls, ')', '(', line); + break; + } + case '{': { /* funcargs -> constructor */ + constructor(ls, &args); + break; + } + case TK_STRING: { /* funcargs -> STRING */ + codestring(ls, &args, ls->t.seminfo.ts); + luaX_next(ls); /* must use `seminfo' before `next' */ + break; + } + default: { + luaX_syntaxerror(ls, "function arguments expected"); + return; + } + } + lua_assert(f->k == VNONRELOC); + base = f->u.s.info; /* base register for call */ + if (hasmultret(args.k)) + nparams = LUA_MULTRET; /* open call */ + else { + if (args.k != VVOID) + luaK_exp2nextreg(fs, &args); /* close last argument */ + nparams = fs->freereg - (base+1); + } + init_exp(f, VCALL, luaK_codeABC(fs, OP_CALL, base, nparams+1, 2)); + luaK_fixline(fs, line); + fs->freereg = base+1; /* call remove function and arguments and leaves + (unless changed) one result */ +} + + + + +/* +** {====================================================================== +** Expression parsing +** ======================================================================= +*/ + + +static void prefixexp (LexState *ls, expdesc *v) { + /* prefixexp -> NAME | '(' expr ')' */ + switch (ls->t.token) { + case '(': { + int line = ls->linenumber; + luaX_next(ls); + expr(ls, v); + check_match(ls, ')', '(', line); + luaK_dischargevars(ls->fs, v); + return; + } + case TK_NAME: { + singlevar(ls, v); + return; + } + default: { + luaX_syntaxerror(ls, "unexpected symbol"); + return; + } + } +} + + +static void primaryexp (LexState *ls, expdesc *v) { + /* primaryexp -> + prefixexp { `.' NAME | `[' exp `]' | `:' NAME funcargs | funcargs } */ + FuncState *fs = ls->fs; + prefixexp(ls, v); + for (;;) { + switch (ls->t.token) { + case '.': { /* field */ + field(ls, v); + break; + } + case '[': { /* `[' exp1 `]' */ + expdesc key; + luaK_exp2anyreg(fs, v); + yindex(ls, &key); + luaK_indexed(fs, v, &key); + break; + } + case ':': { /* `:' NAME funcargs */ + expdesc key; + luaX_next(ls); + checkname(ls, &key); + luaK_self(fs, v, &key); + funcargs(ls, v); + break; + } + case '(': case TK_STRING: case '{': { /* funcargs */ + luaK_exp2nextreg(fs, v); + funcargs(ls, v); + break; + } + default: return; + } + } +} + + +static void simpleexp (LexState *ls, expdesc *v) { + /* simpleexp -> NUMBER | STRING | NIL | true | false | ... | + constructor | FUNCTION body | primaryexp */ + switch (ls->t.token) { + case TK_NUMBER: { + init_exp(v, VKNUM, 0); + v->u.nval = ls->t.seminfo.r; + break; + } + case TK_STRING: { + codestring(ls, v, ls->t.seminfo.ts); + break; + } + case TK_NIL: { + init_exp(v, VNIL, 0); + break; + } + case TK_TRUE: { + init_exp(v, VTRUE, 0); + break; + } + case TK_FALSE: { + init_exp(v, VFALSE, 0); + break; + } + case TK_DOTS: { /* vararg */ + FuncState *fs = ls->fs; + check_condition(ls, fs->f->is_vararg, + "cannot use " LUA_QL("...") " outside a vararg function"); + fs->f->is_vararg &= ~VARARG_NEEDSARG; /* don't need 'arg' */ + init_exp(v, VVARARG, luaK_codeABC(fs, OP_VARARG, 0, 1, 0)); + break; + } + case '{': { /* constructor */ + constructor(ls, v); + return; + } + case TK_FUNCTION: { + luaX_next(ls); + body(ls, v, 0, ls->linenumber); + return; + } + default: { + primaryexp(ls, v); + return; + } + } + luaX_next(ls); +} + + +static UnOpr getunopr (int op) { + switch (op) { + case TK_NOT: return OPR_NOT; + case '-': return OPR_MINUS; + case '#': return OPR_LEN; + default: return OPR_NOUNOPR; + } +} + + +static BinOpr getbinopr (int op) { + switch (op) { + case '+': return OPR_ADD; + case '-': return OPR_SUB; + case '*': return OPR_MUL; + case '/': return OPR_DIV; + case '%': return OPR_MOD; + case '^': return OPR_POW; + case TK_CONCAT: return OPR_CONCAT; + case TK_NE: return OPR_NE; + case TK_EQ: return OPR_EQ; + case '<': return OPR_LT; + case TK_LE: return OPR_LE; + case '>': return OPR_GT; + case TK_GE: return OPR_GE; + case TK_AND: return OPR_AND; + case TK_OR: return OPR_OR; + default: return OPR_NOBINOPR; + } +} + + +static const struct { + lu_byte left; /* left priority for each binary operator */ + lu_byte right; /* right priority */ +} priority[] = { /* ORDER OPR */ + {6, 6}, {6, 6}, {7, 7}, {7, 7}, {7, 7}, /* `+' `-' `/' `%' */ + {10, 9}, {5, 4}, /* power and concat (right associative) */ + {3, 3}, {3, 3}, /* equality and inequality */ + {3, 3}, {3, 3}, {3, 3}, {3, 3}, /* order */ + {2, 2}, {1, 1} /* logical (and/or) */ +}; + +#define UNARY_PRIORITY 8 /* priority for unary operators */ + + +/* +** subexpr -> (simpleexp | unop subexpr) { binop subexpr } +** where `binop' is any binary operator with a priority higher than `limit' +*/ +static BinOpr subexpr (LexState *ls, expdesc *v, unsigned int limit) { + BinOpr op; + UnOpr uop; + enterlevel(ls); + uop = getunopr(ls->t.token); + if (uop != OPR_NOUNOPR) { + luaX_next(ls); + subexpr(ls, v, UNARY_PRIORITY); + luaK_prefix(ls->fs, uop, v); + } + else simpleexp(ls, v); + /* expand while operators have priorities higher than `limit' */ + op = getbinopr(ls->t.token); + while (op != OPR_NOBINOPR && priority[op].left > limit) { + expdesc v2; + BinOpr nextop; + luaX_next(ls); + luaK_infix(ls->fs, op, v); + /* read sub-expression with higher priority */ + nextop = subexpr(ls, &v2, priority[op].right); + luaK_posfix(ls->fs, op, v, &v2); + op = nextop; + } + leavelevel(ls); + return op; /* return first untreated operator */ +} + + +static void expr (LexState *ls, expdesc *v) { + subexpr(ls, v, 0); +} + +/* }==================================================================== */ + + + +/* +** {====================================================================== +** Rules for Statements +** ======================================================================= +*/ + + +static int block_follow (int token) { + switch (token) { + case TK_ELSE: case TK_ELSEIF: case TK_END: + case TK_UNTIL: case TK_EOS: + return 1; + default: return 0; + } +} + + +static void block (LexState *ls) { + /* block -> chunk */ + FuncState *fs = ls->fs; + BlockCnt bl; + enterblock(fs, &bl, 0); + chunk(ls); + lua_assert(bl.breaklist == NO_JUMP); + leaveblock(fs); +} + + +/* +** structure to chain all variables in the left-hand side of an +** assignment +*/ +struct LHS_assign { + struct LHS_assign *prev; + expdesc v; /* variable (global, local, upvalue, or indexed) */ +}; + + +/* +** check whether, in an assignment to a local variable, the local variable +** is needed in a previous assignment (to a table). If so, save original +** local value in a safe place and use this safe copy in the previous +** assignment. +*/ +static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) { + FuncState *fs = ls->fs; + int extra = fs->freereg; /* eventual position to save local variable */ + int conflict = 0; + for (; lh; lh = lh->prev) { + if (lh->v.k == VINDEXED) { + if (lh->v.u.s.info == v->u.s.info) { /* conflict? */ + conflict = 1; + lh->v.u.s.info = extra; /* previous assignment will use safe copy */ + } + if (lh->v.u.s.aux == v->u.s.info) { /* conflict? */ + conflict = 1; + lh->v.u.s.aux = extra; /* previous assignment will use safe copy */ + } + } + } + if (conflict) { + luaK_codeABC(fs, OP_MOVE, fs->freereg, v->u.s.info, 0); /* make copy */ + luaK_reserveregs(fs, 1); + } +} + + +static void assignment (LexState *ls, struct LHS_assign *lh, int nvars) { + expdesc e; + check_condition(ls, VLOCAL <= lh->v.k && lh->v.k <= VINDEXED, + "syntax error"); + if (testnext(ls, ',')) { /* assignment -> `,' primaryexp assignment */ + struct LHS_assign nv; + nv.prev = lh; + primaryexp(ls, &nv.v); + if (nv.v.k == VLOCAL) + check_conflict(ls, lh, &nv.v); + luaY_checklimit(ls->fs, nvars, LUAI_MAXCCALLS - ls->L->nCcalls, + "variables in assignment"); + assignment(ls, &nv, nvars+1); + } + else { /* assignment -> `=' explist1 */ + int nexps; + checknext(ls, '='); + nexps = explist1(ls, &e); + if (nexps != nvars) { + adjust_assign(ls, nvars, nexps, &e); + if (nexps > nvars) + ls->fs->freereg -= nexps - nvars; /* remove extra values */ + } + else { + luaK_setoneret(ls->fs, &e); /* close last expression */ + luaK_storevar(ls->fs, &lh->v, &e); + return; /* avoid default */ + } + } + init_exp(&e, VNONRELOC, ls->fs->freereg-1); /* default assignment */ + luaK_storevar(ls->fs, &lh->v, &e); +} + + +static int cond (LexState *ls) { + /* cond -> exp */ + expdesc v; + expr(ls, &v); /* read condition */ + if (v.k == VNIL) v.k = VFALSE; /* `falses' are all equal here */ + luaK_goiftrue(ls->fs, &v); + return v.f; +} + + +static void breakstat (LexState *ls) { + FuncState *fs = ls->fs; + BlockCnt *bl = fs->bl; + int upval = 0; + while (bl && !bl->isbreakable) { + upval |= bl->upval; + bl = bl->previous; + } + if (!bl) + luaX_syntaxerror(ls, "no loop to break"); + if (upval) + luaK_codeABC(fs, OP_CLOSE, bl->nactvar, 0, 0); + luaK_concat(fs, &bl->breaklist, luaK_jump(fs)); +} + + +static void whilestat (LexState *ls, int line) { + /* whilestat -> WHILE cond DO block END */ + FuncState *fs = ls->fs; + int whileinit; + int condexit; + BlockCnt bl; + luaX_next(ls); /* skip WHILE */ + whileinit = luaK_getlabel(fs); + condexit = cond(ls); + enterblock(fs, &bl, 1); + checknext(ls, TK_DO); + block(ls); + luaK_patchlist(fs, luaK_jump(fs), whileinit); + check_match(ls, TK_END, TK_WHILE, line); + leaveblock(fs); + luaK_patchtohere(fs, condexit); /* false conditions finish the loop */ +} + + +static void repeatstat (LexState *ls, int line) { + /* repeatstat -> REPEAT block UNTIL cond */ + int condexit; + FuncState *fs = ls->fs; + int repeat_init = luaK_getlabel(fs); + BlockCnt bl1, bl2; + enterblock(fs, &bl1, 1); /* loop block */ + enterblock(fs, &bl2, 0); /* scope block */ + luaX_next(ls); /* skip REPEAT */ + chunk(ls); + check_match(ls, TK_UNTIL, TK_REPEAT, line); + condexit = cond(ls); /* read condition (inside scope block) */ + if (!bl2.upval) { /* no upvalues? */ + leaveblock(fs); /* finish scope */ + luaK_patchlist(ls->fs, condexit, repeat_init); /* close the loop */ + } + else { /* complete semantics when there are upvalues */ + breakstat(ls); /* if condition then break */ + luaK_patchtohere(ls->fs, condexit); /* else... */ + leaveblock(fs); /* finish scope... */ + luaK_patchlist(ls->fs, luaK_jump(fs), repeat_init); /* and repeat */ + } + leaveblock(fs); /* finish loop */ +} + + +static int exp1 (LexState *ls) { + expdesc e; + int k; + expr(ls, &e); + k = e.k; + luaK_exp2nextreg(ls->fs, &e); + return k; +} + + +static void forbody (LexState *ls, int base, int line, int nvars, int isnum) { + /* forbody -> DO block */ + BlockCnt bl; + FuncState *fs = ls->fs; + int prep, endfor; + adjustlocalvars(ls, 3); /* control variables */ + checknext(ls, TK_DO); + prep = isnum ? luaK_codeAsBx(fs, OP_FORPREP, base, NO_JUMP) : luaK_jump(fs); + enterblock(fs, &bl, 0); /* scope for declared variables */ + adjustlocalvars(ls, nvars); + luaK_reserveregs(fs, nvars); + block(ls); + leaveblock(fs); /* end of scope for declared variables */ + luaK_patchtohere(fs, prep); + endfor = (isnum) ? luaK_codeAsBx(fs, OP_FORLOOP, base, NO_JUMP) : + luaK_codeABC(fs, OP_TFORLOOP, base, 0, nvars); + luaK_fixline(fs, line); /* pretend that `OP_FOR' starts the loop */ + luaK_patchlist(fs, (isnum ? endfor : luaK_jump(fs)), prep + 1); +} + + +static void fornum (LexState *ls, TString *varname, int line) { + /* fornum -> NAME = exp1,exp1[,exp1] forbody */ + FuncState *fs = ls->fs; + int base = fs->freereg; + new_localvarliteral(ls, "(for index)", 0); + new_localvarliteral(ls, "(for limit)", 1); + new_localvarliteral(ls, "(for step)", 2); + new_localvar(ls, varname, 3); + checknext(ls, '='); + exp1(ls); /* initial value */ + checknext(ls, ','); + exp1(ls); /* limit */ + if (testnext(ls, ',')) + exp1(ls); /* optional step */ + else { /* default step = 1 */ + luaK_codeABx(fs, OP_LOADK, fs->freereg, luaK_numberK(fs, 1)); + luaK_reserveregs(fs, 1); + } + forbody(ls, base, line, 1, 1); +} + + +static void forlist (LexState *ls, TString *indexname) { + /* forlist -> NAME {,NAME} IN explist1 forbody */ + FuncState *fs = ls->fs; + expdesc e; + int nvars = 0; + int line; + int base = fs->freereg; + /* create control variables */ + new_localvarliteral(ls, "(for generator)", nvars++); + new_localvarliteral(ls, "(for state)", nvars++); + new_localvarliteral(ls, "(for control)", nvars++); + /* create declared variables */ + new_localvar(ls, indexname, nvars++); + while (testnext(ls, ',')) + new_localvar(ls, str_checkname(ls), nvars++); + checknext(ls, TK_IN); + line = ls->linenumber; + adjust_assign(ls, 3, explist1(ls, &e), &e); + luaK_checkstack(fs, 3); /* extra space to call generator */ + forbody(ls, base, line, nvars - 3, 0); +} + + +static void forstat (LexState *ls, int line) { + /* forstat -> FOR (fornum | forlist) END */ + FuncState *fs = ls->fs; + TString *varname; + BlockCnt bl; + enterblock(fs, &bl, 1); /* scope for loop and control variables */ + luaX_next(ls); /* skip `for' */ + varname = str_checkname(ls); /* first variable name */ + switch (ls->t.token) { + case '=': fornum(ls, varname, line); break; + case ',': case TK_IN: forlist(ls, varname); break; + default: luaX_syntaxerror(ls, LUA_QL("=") " or " LUA_QL("in") " expected"); + } + check_match(ls, TK_END, TK_FOR, line); + leaveblock(fs); /* loop scope (`break' jumps to this point) */ +} + + +static int test_then_block (LexState *ls) { + /* test_then_block -> [IF | ELSEIF] cond THEN block */ + int condexit; + luaX_next(ls); /* skip IF or ELSEIF */ + condexit = cond(ls); + checknext(ls, TK_THEN); + block(ls); /* `then' part */ + return condexit; +} + + +static void ifstat (LexState *ls, int line) { + /* ifstat -> IF cond THEN block {ELSEIF cond THEN block} [ELSE block] END */ + FuncState *fs = ls->fs; + int flist; + int escapelist = NO_JUMP; + flist = test_then_block(ls); /* IF cond THEN block */ + while (ls->t.token == TK_ELSEIF) { + luaK_concat(fs, &escapelist, luaK_jump(fs)); + luaK_patchtohere(fs, flist); + flist = test_then_block(ls); /* ELSEIF cond THEN block */ + } + if (ls->t.token == TK_ELSE) { + luaK_concat(fs, &escapelist, luaK_jump(fs)); + luaK_patchtohere(fs, flist); + luaX_next(ls); /* skip ELSE (after patch, for correct line info) */ + block(ls); /* `else' part */ + } + else + luaK_concat(fs, &escapelist, flist); + luaK_patchtohere(fs, escapelist); + check_match(ls, TK_END, TK_IF, line); +} + + +static void localfunc (LexState *ls) { + expdesc v, b; + FuncState *fs = ls->fs; + new_localvar(ls, str_checkname(ls), 0); + init_exp(&v, VLOCAL, fs->freereg); + luaK_reserveregs(fs, 1); + adjustlocalvars(ls, 1); + body(ls, &b, 0, ls->linenumber); + luaK_storevar(fs, &v, &b); + /* debug information will only see the variable after this point! */ + getlocvar(fs, fs->nactvar - 1).startpc = fs->pc; +} + + +static void localstat (LexState *ls) { + /* stat -> LOCAL NAME {`,' NAME} [`=' explist1] */ + int nvars = 0; + int nexps; + expdesc e; + do { + new_localvar(ls, str_checkname(ls), nvars++); + } while (testnext(ls, ',')); + if (testnext(ls, '=')) + nexps = explist1(ls, &e); + else { + e.k = VVOID; + nexps = 0; + } + adjust_assign(ls, nvars, nexps, &e); + adjustlocalvars(ls, nvars); +} + + +static int funcname (LexState *ls, expdesc *v) { + /* funcname -> NAME {field} [`:' NAME] */ + int needself = 0; + singlevar(ls, v); + while (ls->t.token == '.') + field(ls, v); + if (ls->t.token == ':') { + needself = 1; + field(ls, v); + } + return needself; +} + + +static void funcstat (LexState *ls, int line) { + /* funcstat -> FUNCTION funcname body */ + int needself; + expdesc v, b; + luaX_next(ls); /* skip FUNCTION */ + needself = funcname(ls, &v); + body(ls, &b, needself, line); + luaK_storevar(ls->fs, &v, &b); + luaK_fixline(ls->fs, line); /* definition `happens' in the first line */ +} + + +static void exprstat (LexState *ls) { + /* stat -> func | assignment */ + FuncState *fs = ls->fs; + struct LHS_assign v; + primaryexp(ls, &v.v); + if (v.v.k == VCALL) /* stat -> func */ + SETARG_C(getcode(fs, &v.v), 1); /* call statement uses no results */ + else { /* stat -> assignment */ + v.prev = NULL; + assignment(ls, &v, 1); + } +} + + +static void retstat (LexState *ls) { + /* stat -> RETURN explist */ + FuncState *fs = ls->fs; + expdesc e; + int first, nret; /* registers with returned values */ + luaX_next(ls); /* skip RETURN */ + if (block_follow(ls->t.token) || ls->t.token == ';') + first = nret = 0; /* return no values */ + else { + nret = explist1(ls, &e); /* optional return values */ + if (hasmultret(e.k)) { + luaK_setmultret(fs, &e); + if (e.k == VCALL && nret == 1) { /* tail call? */ + SET_OPCODE(getcode(fs,&e), OP_TAILCALL); + lua_assert(GETARG_A(getcode(fs,&e)) == fs->nactvar); + } + first = fs->nactvar; + nret = LUA_MULTRET; /* return all values */ + } + else { + if (nret == 1) /* only one single value? */ + first = luaK_exp2anyreg(fs, &e); + else { + luaK_exp2nextreg(fs, &e); /* values must go to the `stack' */ + first = fs->nactvar; /* return all `active' values */ + lua_assert(nret == fs->freereg - first); + } + } + } + luaK_ret(fs, first, nret); +} + + +static int statement (LexState *ls) { + int line = ls->linenumber; /* may be needed for error messages */ + switch (ls->t.token) { + case TK_IF: { /* stat -> ifstat */ + ifstat(ls, line); + return 0; + } + case TK_WHILE: { /* stat -> whilestat */ + whilestat(ls, line); + return 0; + } + case TK_DO: { /* stat -> DO block END */ + luaX_next(ls); /* skip DO */ + block(ls); + check_match(ls, TK_END, TK_DO, line); + return 0; + } + case TK_FOR: { /* stat -> forstat */ + forstat(ls, line); + return 0; + } + case TK_REPEAT: { /* stat -> repeatstat */ + repeatstat(ls, line); + return 0; + } + case TK_FUNCTION: { + funcstat(ls, line); /* stat -> funcstat */ + return 0; + } + case TK_LOCAL: { /* stat -> localstat */ + luaX_next(ls); /* skip LOCAL */ + if (testnext(ls, TK_FUNCTION)) /* local function? */ + localfunc(ls); + else + localstat(ls); + return 0; + } + case TK_RETURN: { /* stat -> retstat */ + retstat(ls); + return 1; /* must be last statement */ + } + case TK_BREAK: { /* stat -> breakstat */ + luaX_next(ls); /* skip BREAK */ + breakstat(ls); + return 1; /* must be last statement */ + } + default: { + exprstat(ls); + return 0; /* to avoid warnings */ + } + } +} + + +static void chunk (LexState *ls) { + /* chunk -> { stat [`;'] } */ + int islast = 0; + enterlevel(ls); + while (!islast && !block_follow(ls->t.token)) { + islast = statement(ls); + testnext(ls, ';'); + lua_assert(ls->fs->f->maxstacksize >= ls->fs->freereg && + ls->fs->freereg >= ls->fs->nactvar); + ls->fs->freereg = ls->fs->nactvar; /* free registers */ + } + leavelevel(ls); +} + +/* }====================================================================== */ diff --git a/src/lua/lparser.h b/src/lua/lparser.h new file mode 100644 index 0000000..18836af --- /dev/null +++ b/src/lua/lparser.h @@ -0,0 +1,82 @@ +/* +** $Id: lparser.h,v 1.57.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua Parser +** See Copyright Notice in lua.h +*/ + +#ifndef lparser_h +#define lparser_h + +#include "llimits.h" +#include "lobject.h" +#include "lzio.h" + + +/* +** Expression descriptor +*/ + +typedef enum { + VVOID, /* no value */ + VNIL, + VTRUE, + VFALSE, + VK, /* info = index of constant in `k' */ + VKNUM, /* nval = numerical value */ + VLOCAL, /* info = local register */ + VUPVAL, /* info = index of upvalue in `upvalues' */ + VGLOBAL, /* info = index of table; aux = index of global name in `k' */ + VINDEXED, /* info = table register; aux = index register (or `k') */ + VJMP, /* info = instruction pc */ + VRELOCABLE, /* info = instruction pc */ + VNONRELOC, /* info = result register */ + VCALL, /* info = instruction pc */ + VVARARG /* info = instruction pc */ +} expkind; + +typedef struct expdesc { + expkind k; + union { + struct { int info, aux; } s; + lua_Number nval; + } u; + int t; /* patch list of `exit when true' */ + int f; /* patch list of `exit when false' */ +} expdesc; + + +typedef struct upvaldesc { + lu_byte k; + lu_byte info; +} upvaldesc; + + +struct BlockCnt; /* defined in lparser.c */ + + +/* state needed to generate code for a given function */ +typedef struct FuncState { + Proto *f; /* current function header */ + Table *h; /* table to find (and reuse) elements in `k' */ + struct FuncState *prev; /* enclosing function */ + struct LexState *ls; /* lexical state */ + struct lua_State *L; /* copy of the Lua state */ + struct BlockCnt *bl; /* chain of current blocks */ + int pc; /* next position to code (equivalent to `ncode') */ + int lasttarget; /* `pc' of last `jump target' */ + int jpc; /* list of pending jumps to `pc' */ + int freereg; /* first free register */ + int nk; /* number of elements in `k' */ + int np; /* number of elements in `p' */ + short nlocvars; /* number of elements in `locvars' */ + lu_byte nactvar; /* number of active local variables */ + upvaldesc upvalues[LUAI_MAXUPVALUES]; /* upvalues */ + unsigned short actvar[LUAI_MAXVARS]; /* declared-variable stack */ +} FuncState; + + +LUAI_FUNC Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, + const char *name); + + +#endif diff --git a/src/lua/lstate.c b/src/lua/lstate.c new file mode 100644 index 0000000..4313b83 --- /dev/null +++ b/src/lua/lstate.c @@ -0,0 +1,214 @@ +/* +** $Id: lstate.c,v 2.36.1.2 2008/01/03 15:20:39 roberto Exp $ +** Global State +** See Copyright Notice in lua.h +*/ + + +#include + +#define lstate_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "llex.h" +#include "lmem.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" + + +#define state_size(x) (sizeof(x) + LUAI_EXTRASPACE) +#define fromstate(l) (cast(lu_byte *, (l)) - LUAI_EXTRASPACE) +#define tostate(l) (cast(lua_State *, cast(lu_byte *, l) + LUAI_EXTRASPACE)) + + +/* +** Main thread combines a thread state and the global state +*/ +typedef struct LG { + lua_State l; + global_State g; +} LG; + + + +static void stack_init (lua_State *L1, lua_State *L) { + /* initialize CallInfo array */ + L1->base_ci = luaM_newvector(L, BASIC_CI_SIZE, CallInfo); + L1->ci = L1->base_ci; + L1->size_ci = BASIC_CI_SIZE; + L1->end_ci = L1->base_ci + L1->size_ci - 1; + /* initialize stack array */ + L1->stack = luaM_newvector(L, BASIC_STACK_SIZE + EXTRA_STACK, TValue); + L1->stacksize = BASIC_STACK_SIZE + EXTRA_STACK; + L1->top = L1->stack; + L1->stack_last = L1->stack+(L1->stacksize - EXTRA_STACK)-1; + /* initialize first ci */ + L1->ci->func = L1->top; + setnilvalue(L1->top++); /* `function' entry for this `ci' */ + L1->base = L1->ci->base = L1->top; + L1->ci->top = L1->top + LUA_MINSTACK; +} + + +static void freestack (lua_State *L, lua_State *L1) { + luaM_freearray(L, L1->base_ci, L1->size_ci, CallInfo); + luaM_freearray(L, L1->stack, L1->stacksize, TValue); +} + + +/* +** open parts that may cause memory-allocation errors +*/ +static void f_luaopen (lua_State *L, void *ud) { + global_State *g = G(L); + UNUSED(ud); + stack_init(L, L); /* init stack */ + sethvalue(L, gt(L), luaH_new(L, 0, 2)); /* table of globals */ + sethvalue(L, registry(L), luaH_new(L, 0, 2)); /* registry */ + luaS_resize(L, MINSTRTABSIZE); /* initial size of string table */ + luaT_init(L); + luaX_init(L); + luaS_fix(luaS_newliteral(L, MEMERRMSG)); + g->GCthreshold = 4*g->totalbytes; +} + + +static void preinit_state (lua_State *L, global_State *g) { + G(L) = g; + L->stack = NULL; + L->stacksize = 0; + L->errorJmp = NULL; + L->hook = NULL; + L->hookmask = 0; + L->basehookcount = 0; + L->allowhook = 1; + resethookcount(L); + L->openupval = NULL; + L->size_ci = 0; + L->nCcalls = L->baseCcalls = 0; + L->status = 0; + L->base_ci = L->ci = NULL; + L->savedpc = NULL; + L->errfunc = 0; + setnilvalue(gt(L)); +} + + +static void close_state (lua_State *L) { + global_State *g = G(L); + luaF_close(L, L->stack); /* close all upvalues for this thread */ + luaC_freeall(L); /* collect all objects */ + lua_assert(g->rootgc == obj2gco(L)); + lua_assert(g->strt.nuse == 0); + luaM_freearray(L, G(L)->strt.hash, G(L)->strt.size, TString *); + luaZ_freebuffer(L, &g->buff); + freestack(L, L); + lua_assert(g->totalbytes == sizeof(LG)); + (*g->frealloc)(g->ud, fromstate(L), state_size(LG), 0); +} + + +lua_State *luaE_newthread (lua_State *L) { + lua_State *L1 = tostate(luaM_malloc(L, state_size(lua_State))); + luaC_link(L, obj2gco(L1), LUA_TTHREAD); + preinit_state(L1, G(L)); + stack_init(L1, L); /* init stack */ + setobj2n(L, gt(L1), gt(L)); /* share table of globals */ + L1->hookmask = L->hookmask; + L1->basehookcount = L->basehookcount; + L1->hook = L->hook; + resethookcount(L1); + lua_assert(iswhite(obj2gco(L1))); + return L1; +} + + +void luaE_freethread (lua_State *L, lua_State *L1) { + luaF_close(L1, L1->stack); /* close all upvalues for this thread */ + lua_assert(L1->openupval == NULL); + luai_userstatefree(L1); + freestack(L, L1); + luaM_freemem(L, fromstate(L1), state_size(lua_State)); +} + + +LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { + int i; + lua_State *L; + global_State *g; + void *l = (*f)(ud, NULL, 0, state_size(LG)); + if (l == NULL) return NULL; + L = tostate(l); + g = &((LG *)L)->g; + L->next = NULL; + L->tt = LUA_TTHREAD; + g->currentwhite = bit2mask(WHITE0BIT, FIXEDBIT); + L->marked = luaC_white(g); + set2bits(L->marked, FIXEDBIT, SFIXEDBIT); + preinit_state(L, g); + g->frealloc = f; + g->ud = ud; + g->mainthread = L; + g->uvhead.u.l.prev = &g->uvhead; + g->uvhead.u.l.next = &g->uvhead; + g->GCthreshold = 0; /* mark it as unfinished state */ + g->strt.size = 0; + g->strt.nuse = 0; + g->strt.hash = NULL; + setnilvalue(registry(L)); + luaZ_initbuffer(L, &g->buff); + g->panic = NULL; + g->gcstate = GCSpause; + g->rootgc = obj2gco(L); + g->sweepstrgc = 0; + g->sweepgc = &g->rootgc; + g->gray = NULL; + g->grayagain = NULL; + g->weak = NULL; + g->tmudata = NULL; + g->totalbytes = sizeof(LG); + g->gcpause = LUAI_GCPAUSE; + g->gcstepmul = LUAI_GCMUL; + g->gcdept = 0; + for (i=0; imt[i] = NULL; + if (luaD_rawrunprotected(L, f_luaopen, NULL) != 0) { + /* memory allocation error: free partial state */ + close_state(L); + L = NULL; + } + else + luai_userstateopen(L); + return L; +} + + +static void callallgcTM (lua_State *L, void *ud) { + UNUSED(ud); + luaC_callGCTM(L); /* call GC metamethods for all udata */ +} + + +LUA_API void lua_close (lua_State *L) { + L = G(L)->mainthread; /* only the main thread can be closed */ + lua_lock(L); + luaF_close(L, L->stack); /* close all upvalues for this thread */ + luaC_separateudata(L, 1); /* separate udata that have GC metamethods */ + L->errfunc = 0; /* no error function during GC metamethods */ + do { /* repeat until no more errors */ + L->ci = L->base_ci; + L->base = L->top = L->ci->base; + L->nCcalls = L->baseCcalls = 0; + } while (luaD_rawrunprotected(L, callallgcTM, NULL) != 0); + lua_assert(G(L)->tmudata == NULL); + luai_userstateclose(L); + close_state(L); +} + diff --git a/src/lua/lstate.h b/src/lua/lstate.h new file mode 100644 index 0000000..3bc575b --- /dev/null +++ b/src/lua/lstate.h @@ -0,0 +1,169 @@ +/* +** $Id: lstate.h,v 2.24.1.2 2008/01/03 15:20:39 roberto Exp $ +** Global State +** See Copyright Notice in lua.h +*/ + +#ifndef lstate_h +#define lstate_h + +#include "lua.h" + +#include "lobject.h" +#include "ltm.h" +#include "lzio.h" + + + +struct lua_longjmp; /* defined in ldo.c */ + + +/* table of globals */ +#define gt(L) (&L->l_gt) + +/* registry */ +#define registry(L) (&G(L)->l_registry) + + +/* extra stack space to handle TM calls and some other extras */ +#define EXTRA_STACK 5 + + +#define BASIC_CI_SIZE 8 + +#define BASIC_STACK_SIZE (2*LUA_MINSTACK) + + + +typedef struct stringtable { + GCObject **hash; + lu_int32 nuse; /* number of elements */ + int size; +} stringtable; + + +/* +** informations about a call +*/ +typedef struct CallInfo { + StkId base; /* base for this function */ + StkId func; /* function index in the stack */ + StkId top; /* top for this function */ + const Instruction *savedpc; + int nresults; /* expected number of results from this function */ + int tailcalls; /* number of tail calls lost under this entry */ +} CallInfo; + + + +#define curr_func(L) (clvalue(L->ci->func)) +#define ci_func(ci) (clvalue((ci)->func)) +#define f_isLua(ci) (!ci_func(ci)->c.isC) +#define isLua(ci) (ttisfunction((ci)->func) && f_isLua(ci)) + + +/* +** `global state', shared by all threads of this state +*/ +typedef struct global_State { + stringtable strt; /* hash table for strings */ + lua_Alloc frealloc; /* function to reallocate memory */ + void *ud; /* auxiliary data to `frealloc' */ + lu_byte currentwhite; + lu_byte gcstate; /* state of garbage collector */ + int sweepstrgc; /* position of sweep in `strt' */ + GCObject *rootgc; /* list of all collectable objects */ + GCObject **sweepgc; /* position of sweep in `rootgc' */ + GCObject *gray; /* list of gray objects */ + GCObject *grayagain; /* list of objects to be traversed atomically */ + GCObject *weak; /* list of weak tables (to be cleared) */ + GCObject *tmudata; /* last element of list of userdata to be GC */ + Mbuffer buff; /* temporary buffer for string concatentation */ + lu_mem GCthreshold; + lu_mem totalbytes; /* number of bytes currently allocated */ + lu_mem estimate; /* an estimate of number of bytes actually in use */ + lu_mem gcdept; /* how much GC is `behind schedule' */ + int gcpause; /* size of pause between successive GCs */ + int gcstepmul; /* GC `granularity' */ + lua_CFunction panic; /* to be called in unprotected errors */ + TValue l_registry; + struct lua_State *mainthread; + UpVal uvhead; /* head of double-linked list of all open upvalues */ + struct Table *mt[NUM_TAGS]; /* metatables for basic types */ + TString *tmname[TM_N]; /* array with tag-method names */ +} global_State; + + +/* +** `per thread' state +*/ +struct lua_State { + CommonHeader; + lu_byte status; + StkId top; /* first free slot in the stack */ + StkId base; /* base of current function */ + global_State *l_G; + CallInfo *ci; /* call info for current function */ + const Instruction *savedpc; /* `savedpc' of current function */ + StkId stack_last; /* last free slot in the stack */ + StkId stack; /* stack base */ + CallInfo *end_ci; /* points after end of ci array*/ + CallInfo *base_ci; /* array of CallInfo's */ + int stacksize; + int size_ci; /* size of array `base_ci' */ + unsigned short nCcalls; /* number of nested C calls */ + unsigned short baseCcalls; /* nested C calls when resuming coroutine */ + lu_byte hookmask; + lu_byte allowhook; + int basehookcount; + int hookcount; + lua_Hook hook; + TValue l_gt; /* table of globals */ + TValue env; /* temporary place for environments */ + GCObject *openupval; /* list of open upvalues in this stack */ + GCObject *gclist; + struct lua_longjmp *errorJmp; /* current error recover point */ + ptrdiff_t errfunc; /* current error handling function (stack index) */ +}; + + +#define G(L) (L->l_G) + + +/* +** Union of all collectable objects +*/ +union GCObject { + GCheader gch; + union TString ts; + union Udata u; + union Closure cl; + struct Table h; + struct Proto p; + struct UpVal uv; + struct lua_State th; /* thread */ +}; + + +/* macros to convert a GCObject into a specific value */ +#define rawgco2ts(o) check_exp((o)->gch.tt == LUA_TSTRING, &((o)->ts)) +#define gco2ts(o) (&rawgco2ts(o)->tsv) +#define rawgco2u(o) check_exp((o)->gch.tt == LUA_TUSERDATA, &((o)->u)) +#define gco2u(o) (&rawgco2u(o)->uv) +#define gco2cl(o) check_exp((o)->gch.tt == LUA_TFUNCTION, &((o)->cl)) +#define gco2h(o) check_exp((o)->gch.tt == LUA_TTABLE, &((o)->h)) +#define gco2p(o) check_exp((o)->gch.tt == LUA_TPROTO, &((o)->p)) +#define gco2uv(o) check_exp((o)->gch.tt == LUA_TUPVAL, &((o)->uv)) +#define ngcotouv(o) \ + check_exp((o) == NULL || (o)->gch.tt == LUA_TUPVAL, &((o)->uv)) +#define gco2th(o) check_exp((o)->gch.tt == LUA_TTHREAD, &((o)->th)) + +/* macro to convert any Lua object into a GCObject */ +#define obj2gco(v) (cast(GCObject *, (v))) + + +LUAI_FUNC lua_State *luaE_newthread (lua_State *L); +LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1); + +#endif + diff --git a/src/lua/lstring.c b/src/lua/lstring.c new file mode 100644 index 0000000..4911315 --- /dev/null +++ b/src/lua/lstring.c @@ -0,0 +1,111 @@ +/* +** $Id: lstring.c,v 2.8.1.1 2007/12/27 13:02:25 roberto Exp $ +** String table (keeps all strings handled by Lua) +** See Copyright Notice in lua.h +*/ + + +#include + +#define lstring_c +#define LUA_CORE + +#include "lua.h" + +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" + + + +void luaS_resize (lua_State *L, int newsize) { + GCObject **newhash; + stringtable *tb; + int i; + if (G(L)->gcstate == GCSsweepstring) + return; /* cannot resize during GC traverse */ + newhash = luaM_newvector(L, newsize, GCObject *); + tb = &G(L)->strt; + for (i=0; isize; i++) { + GCObject *p = tb->hash[i]; + while (p) { /* for each node in the list */ + GCObject *next = p->gch.next; /* save next */ + unsigned int h = gco2ts(p)->hash; + int h1 = lmod(h, newsize); /* new position */ + lua_assert(cast_int(h%newsize) == lmod(h, newsize)); + p->gch.next = newhash[h1]; /* chain it */ + newhash[h1] = p; + p = next; + } + } + luaM_freearray(L, tb->hash, tb->size, TString *); + tb->size = newsize; + tb->hash = newhash; +} + + +static TString *newlstr (lua_State *L, const char *str, size_t l, + unsigned int h) { + TString *ts; + stringtable *tb; + if (l+1 > (MAX_SIZET - sizeof(TString))/sizeof(char)) + luaM_toobig(L); + ts = cast(TString *, luaM_malloc(L, (l+1)*sizeof(char)+sizeof(TString))); + ts->tsv.len = l; + ts->tsv.hash = h; + ts->tsv.marked = luaC_white(G(L)); + ts->tsv.tt = LUA_TSTRING; + ts->tsv.reserved = 0; + memcpy(ts+1, str, l*sizeof(char)); + ((char *)(ts+1))[l] = '\0'; /* ending 0 */ + tb = &G(L)->strt; + h = lmod(h, tb->size); + ts->tsv.next = tb->hash[h]; /* chain new entry */ + tb->hash[h] = obj2gco(ts); + tb->nuse++; + if (tb->nuse > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2) + luaS_resize(L, tb->size*2); /* too crowded */ + return ts; +} + + +TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { + GCObject *o; + unsigned int h = cast(unsigned int, l); /* seed */ + size_t step = (l>>5)+1; /* if string is too long, don't hash all its chars */ + size_t l1; + for (l1=l; l1>=step; l1-=step) /* compute hash */ + h = h ^ ((h<<5)+(h>>2)+cast(unsigned char, str[l1-1])); + for (o = G(L)->strt.hash[lmod(h, G(L)->strt.size)]; + o != NULL; + o = o->gch.next) { + TString *ts = rawgco2ts(o); + if (ts->tsv.len == l && (memcmp(str, getstr(ts), l) == 0)) { + /* string may be dead */ + if (isdead(G(L), o)) changewhite(o); + return ts; + } + } + return newlstr(L, str, l, h); /* not found */ +} + + +Udata *luaS_newudata (lua_State *L, size_t s, Table *e) { + Udata *u; + if (s > MAX_SIZET - sizeof(Udata)) + luaM_toobig(L); + u = cast(Udata *, luaM_malloc(L, s + sizeof(Udata))); + u->uv.marked = luaC_white(G(L)); /* is not finalized */ + u->uv.tt = LUA_TUSERDATA; + u->uv.len = s; + u->uv.metatable = NULL; + u->uv.env = e; + /* chain it on udata list (after main thread) */ + u->uv.next = G(L)->mainthread->next; + G(L)->mainthread->next = obj2gco(u); + return u; +} + diff --git a/src/lua/lstring.h b/src/lua/lstring.h new file mode 100644 index 0000000..73a2ff8 --- /dev/null +++ b/src/lua/lstring.h @@ -0,0 +1,31 @@ +/* +** $Id: lstring.h,v 1.43.1.1 2007/12/27 13:02:25 roberto Exp $ +** String table (keep all strings handled by Lua) +** See Copyright Notice in lua.h +*/ + +#ifndef lstring_h +#define lstring_h + + +#include "lgc.h" +#include "lobject.h" +#include "lstate.h" + + +#define sizestring(s) (sizeof(union TString)+((s)->len+1)*sizeof(char)) + +#define sizeudata(u) (sizeof(union Udata)+(u)->len) + +#define luaS_new(L, s) (luaS_newlstr(L, s, strlen(s))) +#define luaS_newliteral(L, s) (luaS_newlstr(L, "" s, \ + (sizeof(s)/sizeof(char))-1)) + +#define luaS_fix(s) l_setbit((s)->tsv.marked, FIXEDBIT) + +LUAI_FUNC void luaS_resize (lua_State *L, int newsize); +LUAI_FUNC Udata *luaS_newudata (lua_State *L, size_t s, Table *e); +LUAI_FUNC TString *luaS_newlstr (lua_State *L, const char *str, size_t l); + + +#endif diff --git a/src/lua/lstrlib.c b/src/lua/lstrlib.c new file mode 100644 index 0000000..7a03489 --- /dev/null +++ b/src/lua/lstrlib.c @@ -0,0 +1,871 @@ +/* +** $Id: lstrlib.c,v 1.132.1.5 2010/05/14 15:34:19 roberto Exp $ +** Standard library for string operations and pattern-matching +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include +#include + +#define lstrlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +/* macro to `unsign' a character */ +#define uchar(c) ((unsigned char)(c)) + + + +static int str_len (lua_State *L) { + size_t l; + luaL_checklstring(L, 1, &l); + lua_pushinteger(L, l); + return 1; +} + + +static ptrdiff_t posrelat (ptrdiff_t pos, size_t len) { + /* relative string position: negative means back from end */ + if (pos < 0) pos += (ptrdiff_t)len + 1; + return (pos >= 0) ? pos : 0; +} + + +static int str_sub (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + ptrdiff_t start = posrelat(luaL_checkinteger(L, 2), l); + ptrdiff_t end = posrelat(luaL_optinteger(L, 3, -1), l); + if (start < 1) start = 1; + if (end > (ptrdiff_t)l) end = (ptrdiff_t)l; + if (start <= end) + lua_pushlstring(L, s+start-1, end-start+1); + else lua_pushliteral(L, ""); + return 1; +} + + +static int str_reverse (lua_State *L) { + size_t l; + luaL_Buffer b; + const char *s = luaL_checklstring(L, 1, &l); + luaL_buffinit(L, &b); + while (l--) luaL_addchar(&b, s[l]); + luaL_pushresult(&b); + return 1; +} + + +static int str_lower (lua_State *L) { + size_t l; + size_t i; + luaL_Buffer b; + const char *s = luaL_checklstring(L, 1, &l); + luaL_buffinit(L, &b); + for (i=0; i 0) + luaL_addlstring(&b, s, l); + luaL_pushresult(&b); + return 1; +} + + +static int str_byte (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + ptrdiff_t posi = posrelat(luaL_optinteger(L, 2, 1), l); + ptrdiff_t pose = posrelat(luaL_optinteger(L, 3, posi), l); + int n, i; + if (posi <= 0) posi = 1; + if ((size_t)pose > l) pose = l; + if (posi > pose) return 0; /* empty interval; return no values */ + n = (int)(pose - posi + 1); + if (posi + n <= pose) /* overflow? */ + luaL_error(L, "string slice too long"); + luaL_checkstack(L, n, "string slice too long"); + for (i=0; i= ms->level || ms->capture[l].len == CAP_UNFINISHED) + return luaL_error(ms->L, "invalid capture index"); + return l; +} + + +static int capture_to_close (MatchState *ms) { + int level = ms->level; + for (level--; level>=0; level--) + if (ms->capture[level].len == CAP_UNFINISHED) return level; + return luaL_error(ms->L, "invalid pattern capture"); +} + + +static const char *classend (MatchState *ms, const char *p) { + switch (*p++) { + case L_ESC: { + if (*p == '\0') + luaL_error(ms->L, "malformed pattern (ends with " LUA_QL("%%") ")"); + return p+1; + } + case '[': { + if (*p == '^') p++; + do { /* look for a `]' */ + if (*p == '\0') + luaL_error(ms->L, "malformed pattern (missing " LUA_QL("]") ")"); + if (*(p++) == L_ESC && *p != '\0') + p++; /* skip escapes (e.g. `%]') */ + } while (*p != ']'); + return p+1; + } + default: { + return p; + } + } +} + + +static int match_class (int c, int cl) { + int res; + switch (tolower(cl)) { + case 'a' : res = isalpha(c); break; + case 'c' : res = iscntrl(c); break; + case 'd' : res = isdigit(c); break; + case 'l' : res = islower(c); break; + case 'p' : res = ispunct(c); break; + case 's' : res = isspace(c); break; + case 'u' : res = isupper(c); break; + case 'w' : res = isalnum(c); break; + case 'x' : res = isxdigit(c); break; + case 'z' : res = (c == 0); break; + default: return (cl == c); + } + return (islower(cl) ? res : !res); +} + + +static int matchbracketclass (int c, const char *p, const char *ec) { + int sig = 1; + if (*(p+1) == '^') { + sig = 0; + p++; /* skip the `^' */ + } + while (++p < ec) { + if (*p == L_ESC) { + p++; + if (match_class(c, uchar(*p))) + return sig; + } + else if ((*(p+1) == '-') && (p+2 < ec)) { + p+=2; + if (uchar(*(p-2)) <= c && c <= uchar(*p)) + return sig; + } + else if (uchar(*p) == c) return sig; + } + return !sig; +} + + +static int singlematch (int c, const char *p, const char *ep) { + switch (*p) { + case '.': return 1; /* matches any char */ + case L_ESC: return match_class(c, uchar(*(p+1))); + case '[': return matchbracketclass(c, p, ep-1); + default: return (uchar(*p) == c); + } +} + + +static const char *match (MatchState *ms, const char *s, const char *p); + + +static const char *matchbalance (MatchState *ms, const char *s, + const char *p) { + if (*p == 0 || *(p+1) == 0) + luaL_error(ms->L, "unbalanced pattern"); + if (*s != *p) return NULL; + else { + int b = *p; + int e = *(p+1); + int cont = 1; + while (++s < ms->src_end) { + if (*s == e) { + if (--cont == 0) return s+1; + } + else if (*s == b) cont++; + } + } + return NULL; /* string ends out of balance */ +} + + +static const char *max_expand (MatchState *ms, const char *s, + const char *p, const char *ep) { + ptrdiff_t i = 0; /* counts maximum expand for item */ + while ((s+i)src_end && singlematch(uchar(*(s+i)), p, ep)) + i++; + /* keeps trying to match with the maximum repetitions */ + while (i>=0) { + const char *res = match(ms, (s+i), ep+1); + if (res) return res; + i--; /* else didn't match; reduce 1 repetition to try again */ + } + return NULL; +} + + +static const char *min_expand (MatchState *ms, const char *s, + const char *p, const char *ep) { + for (;;) { + const char *res = match(ms, s, ep+1); + if (res != NULL) + return res; + else if (ssrc_end && singlematch(uchar(*s), p, ep)) + s++; /* try with one more repetition */ + else return NULL; + } +} + + +static const char *start_capture (MatchState *ms, const char *s, + const char *p, int what) { + const char *res; + int level = ms->level; + if (level >= LUA_MAXCAPTURES) luaL_error(ms->L, "too many captures"); + ms->capture[level].init = s; + ms->capture[level].len = what; + ms->level = level+1; + if ((res=match(ms, s, p)) == NULL) /* match failed? */ + ms->level--; /* undo capture */ + return res; +} + + +static const char *end_capture (MatchState *ms, const char *s, + const char *p) { + int l = capture_to_close(ms); + const char *res; + ms->capture[l].len = s - ms->capture[l].init; /* close capture */ + if ((res = match(ms, s, p)) == NULL) /* match failed? */ + ms->capture[l].len = CAP_UNFINISHED; /* undo capture */ + return res; +} + + +static const char *match_capture (MatchState *ms, const char *s, int l) { + size_t len; + l = check_capture(ms, l); + len = ms->capture[l].len; + if ((size_t)(ms->src_end-s) >= len && + memcmp(ms->capture[l].init, s, len) == 0) + return s+len; + else return NULL; +} + + +static const char *match (MatchState *ms, const char *s, const char *p) { + init: /* using goto's to optimize tail recursion */ + switch (*p) { + case '(': { /* start capture */ + if (*(p+1) == ')') /* position capture? */ + return start_capture(ms, s, p+2, CAP_POSITION); + else + return start_capture(ms, s, p+1, CAP_UNFINISHED); + } + case ')': { /* end capture */ + return end_capture(ms, s, p+1); + } + case L_ESC: { + switch (*(p+1)) { + case 'b': { /* balanced string? */ + s = matchbalance(ms, s, p+2); + if (s == NULL) return NULL; + p+=4; goto init; /* else return match(ms, s, p+4); */ + } + case 'f': { /* frontier? */ + const char *ep; char previous; + p += 2; + if (*p != '[') + luaL_error(ms->L, "missing " LUA_QL("[") " after " + LUA_QL("%%f") " in pattern"); + ep = classend(ms, p); /* points to what is next */ + previous = (s == ms->src_init) ? '\0' : *(s-1); + if (matchbracketclass(uchar(previous), p, ep-1) || + !matchbracketclass(uchar(*s), p, ep-1)) return NULL; + p=ep; goto init; /* else return match(ms, s, ep); */ + } + default: { + if (isdigit(uchar(*(p+1)))) { /* capture results (%0-%9)? */ + s = match_capture(ms, s, uchar(*(p+1))); + if (s == NULL) return NULL; + p+=2; goto init; /* else return match(ms, s, p+2) */ + } + goto dflt; /* case default */ + } + } + } + case '\0': { /* end of pattern */ + return s; /* match succeeded */ + } + case '$': { + if (*(p+1) == '\0') /* is the `$' the last char in pattern? */ + return (s == ms->src_end) ? s : NULL; /* check end of string */ + else goto dflt; + } + default: dflt: { /* it is a pattern item */ + const char *ep = classend(ms, p); /* points to what is next */ + int m = ssrc_end && singlematch(uchar(*s), p, ep); + switch (*ep) { + case '?': { /* optional */ + const char *res; + if (m && ((res=match(ms, s+1, ep+1)) != NULL)) + return res; + p=ep+1; goto init; /* else return match(ms, s, ep+1); */ + } + case '*': { /* 0 or more repetitions */ + return max_expand(ms, s, p, ep); + } + case '+': { /* 1 or more repetitions */ + return (m ? max_expand(ms, s+1, p, ep) : NULL); + } + case '-': { /* 0 or more repetitions (minimum) */ + return min_expand(ms, s, p, ep); + } + default: { + if (!m) return NULL; + s++; p=ep; goto init; /* else return match(ms, s+1, ep); */ + } + } + } + } +} + + + +static const char *lmemfind (const char *s1, size_t l1, + const char *s2, size_t l2) { + if (l2 == 0) return s1; /* empty strings are everywhere */ + else if (l2 > l1) return NULL; /* avoids a negative `l1' */ + else { + const char *init; /* to search for a `*s2' inside `s1' */ + l2--; /* 1st char will be checked by `memchr' */ + l1 = l1-l2; /* `s2' cannot be found after that */ + while (l1 > 0 && (init = (const char *)memchr(s1, *s2, l1)) != NULL) { + init++; /* 1st char is already checked */ + if (memcmp(init, s2+1, l2) == 0) + return init-1; + else { /* correct `l1' and `s1' to try again */ + l1 -= init-s1; + s1 = init; + } + } + return NULL; /* not found */ + } +} + + +static void push_onecapture (MatchState *ms, int i, const char *s, + const char *e) { + if (i >= ms->level) { + if (i == 0) /* ms->level == 0, too */ + lua_pushlstring(ms->L, s, e - s); /* add whole match */ + else + luaL_error(ms->L, "invalid capture index"); + } + else { + ptrdiff_t l = ms->capture[i].len; + if (l == CAP_UNFINISHED) luaL_error(ms->L, "unfinished capture"); + if (l == CAP_POSITION) + lua_pushinteger(ms->L, ms->capture[i].init - ms->src_init + 1); + else + lua_pushlstring(ms->L, ms->capture[i].init, l); + } +} + + +static int push_captures (MatchState *ms, const char *s, const char *e) { + int i; + int nlevels = (ms->level == 0 && s) ? 1 : ms->level; + luaL_checkstack(ms->L, nlevels, "too many captures"); + for (i = 0; i < nlevels; i++) + push_onecapture(ms, i, s, e); + return nlevels; /* number of strings pushed */ +} + + +static int str_find_aux (lua_State *L, int find) { + size_t l1, l2; + const char *s = luaL_checklstring(L, 1, &l1); + const char *p = luaL_checklstring(L, 2, &l2); + ptrdiff_t init = posrelat(luaL_optinteger(L, 3, 1), l1) - 1; + if (init < 0) init = 0; + else if ((size_t)(init) > l1) init = (ptrdiff_t)l1; + if (find && (lua_toboolean(L, 4) || /* explicit request? */ + strpbrk(p, SPECIALS) == NULL)) { /* or no special characters? */ + /* do a plain search */ + const char *s2 = lmemfind(s+init, l1-init, p, l2); + if (s2) { + lua_pushinteger(L, s2-s+1); + lua_pushinteger(L, s2-s+l2); + return 2; + } + } + else { + MatchState ms; + int anchor = (*p == '^') ? (p++, 1) : 0; + const char *s1=s+init; + ms.L = L; + ms.src_init = s; + ms.src_end = s+l1; + do { + const char *res; + ms.level = 0; + if ((res=match(&ms, s1, p)) != NULL) { + if (find) { + lua_pushinteger(L, s1-s+1); /* start */ + lua_pushinteger(L, res-s); /* end */ + return push_captures(&ms, NULL, 0) + 2; + } + else + return push_captures(&ms, s1, res); + } + } while (s1++ < ms.src_end && !anchor); + } + lua_pushnil(L); /* not found */ + return 1; +} + + +static int str_find (lua_State *L) { + return str_find_aux(L, 1); +} + + +static int str_match (lua_State *L) { + return str_find_aux(L, 0); +} + + +static int gmatch_aux (lua_State *L) { + MatchState ms; + size_t ls; + const char *s = lua_tolstring(L, lua_upvalueindex(1), &ls); + const char *p = lua_tostring(L, lua_upvalueindex(2)); + const char *src; + ms.L = L; + ms.src_init = s; + ms.src_end = s+ls; + for (src = s + (size_t)lua_tointeger(L, lua_upvalueindex(3)); + src <= ms.src_end; + src++) { + const char *e; + ms.level = 0; + if ((e = match(&ms, src, p)) != NULL) { + lua_Integer newstart = e-s; + if (e == src) newstart++; /* empty match? go at least one position */ + lua_pushinteger(L, newstart); + lua_replace(L, lua_upvalueindex(3)); + return push_captures(&ms, src, e); + } + } + return 0; /* not found */ +} + + +static int gmatch (lua_State *L) { + luaL_checkstring(L, 1); + luaL_checkstring(L, 2); + lua_settop(L, 2); + lua_pushinteger(L, 0); + lua_pushcclosure(L, gmatch_aux, 3); + return 1; +} + + +static int gfind_nodef (lua_State *L) { + return luaL_error(L, LUA_QL("string.gfind") " was renamed to " + LUA_QL("string.gmatch")); +} + + +static void add_s (MatchState *ms, luaL_Buffer *b, const char *s, + const char *e) { + size_t l, i; + const char *news = lua_tolstring(ms->L, 3, &l); + for (i = 0; i < l; i++) { + if (news[i] != L_ESC) + luaL_addchar(b, news[i]); + else { + i++; /* skip ESC */ + if (!isdigit(uchar(news[i]))) + luaL_addchar(b, news[i]); + else if (news[i] == '0') + luaL_addlstring(b, s, e - s); + else { + push_onecapture(ms, news[i] - '1', s, e); + luaL_addvalue(b); /* add capture to accumulated result */ + } + } + } +} + + +static void add_value (MatchState *ms, luaL_Buffer *b, const char *s, + const char *e) { + lua_State *L = ms->L; + switch (lua_type(L, 3)) { + case LUA_TNUMBER: + case LUA_TSTRING: { + add_s(ms, b, s, e); + return; + } + case LUA_TFUNCTION: { + int n; + lua_pushvalue(L, 3); + n = push_captures(ms, s, e); + lua_call(L, n, 1); + break; + } + case LUA_TTABLE: { + push_onecapture(ms, 0, s, e); + lua_gettable(L, 3); + break; + } + } + if (!lua_toboolean(L, -1)) { /* nil or false? */ + lua_pop(L, 1); + lua_pushlstring(L, s, e - s); /* keep original text */ + } + else if (!lua_isstring(L, -1)) + luaL_error(L, "invalid replacement value (a %s)", luaL_typename(L, -1)); + luaL_addvalue(b); /* add result to accumulator */ +} + + +static int str_gsub (lua_State *L) { + size_t srcl; + const char *src = luaL_checklstring(L, 1, &srcl); + const char *p = luaL_checkstring(L, 2); + int tr = lua_type(L, 3); + int max_s = luaL_optint(L, 4, srcl+1); + int anchor = (*p == '^') ? (p++, 1) : 0; + int n = 0; + MatchState ms; + luaL_Buffer b; + luaL_argcheck(L, tr == LUA_TNUMBER || tr == LUA_TSTRING || + tr == LUA_TFUNCTION || tr == LUA_TTABLE, 3, + "string/function/table expected"); + luaL_buffinit(L, &b); + ms.L = L; + ms.src_init = src; + ms.src_end = src+srcl; + while (n < max_s) { + const char *e; + ms.level = 0; + e = match(&ms, src, p); + if (e) { + n++; + add_value(&ms, &b, src, e); + } + if (e && e>src) /* non empty match? */ + src = e; /* skip it */ + else if (src < ms.src_end) + luaL_addchar(&b, *src++); + else break; + if (anchor) break; + } + luaL_addlstring(&b, src, ms.src_end-src); + luaL_pushresult(&b); + lua_pushinteger(L, n); /* number of substitutions */ + return 2; +} + +/* }====================================================== */ + + +/* maximum size of each formatted item (> len(format('%99.99f', -1e308))) */ +#define MAX_ITEM 512 +/* valid flags in a format specification */ +#define FLAGS "-+ #0" +/* +** maximum size of each format specification (such as '%-099.99d') +** (+10 accounts for %99.99x plus margin of error) +*/ +#define MAX_FORMAT (sizeof(FLAGS) + sizeof(LUA_INTFRMLEN) + 10) + + +static void addquoted (lua_State *L, luaL_Buffer *b, int arg) { + size_t l; + const char *s = luaL_checklstring(L, arg, &l); + luaL_addchar(b, '"'); + while (l--) { + switch (*s) { + case '"': case '\\': case '\n': { + luaL_addchar(b, '\\'); + luaL_addchar(b, *s); + break; + } + case '\r': { + luaL_addlstring(b, "\\r", 2); + break; + } + case '\0': { + luaL_addlstring(b, "\\000", 4); + break; + } + default: { + luaL_addchar(b, *s); + break; + } + } + s++; + } + luaL_addchar(b, '"'); +} + +static const char *scanformat (lua_State *L, const char *strfrmt, char *form) { + const char *p = strfrmt; + while (*p != '\0' && strchr(FLAGS, *p) != NULL) p++; /* skip flags */ + if ((size_t)(p - strfrmt) >= sizeof(FLAGS)) + luaL_error(L, "invalid format (repeated flags)"); + if (isdigit(uchar(*p))) p++; /* skip width */ + if (isdigit(uchar(*p))) p++; /* (2 digits at most) */ + if (*p == '.') { + p++; + if (isdigit(uchar(*p))) p++; /* skip precision */ + if (isdigit(uchar(*p))) p++; /* (2 digits at most) */ + } + if (isdigit(uchar(*p))) + luaL_error(L, "invalid format (width or precision too long)"); + *(form++) = '%'; + strncpy(form, strfrmt, p - strfrmt + 1); + form += p - strfrmt + 1; + *form = '\0'; + return p; +} + + +static void addintlen (char *form) { + size_t l = strlen(form); + char spec = form[l - 1]; + strcpy(form + l - 1, LUA_INTFRMLEN); + form[l + sizeof(LUA_INTFRMLEN) - 2] = spec; + form[l + sizeof(LUA_INTFRMLEN) - 1] = '\0'; +} + + +static int str_format (lua_State *L) { + int top = lua_gettop(L); + int arg = 1; + size_t sfl; + const char *strfrmt = luaL_checklstring(L, arg, &sfl); + const char *strfrmt_end = strfrmt+sfl; + luaL_Buffer b; + luaL_buffinit(L, &b); + while (strfrmt < strfrmt_end) { + if (*strfrmt != L_ESC) + luaL_addchar(&b, *strfrmt++); + else if (*++strfrmt == L_ESC) + luaL_addchar(&b, *strfrmt++); /* %% */ + else { /* format item */ + char form[MAX_FORMAT]; /* to store the format (`%...') */ + char buff[MAX_ITEM]; /* to store the formatted item */ + if (++arg > top) + luaL_argerror(L, arg, "no value"); + strfrmt = scanformat(L, strfrmt, form); + switch (*strfrmt++) { + case 'c': { + sprintf(buff, form, (int)luaL_checknumber(L, arg)); + break; + } + case 'd': case 'i': { + addintlen(form); + sprintf(buff, form, (LUA_INTFRM_T)luaL_checknumber(L, arg)); + break; + } + case 'o': case 'u': case 'x': case 'X': { + addintlen(form); + sprintf(buff, form, (unsigned LUA_INTFRM_T)luaL_checknumber(L, arg)); + break; + } + case 'e': case 'E': case 'f': + case 'g': case 'G': { + sprintf(buff, form, (double)luaL_checknumber(L, arg)); + break; + } + case 'q': { + addquoted(L, &b, arg); + continue; /* skip the 'addsize' at the end */ + } + case 's': { + size_t l; + const char *s = luaL_checklstring(L, arg, &l); + if (!strchr(form, '.') && l >= 100) { + /* no precision and string is too long to be formatted; + keep original string */ + lua_pushvalue(L, arg); + luaL_addvalue(&b); + continue; /* skip the `addsize' at the end */ + } + else { + sprintf(buff, form, s); + break; + } + } + default: { /* also treat cases `pnLlh' */ + return luaL_error(L, "invalid option " LUA_QL("%%%c") " to " + LUA_QL("format"), *(strfrmt - 1)); + } + } + luaL_addlstring(&b, buff, strlen(buff)); + } + } + luaL_pushresult(&b); + return 1; +} + + +static const luaL_Reg strlib[] = { + {"byte", str_byte}, + {"char", str_char}, + {"dump", str_dump}, + {"find", str_find}, + {"format", str_format}, + {"gfind", gfind_nodef}, + {"gmatch", gmatch}, + {"gsub", str_gsub}, + {"len", str_len}, + {"lower", str_lower}, + {"match", str_match}, + {"rep", str_rep}, + {"reverse", str_reverse}, + {"sub", str_sub}, + {"upper", str_upper}, + {NULL, NULL} +}; + + +static void createmetatable (lua_State *L) { + lua_createtable(L, 0, 1); /* create metatable for strings */ + lua_pushliteral(L, ""); /* dummy string */ + lua_pushvalue(L, -2); + lua_setmetatable(L, -2); /* set string metatable */ + lua_pop(L, 1); /* pop dummy string */ + lua_pushvalue(L, -2); /* string library... */ + lua_setfield(L, -2, "__index"); /* ...is the __index metamethod */ + lua_pop(L, 1); /* pop metatable */ +} + + +/* +** Open string library +*/ +LUALIB_API int luaopen_string (lua_State *L) { + luaL_register(L, LUA_STRLIBNAME, strlib); +#if defined(LUA_COMPAT_GFIND) + lua_getfield(L, -1, "gmatch"); + lua_setfield(L, -2, "gfind"); +#endif + createmetatable(L); + return 1; +} + diff --git a/src/lua/ltable.c b/src/lua/ltable.c new file mode 100644 index 0000000..ec84f4f --- /dev/null +++ b/src/lua/ltable.c @@ -0,0 +1,588 @@ +/* +** $Id: ltable.c,v 2.32.1.2 2007/12/28 15:32:23 roberto Exp $ +** Lua tables (hash) +** See Copyright Notice in lua.h +*/ + + +/* +** Implementation of tables (aka arrays, objects, or hash tables). +** Tables keep its elements in two parts: an array part and a hash part. +** Non-negative integer keys are all candidates to be kept in the array +** part. The actual size of the array is the largest `n' such that at +** least half the slots between 0 and n are in use. +** Hash uses a mix of chained scatter table with Brent's variation. +** A main invariant of these tables is that, if an element is not +** in its main position (i.e. the `original' position that its hash gives +** to it), then the colliding element is in its own main position. +** Hence even when the load factor reaches 100%, performance remains good. +*/ + +#include +#include + +#define ltable_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "ltable.h" + + +/* +** max size of array part is 2^MAXBITS +*/ +#if LUAI_BITSINT > 26 +#define MAXBITS 26 +#else +#define MAXBITS (LUAI_BITSINT-2) +#endif + +#define MAXASIZE (1 << MAXBITS) + + +#define hashpow2(t,n) (gnode(t, lmod((n), sizenode(t)))) + +#define hashstr(t,str) hashpow2(t, (str)->tsv.hash) +#define hashboolean(t,p) hashpow2(t, p) + + +/* +** for some types, it is better to avoid modulus by power of 2, as +** they tend to have many 2 factors. +*/ +#define hashmod(t,n) (gnode(t, ((n) % ((sizenode(t)-1)|1)))) + + +#define hashpointer(t,p) hashmod(t, IntPoint(p)) + + +/* +** number of ints inside a lua_Number +*/ +#define numints cast_int(sizeof(lua_Number)/sizeof(int)) + + + +#define dummynode (&dummynode_) + +static const Node dummynode_ = { + {{NULL}, LUA_TNIL}, /* value */ + {{{NULL}, LUA_TNIL, NULL}} /* key */ +}; + + +/* +** hash for lua_Numbers +*/ +static Node *hashnum (const Table *t, lua_Number n) { + unsigned int a[numints]; + int i; + if (luai_numeq(n, 0)) /* avoid problems with -0 */ + return gnode(t, 0); + memcpy(a, &n, sizeof(a)); + for (i = 1; i < numints; i++) a[0] += a[i]; + return hashmod(t, a[0]); +} + + + +/* +** returns the `main' position of an element in a table (that is, the index +** of its hash value) +*/ +static Node *mainposition (const Table *t, const TValue *key) { + switch (ttype(key)) { + case LUA_TNUMBER: + return hashnum(t, nvalue(key)); + case LUA_TSTRING: + return hashstr(t, rawtsvalue(key)); + case LUA_TBOOLEAN: + return hashboolean(t, bvalue(key)); + case LUA_TLIGHTUSERDATA: + return hashpointer(t, pvalue(key)); + default: + return hashpointer(t, gcvalue(key)); + } +} + + +/* +** returns the index for `key' if `key' is an appropriate key to live in +** the array part of the table, -1 otherwise. +*/ +static int arrayindex (const TValue *key) { + if (ttisnumber(key)) { + lua_Number n = nvalue(key); + int k; + lua_number2int(k, n); + if (luai_numeq(cast_num(k), n)) + return k; + } + return -1; /* `key' did not match some condition */ +} + + +/* +** returns the index of a `key' for table traversals. First goes all +** elements in the array part, then elements in the hash part. The +** beginning of a traversal is signalled by -1. +*/ +static int findindex (lua_State *L, Table *t, StkId key) { + int i; + if (ttisnil(key)) return -1; /* first iteration */ + i = arrayindex(key); + if (0 < i && i <= t->sizearray) /* is `key' inside array part? */ + return i-1; /* yes; that's the index (corrected to C) */ + else { + Node *n = mainposition(t, key); + do { /* check whether `key' is somewhere in the chain */ + /* key may be dead already, but it is ok to use it in `next' */ + if (luaO_rawequalObj(key2tval(n), key) || + (ttype(gkey(n)) == LUA_TDEADKEY && iscollectable(key) && + gcvalue(gkey(n)) == gcvalue(key))) { + i = cast_int(n - gnode(t, 0)); /* key index in hash table */ + /* hash elements are numbered after array ones */ + return i + t->sizearray; + } + else n = gnext(n); + } while (n); + luaG_runerror(L, "invalid key to " LUA_QL("next")); /* key not found */ + return 0; /* to avoid warnings */ + } +} + + +int luaH_next (lua_State *L, Table *t, StkId key) { + int i = findindex(L, t, key); /* find original element */ + for (i++; i < t->sizearray; i++) { /* try first array part */ + if (!ttisnil(&t->array[i])) { /* a non-nil value? */ + setnvalue(key, cast_num(i+1)); + setobj2s(L, key+1, &t->array[i]); + return 1; + } + } + for (i -= t->sizearray; i < sizenode(t); i++) { /* then hash part */ + if (!ttisnil(gval(gnode(t, i)))) { /* a non-nil value? */ + setobj2s(L, key, key2tval(gnode(t, i))); + setobj2s(L, key+1, gval(gnode(t, i))); + return 1; + } + } + return 0; /* no more elements */ +} + + +/* +** {============================================================= +** Rehash +** ============================================================== +*/ + + +static int computesizes (int nums[], int *narray) { + int i; + int twotoi; /* 2^i */ + int a = 0; /* number of elements smaller than 2^i */ + int na = 0; /* number of elements to go to array part */ + int n = 0; /* optimal size for array part */ + for (i = 0, twotoi = 1; twotoi/2 < *narray; i++, twotoi *= 2) { + if (nums[i] > 0) { + a += nums[i]; + if (a > twotoi/2) { /* more than half elements present? */ + n = twotoi; /* optimal size (till now) */ + na = a; /* all elements smaller than n will go to array part */ + } + } + if (a == *narray) break; /* all elements already counted */ + } + *narray = n; + lua_assert(*narray/2 <= na && na <= *narray); + return na; +} + + +static int countint (const TValue *key, int *nums) { + int k = arrayindex(key); + if (0 < k && k <= MAXASIZE) { /* is `key' an appropriate array index? */ + nums[ceillog2(k)]++; /* count as such */ + return 1; + } + else + return 0; +} + + +static int numusearray (const Table *t, int *nums) { + int lg; + int ttlg; /* 2^lg */ + int ause = 0; /* summation of `nums' */ + int i = 1; /* count to traverse all array keys */ + for (lg=0, ttlg=1; lg<=MAXBITS; lg++, ttlg*=2) { /* for each slice */ + int lc = 0; /* counter */ + int lim = ttlg; + if (lim > t->sizearray) { + lim = t->sizearray; /* adjust upper limit */ + if (i > lim) + break; /* no more elements to count */ + } + /* count elements in range (2^(lg-1), 2^lg] */ + for (; i <= lim; i++) { + if (!ttisnil(&t->array[i-1])) + lc++; + } + nums[lg] += lc; + ause += lc; + } + return ause; +} + + +static int numusehash (const Table *t, int *nums, int *pnasize) { + int totaluse = 0; /* total number of elements */ + int ause = 0; /* summation of `nums' */ + int i = sizenode(t); + while (i--) { + Node *n = &t->node[i]; + if (!ttisnil(gval(n))) { + ause += countint(key2tval(n), nums); + totaluse++; + } + } + *pnasize += ause; + return totaluse; +} + + +static void setarrayvector (lua_State *L, Table *t, int size) { + int i; + luaM_reallocvector(L, t->array, t->sizearray, size, TValue); + for (i=t->sizearray; iarray[i]); + t->sizearray = size; +} + + +static void setnodevector (lua_State *L, Table *t, int size) { + int lsize; + if (size == 0) { /* no elements to hash part? */ + t->node = cast(Node *, dummynode); /* use common `dummynode' */ + lsize = 0; + } + else { + int i; + lsize = ceillog2(size); + if (lsize > MAXBITS) + luaG_runerror(L, "table overflow"); + size = twoto(lsize); + t->node = luaM_newvector(L, size, Node); + for (i=0; ilsizenode = cast_byte(lsize); + t->lastfree = gnode(t, size); /* all positions are free */ +} + + +static void resize (lua_State *L, Table *t, int nasize, int nhsize) { + int i; + int oldasize = t->sizearray; + int oldhsize = t->lsizenode; + Node *nold = t->node; /* save old hash ... */ + if (nasize > oldasize) /* array part must grow? */ + setarrayvector(L, t, nasize); + /* create new hash part with appropriate size */ + setnodevector(L, t, nhsize); + if (nasize < oldasize) { /* array part must shrink? */ + t->sizearray = nasize; + /* re-insert elements from vanishing slice */ + for (i=nasize; iarray[i])) + setobjt2t(L, luaH_setnum(L, t, i+1), &t->array[i]); + } + /* shrink array */ + luaM_reallocvector(L, t->array, oldasize, nasize, TValue); + } + /* re-insert elements from hash part */ + for (i = twoto(oldhsize) - 1; i >= 0; i--) { + Node *old = nold+i; + if (!ttisnil(gval(old))) + setobjt2t(L, luaH_set(L, t, key2tval(old)), gval(old)); + } + if (nold != dummynode) + luaM_freearray(L, nold, twoto(oldhsize), Node); /* free old array */ +} + + +void luaH_resizearray (lua_State *L, Table *t, int nasize) { + int nsize = (t->node == dummynode) ? 0 : sizenode(t); + resize(L, t, nasize, nsize); +} + + +static void rehash (lua_State *L, Table *t, const TValue *ek) { + int nasize, na; + int nums[MAXBITS+1]; /* nums[i] = number of keys between 2^(i-1) and 2^i */ + int i; + int totaluse; + for (i=0; i<=MAXBITS; i++) nums[i] = 0; /* reset counts */ + nasize = numusearray(t, nums); /* count keys in array part */ + totaluse = nasize; /* all those keys are integer keys */ + totaluse += numusehash(t, nums, &nasize); /* count keys in hash part */ + /* count extra key */ + nasize += countint(ek, nums); + totaluse++; + /* compute new size for array part */ + na = computesizes(nums, &nasize); + /* resize the table to new computed sizes */ + resize(L, t, nasize, totaluse - na); +} + + + +/* +** }============================================================= +*/ + + +Table *luaH_new (lua_State *L, int narray, int nhash) { + Table *t = luaM_new(L, Table); + luaC_link(L, obj2gco(t), LUA_TTABLE); + t->metatable = NULL; + t->flags = cast_byte(~0); + /* temporary values (kept only if some malloc fails) */ + t->array = NULL; + t->sizearray = 0; + t->lsizenode = 0; + t->node = cast(Node *, dummynode); + setarrayvector(L, t, narray); + setnodevector(L, t, nhash); + return t; +} + + +void luaH_free (lua_State *L, Table *t) { + if (t->node != dummynode) + luaM_freearray(L, t->node, sizenode(t), Node); + luaM_freearray(L, t->array, t->sizearray, TValue); + luaM_free(L, t); +} + + +static Node *getfreepos (Table *t) { + while (t->lastfree-- > t->node) { + if (ttisnil(gkey(t->lastfree))) + return t->lastfree; + } + return NULL; /* could not find a free place */ +} + + + +/* +** inserts a new key into a hash table; first, check whether key's main +** position is free. If not, check whether colliding node is in its main +** position or not: if it is not, move colliding node to an empty place and +** put new key in its main position; otherwise (colliding node is in its main +** position), new key goes to an empty position. +*/ +static TValue *newkey (lua_State *L, Table *t, const TValue *key) { + Node *mp = mainposition(t, key); + if (!ttisnil(gval(mp)) || mp == dummynode) { + Node *othern; + Node *n = getfreepos(t); /* get a free place */ + if (n == NULL) { /* cannot find a free place? */ + rehash(L, t, key); /* grow table */ + return luaH_set(L, t, key); /* re-insert key into grown table */ + } + lua_assert(n != dummynode); + othern = mainposition(t, key2tval(mp)); + if (othern != mp) { /* is colliding node out of its main position? */ + /* yes; move colliding node into free position */ + while (gnext(othern) != mp) othern = gnext(othern); /* find previous */ + gnext(othern) = n; /* redo the chain with `n' in place of `mp' */ + *n = *mp; /* copy colliding node into free pos. (mp->next also goes) */ + gnext(mp) = NULL; /* now `mp' is free */ + setnilvalue(gval(mp)); + } + else { /* colliding node is in its own main position */ + /* new node will go into free position */ + gnext(n) = gnext(mp); /* chain new position */ + gnext(mp) = n; + mp = n; + } + } + gkey(mp)->value = key->value; gkey(mp)->tt = key->tt; + luaC_barriert(L, t, key); + lua_assert(ttisnil(gval(mp))); + return gval(mp); +} + + +/* +** search function for integers +*/ +const TValue *luaH_getnum (Table *t, int key) { + /* (1 <= key && key <= t->sizearray) */ + if (cast(unsigned int, key-1) < cast(unsigned int, t->sizearray)) + return &t->array[key-1]; + else { + lua_Number nk = cast_num(key); + Node *n = hashnum(t, nk); + do { /* check whether `key' is somewhere in the chain */ + if (ttisnumber(gkey(n)) && luai_numeq(nvalue(gkey(n)), nk)) + return gval(n); /* that's it */ + else n = gnext(n); + } while (n); + return luaO_nilobject; + } +} + + +/* +** search function for strings +*/ +const TValue *luaH_getstr (Table *t, TString *key) { + Node *n = hashstr(t, key); + do { /* check whether `key' is somewhere in the chain */ + if (ttisstring(gkey(n)) && rawtsvalue(gkey(n)) == key) + return gval(n); /* that's it */ + else n = gnext(n); + } while (n); + return luaO_nilobject; +} + + +/* +** main search function +*/ +const TValue *luaH_get (Table *t, const TValue *key) { + switch (ttype(key)) { + case LUA_TNIL: return luaO_nilobject; + case LUA_TSTRING: return luaH_getstr(t, rawtsvalue(key)); + case LUA_TNUMBER: { + int k; + lua_Number n = nvalue(key); + lua_number2int(k, n); + if (luai_numeq(cast_num(k), nvalue(key))) /* index is int? */ + return luaH_getnum(t, k); /* use specialized version */ + /* else go through */ + } + default: { + Node *n = mainposition(t, key); + do { /* check whether `key' is somewhere in the chain */ + if (luaO_rawequalObj(key2tval(n), key)) + return gval(n); /* that's it */ + else n = gnext(n); + } while (n); + return luaO_nilobject; + } + } +} + + +TValue *luaH_set (lua_State *L, Table *t, const TValue *key) { + const TValue *p = luaH_get(t, key); + t->flags = 0; + if (p != luaO_nilobject) + return cast(TValue *, p); + else { + if (ttisnil(key)) luaG_runerror(L, "table index is nil"); + else if (ttisnumber(key) && luai_numisnan(nvalue(key))) + luaG_runerror(L, "table index is NaN"); + return newkey(L, t, key); + } +} + + +TValue *luaH_setnum (lua_State *L, Table *t, int key) { + const TValue *p = luaH_getnum(t, key); + if (p != luaO_nilobject) + return cast(TValue *, p); + else { + TValue k; + setnvalue(&k, cast_num(key)); + return newkey(L, t, &k); + } +} + + +TValue *luaH_setstr (lua_State *L, Table *t, TString *key) { + const TValue *p = luaH_getstr(t, key); + if (p != luaO_nilobject) + return cast(TValue *, p); + else { + TValue k; + setsvalue(L, &k, key); + return newkey(L, t, &k); + } +} + + +static int unbound_search (Table *t, unsigned int j) { + unsigned int i = j; /* i is zero or a present index */ + j++; + /* find `i' and `j' such that i is present and j is not */ + while (!ttisnil(luaH_getnum(t, j))) { + i = j; + j *= 2; + if (j > cast(unsigned int, MAX_INT)) { /* overflow? */ + /* table was built with bad purposes: resort to linear search */ + i = 1; + while (!ttisnil(luaH_getnum(t, i))) i++; + return i - 1; + } + } + /* now do a binary search between them */ + while (j - i > 1) { + unsigned int m = (i+j)/2; + if (ttisnil(luaH_getnum(t, m))) j = m; + else i = m; + } + return i; +} + + +/* +** Try to find a boundary in table `t'. A `boundary' is an integer index +** such that t[i] is non-nil and t[i+1] is nil (and 0 if t[1] is nil). +*/ +int luaH_getn (Table *t) { + unsigned int j = t->sizearray; + if (j > 0 && ttisnil(&t->array[j - 1])) { + /* there is a boundary in the array part: (binary) search for it */ + unsigned int i = 0; + while (j - i > 1) { + unsigned int m = (i+j)/2; + if (ttisnil(&t->array[m - 1])) j = m; + else i = m; + } + return i; + } + /* else must find a boundary in hash part */ + else if (t->node == dummynode) /* hash part is empty? */ + return j; /* that is easy... */ + else return unbound_search(t, j); +} + + + +#if defined(LUA_DEBUG) + +Node *luaH_mainposition (const Table *t, const TValue *key) { + return mainposition(t, key); +} + +int luaH_isdummy (Node *n) { return n == dummynode; } + +#endif diff --git a/src/lua/ltable.h b/src/lua/ltable.h new file mode 100644 index 0000000..f5b9d5e --- /dev/null +++ b/src/lua/ltable.h @@ -0,0 +1,40 @@ +/* +** $Id: ltable.h,v 2.10.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua tables (hash) +** See Copyright Notice in lua.h +*/ + +#ifndef ltable_h +#define ltable_h + +#include "lobject.h" + + +#define gnode(t,i) (&(t)->node[i]) +#define gkey(n) (&(n)->i_key.nk) +#define gval(n) (&(n)->i_val) +#define gnext(n) ((n)->i_key.nk.next) + +#define key2tval(n) (&(n)->i_key.tvk) + + +LUAI_FUNC const TValue *luaH_getnum (Table *t, int key); +LUAI_FUNC TValue *luaH_setnum (lua_State *L, Table *t, int key); +LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key); +LUAI_FUNC TValue *luaH_setstr (lua_State *L, Table *t, TString *key); +LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key); +LUAI_FUNC TValue *luaH_set (lua_State *L, Table *t, const TValue *key); +LUAI_FUNC Table *luaH_new (lua_State *L, int narray, int lnhash); +LUAI_FUNC void luaH_resizearray (lua_State *L, Table *t, int nasize); +LUAI_FUNC void luaH_free (lua_State *L, Table *t); +LUAI_FUNC int luaH_next (lua_State *L, Table *t, StkId key); +LUAI_FUNC int luaH_getn (Table *t); + + +#if defined(LUA_DEBUG) +LUAI_FUNC Node *luaH_mainposition (const Table *t, const TValue *key); +LUAI_FUNC int luaH_isdummy (Node *n); +#endif + + +#endif diff --git a/src/lua/ltablib.c b/src/lua/ltablib.c new file mode 100644 index 0000000..b6d9cb4 --- /dev/null +++ b/src/lua/ltablib.c @@ -0,0 +1,287 @@ +/* +** $Id: ltablib.c,v 1.38.1.3 2008/02/14 16:46:58 roberto Exp $ +** Library for Table Manipulation +** See Copyright Notice in lua.h +*/ + + +#include + +#define ltablib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +#define aux_getn(L,n) (luaL_checktype(L, n, LUA_TTABLE), luaL_getn(L, n)) + + +static int foreachi (lua_State *L) { + int i; + int n = aux_getn(L, 1); + luaL_checktype(L, 2, LUA_TFUNCTION); + for (i=1; i <= n; i++) { + lua_pushvalue(L, 2); /* function */ + lua_pushinteger(L, i); /* 1st argument */ + lua_rawgeti(L, 1, i); /* 2nd argument */ + lua_call(L, 2, 1); + if (!lua_isnil(L, -1)) + return 1; + lua_pop(L, 1); /* remove nil result */ + } + return 0; +} + + +static int foreach (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + luaL_checktype(L, 2, LUA_TFUNCTION); + lua_pushnil(L); /* first key */ + while (lua_next(L, 1)) { + lua_pushvalue(L, 2); /* function */ + lua_pushvalue(L, -3); /* key */ + lua_pushvalue(L, -3); /* value */ + lua_call(L, 2, 1); + if (!lua_isnil(L, -1)) + return 1; + lua_pop(L, 2); /* remove value and result */ + } + return 0; +} + + +static int maxn (lua_State *L) { + lua_Number max = 0; + luaL_checktype(L, 1, LUA_TTABLE); + lua_pushnil(L); /* first key */ + while (lua_next(L, 1)) { + lua_pop(L, 1); /* remove value */ + if (lua_type(L, -1) == LUA_TNUMBER) { + lua_Number v = lua_tonumber(L, -1); + if (v > max) max = v; + } + } + lua_pushnumber(L, max); + return 1; +} + + +static int getn (lua_State *L) { + lua_pushinteger(L, aux_getn(L, 1)); + return 1; +} + + +static int setn (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); +#ifndef luaL_setn + luaL_setn(L, 1, luaL_checkint(L, 2)); +#else + luaL_error(L, LUA_QL("setn") " is obsolete"); +#endif + lua_pushvalue(L, 1); + return 1; +} + + +static int tinsert (lua_State *L) { + int e = aux_getn(L, 1) + 1; /* first empty element */ + int pos; /* where to insert new element */ + switch (lua_gettop(L)) { + case 2: { /* called with only 2 arguments */ + pos = e; /* insert new element at the end */ + break; + } + case 3: { + int i; + pos = luaL_checkint(L, 2); /* 2nd argument is the position */ + if (pos > e) e = pos; /* `grow' array if necessary */ + for (i = e; i > pos; i--) { /* move up elements */ + lua_rawgeti(L, 1, i-1); + lua_rawseti(L, 1, i); /* t[i] = t[i-1] */ + } + break; + } + default: { + return luaL_error(L, "wrong number of arguments to " LUA_QL("insert")); + } + } + luaL_setn(L, 1, e); /* new size */ + lua_rawseti(L, 1, pos); /* t[pos] = v */ + return 0; +} + + +static int tremove (lua_State *L) { + int e = aux_getn(L, 1); + int pos = luaL_optint(L, 2, e); + if (!(1 <= pos && pos <= e)) /* position is outside bounds? */ + return 0; /* nothing to remove */ + luaL_setn(L, 1, e - 1); /* t.n = n-1 */ + lua_rawgeti(L, 1, pos); /* result = t[pos] */ + for ( ;pos= P */ + while (lua_rawgeti(L, 1, ++i), sort_comp(L, -1, -2)) { + if (i>u) luaL_error(L, "invalid order function for sorting"); + lua_pop(L, 1); /* remove a[i] */ + } + /* repeat --j until a[j] <= P */ + while (lua_rawgeti(L, 1, --j), sort_comp(L, -3, -1)) { + if (j + +#define ltm_c +#define LUA_CORE + +#include "lua.h" + +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" + + + +const char *const luaT_typenames[] = { + "nil", "boolean", "userdata", "number", + "string", "table", "function", "userdata", "thread", + "proto", "upval" +}; + + +void luaT_init (lua_State *L) { + static const char *const luaT_eventname[] = { /* ORDER TM */ + "__index", "__newindex", + "__gc", "__mode", "__eq", + "__add", "__sub", "__mul", "__div", "__mod", + "__pow", "__unm", "__len", "__lt", "__le", + "__concat", "__call" + }; + int i; + for (i=0; itmname[i] = luaS_new(L, luaT_eventname[i]); + luaS_fix(G(L)->tmname[i]); /* never collect these names */ + } +} + + +/* +** function to be used with macro "fasttm": optimized for absence of +** tag methods +*/ +const TValue *luaT_gettm (Table *events, TMS event, TString *ename) { + const TValue *tm = luaH_getstr(events, ename); + lua_assert(event <= TM_EQ); + if (ttisnil(tm)) { /* no tag method? */ + events->flags |= cast_byte(1u<metatable; + break; + case LUA_TUSERDATA: + mt = uvalue(o)->metatable; + break; + default: + mt = G(L)->mt[ttype(o)]; + } + return (mt ? luaH_getstr(mt, G(L)->tmname[event]) : luaO_nilobject); +} + diff --git a/src/lua/ltm.h b/src/lua/ltm.h new file mode 100644 index 0000000..64343b7 --- /dev/null +++ b/src/lua/ltm.h @@ -0,0 +1,54 @@ +/* +** $Id: ltm.h,v 2.6.1.1 2007/12/27 13:02:25 roberto Exp $ +** Tag methods +** See Copyright Notice in lua.h +*/ + +#ifndef ltm_h +#define ltm_h + + +#include "lobject.h" + + +/* +* WARNING: if you change the order of this enumeration, +* grep "ORDER TM" +*/ +typedef enum { + TM_INDEX, + TM_NEWINDEX, + TM_GC, + TM_MODE, + TM_EQ, /* last tag method with `fast' access */ + TM_ADD, + TM_SUB, + TM_MUL, + TM_DIV, + TM_MOD, + TM_POW, + TM_UNM, + TM_LEN, + TM_LT, + TM_LE, + TM_CONCAT, + TM_CALL, + TM_N /* number of elements in the enum */ +} TMS; + + + +#define gfasttm(g,et,e) ((et) == NULL ? NULL : \ + ((et)->flags & (1u<<(e))) ? NULL : luaT_gettm(et, e, (g)->tmname[e])) + +#define fasttm(l,et,e) gfasttm(G(l), et, e) + +LUAI_DATA const char *const luaT_typenames[]; + + +LUAI_FUNC const TValue *luaT_gettm (Table *events, TMS event, TString *ename); +LUAI_FUNC const TValue *luaT_gettmbyobj (lua_State *L, const TValue *o, + TMS event); +LUAI_FUNC void luaT_init (lua_State *L); + +#endif diff --git a/src/lua/lundump.c b/src/lua/lundump.c new file mode 100644 index 0000000..8010a45 --- /dev/null +++ b/src/lua/lundump.c @@ -0,0 +1,227 @@ +/* +** $Id: lundump.c,v 2.7.1.4 2008/04/04 19:51:41 roberto Exp $ +** load precompiled Lua chunks +** See Copyright Notice in lua.h +*/ + +#include + +#define lundump_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstring.h" +#include "lundump.h" +#include "lzio.h" + +typedef struct { + lua_State* L; + ZIO* Z; + Mbuffer* b; + const char* name; +} LoadState; + +#ifdef LUAC_TRUST_BINARIES +#define IF(c,s) +#define error(S,s) +#else +#define IF(c,s) if (c) error(S,s) + +static void error(LoadState* S, const char* why) +{ + luaO_pushfstring(S->L,"%s: %s in precompiled chunk",S->name,why); + luaD_throw(S->L,LUA_ERRSYNTAX); +} +#endif + +#define LoadMem(S,b,n,size) LoadBlock(S,b,(n)*(size)) +#define LoadByte(S) (lu_byte)LoadChar(S) +#define LoadVar(S,x) LoadMem(S,&x,1,sizeof(x)) +#define LoadVector(S,b,n,size) LoadMem(S,b,n,size) + +static void LoadBlock(LoadState* S, void* b, size_t size) +{ + size_t r=luaZ_read(S->Z,b,size); + IF (r!=0, "unexpected end"); +} + +static int LoadChar(LoadState* S) +{ + char x; + LoadVar(S,x); + return x; +} + +static int LoadInt(LoadState* S) +{ + int x; + LoadVar(S,x); + IF (x<0, "bad integer"); + return x; +} + +static lua_Number LoadNumber(LoadState* S) +{ + lua_Number x; + LoadVar(S,x); + return x; +} + +static TString* LoadString(LoadState* S) +{ + size_t size; + LoadVar(S,size); + if (size==0) + return NULL; + else + { + char* s=luaZ_openspace(S->L,S->b,size); + LoadBlock(S,s,size); + return luaS_newlstr(S->L,s,size-1); /* remove trailing '\0' */ + } +} + +static void LoadCode(LoadState* S, Proto* f) +{ + int n=LoadInt(S); + f->code=luaM_newvector(S->L,n,Instruction); + f->sizecode=n; + LoadVector(S,f->code,n,sizeof(Instruction)); +} + +static Proto* LoadFunction(LoadState* S, TString* p); + +static void LoadConstants(LoadState* S, Proto* f) +{ + int i,n; + n=LoadInt(S); + f->k=luaM_newvector(S->L,n,TValue); + f->sizek=n; + for (i=0; ik[i]); + for (i=0; ik[i]; + int t=LoadChar(S); + switch (t) + { + case LUA_TNIL: + setnilvalue(o); + break; + case LUA_TBOOLEAN: + setbvalue(o,LoadChar(S)!=0); + break; + case LUA_TNUMBER: + setnvalue(o,LoadNumber(S)); + break; + case LUA_TSTRING: + setsvalue2n(S->L,o,LoadString(S)); + break; + default: + error(S,"bad constant"); + break; + } + } + n=LoadInt(S); + f->p=luaM_newvector(S->L,n,Proto*); + f->sizep=n; + for (i=0; ip[i]=NULL; + for (i=0; ip[i]=LoadFunction(S,f->source); +} + +static void LoadDebug(LoadState* S, Proto* f) +{ + int i,n; + n=LoadInt(S); + f->lineinfo=luaM_newvector(S->L,n,int); + f->sizelineinfo=n; + LoadVector(S,f->lineinfo,n,sizeof(int)); + n=LoadInt(S); + f->locvars=luaM_newvector(S->L,n,LocVar); + f->sizelocvars=n; + for (i=0; ilocvars[i].varname=NULL; + for (i=0; ilocvars[i].varname=LoadString(S); + f->locvars[i].startpc=LoadInt(S); + f->locvars[i].endpc=LoadInt(S); + } + n=LoadInt(S); + f->upvalues=luaM_newvector(S->L,n,TString*); + f->sizeupvalues=n; + for (i=0; iupvalues[i]=NULL; + for (i=0; iupvalues[i]=LoadString(S); +} + +static Proto* LoadFunction(LoadState* S, TString* p) +{ + Proto* f; + if (++S->L->nCcalls > LUAI_MAXCCALLS) error(S,"code too deep"); + f=luaF_newproto(S->L); + setptvalue2s(S->L,S->L->top,f); incr_top(S->L); + f->source=LoadString(S); if (f->source==NULL) f->source=p; + f->linedefined=LoadInt(S); + f->lastlinedefined=LoadInt(S); + f->nups=LoadByte(S); + f->numparams=LoadByte(S); + f->is_vararg=LoadByte(S); + f->maxstacksize=LoadByte(S); + LoadCode(S,f); + LoadConstants(S,f); + LoadDebug(S,f); + IF (!luaG_checkcode(f), "bad code"); + S->L->top--; + S->L->nCcalls--; + return f; +} + +static void LoadHeader(LoadState* S) +{ + char h[LUAC_HEADERSIZE]; + char s[LUAC_HEADERSIZE]; + luaU_header(h); + LoadBlock(S,s,LUAC_HEADERSIZE); + IF (memcmp(h,s,LUAC_HEADERSIZE)!=0, "bad header"); +} + +/* +** load precompiled chunk +*/ +Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name) +{ + LoadState S; + if (*name=='@' || *name=='=') + S.name=name+1; + else if (*name==LUA_SIGNATURE[0]) + S.name="binary string"; + else + S.name=name; + S.L=L; + S.Z=Z; + S.b=buff; + LoadHeader(&S); + return LoadFunction(&S,luaS_newliteral(L,"=?")); +} + +/* +* make header +*/ +void luaU_header (char* h) +{ + int x=1; + memcpy(h,LUA_SIGNATURE,sizeof(LUA_SIGNATURE)-1); + h+=sizeof(LUA_SIGNATURE)-1; + *h++=(char)LUAC_VERSION; + *h++=(char)LUAC_FORMAT; + *h++=(char)*(char*)&x; /* endianness */ + *h++=(char)sizeof(int); + *h++=(char)sizeof(size_t); + *h++=(char)sizeof(Instruction); + *h++=(char)sizeof(lua_Number); + *h++=(char)(((lua_Number)0.5)==0); /* is lua_Number integral? */ +} diff --git a/src/lua/lundump.h b/src/lua/lundump.h new file mode 100644 index 0000000..c80189d --- /dev/null +++ b/src/lua/lundump.h @@ -0,0 +1,36 @@ +/* +** $Id: lundump.h,v 1.37.1.1 2007/12/27 13:02:25 roberto Exp $ +** load precompiled Lua chunks +** See Copyright Notice in lua.h +*/ + +#ifndef lundump_h +#define lundump_h + +#include "lobject.h" +#include "lzio.h" + +/* load one chunk; from lundump.c */ +LUAI_FUNC Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name); + +/* make header; from lundump.c */ +LUAI_FUNC void luaU_header (char* h); + +/* dump one chunk; from ldump.c */ +LUAI_FUNC int luaU_dump (lua_State* L, const Proto* f, lua_Writer w, void* data, int strip); + +#ifdef luac_c +/* print one chunk; from print.c */ +LUAI_FUNC void luaU_print (const Proto* f, int full); +#endif + +/* for header of binary files -- this is Lua 5.1 */ +#define LUAC_VERSION 0x51 + +/* for header of binary files -- this is the official format */ +#define LUAC_FORMAT 0 + +/* size of header of binary files */ +#define LUAC_HEADERSIZE 12 + +#endif diff --git a/src/lua/lvm.c b/src/lua/lvm.c new file mode 100644 index 0000000..e0a0cd8 --- /dev/null +++ b/src/lua/lvm.c @@ -0,0 +1,767 @@ +/* +** $Id: lvm.c,v 2.63.1.5 2011/08/17 20:43:11 roberto Exp $ +** Lua virtual machine +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define lvm_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lvm.h" + + + +/* limit for table tag-method chains (to avoid loops) */ +#define MAXTAGLOOP 100 + + +const TValue *luaV_tonumber (const TValue *obj, TValue *n) { + lua_Number num; + if (ttisnumber(obj)) return obj; + if (ttisstring(obj) && luaO_str2d(svalue(obj), &num)) { + setnvalue(n, num); + return n; + } + else + return NULL; +} + + +int luaV_tostring (lua_State *L, StkId obj) { + if (!ttisnumber(obj)) + return 0; + else { + char s[LUAI_MAXNUMBER2STR]; + lua_Number n = nvalue(obj); + lua_number2str(s, n); + setsvalue2s(L, obj, luaS_new(L, s)); + return 1; + } +} + + +static void traceexec (lua_State *L, const Instruction *pc) { + lu_byte mask = L->hookmask; + const Instruction *oldpc = L->savedpc; + L->savedpc = pc; + if ((mask & LUA_MASKCOUNT) && L->hookcount == 0) { + resethookcount(L); + luaD_callhook(L, LUA_HOOKCOUNT, -1); + } + if (mask & LUA_MASKLINE) { + Proto *p = ci_func(L->ci)->l.p; + int npc = pcRel(pc, p); + int newline = getline(p, npc); + /* call linehook when enter a new function, when jump back (loop), + or when enter a new line */ + if (npc == 0 || pc <= oldpc || newline != getline(p, pcRel(oldpc, p))) + luaD_callhook(L, LUA_HOOKLINE, newline); + } +} + + +static void callTMres (lua_State *L, StkId res, const TValue *f, + const TValue *p1, const TValue *p2) { + ptrdiff_t result = savestack(L, res); + setobj2s(L, L->top, f); /* push function */ + setobj2s(L, L->top+1, p1); /* 1st argument */ + setobj2s(L, L->top+2, p2); /* 2nd argument */ + luaD_checkstack(L, 3); + L->top += 3; + luaD_call(L, L->top - 3, 1); + res = restorestack(L, result); + L->top--; + setobjs2s(L, res, L->top); +} + + + +static void callTM (lua_State *L, const TValue *f, const TValue *p1, + const TValue *p2, const TValue *p3) { + setobj2s(L, L->top, f); /* push function */ + setobj2s(L, L->top+1, p1); /* 1st argument */ + setobj2s(L, L->top+2, p2); /* 2nd argument */ + setobj2s(L, L->top+3, p3); /* 3th argument */ + luaD_checkstack(L, 4); + L->top += 4; + luaD_call(L, L->top - 4, 0); +} + + +void luaV_gettable (lua_State *L, const TValue *t, TValue *key, StkId val) { + int loop; + for (loop = 0; loop < MAXTAGLOOP; loop++) { + const TValue *tm; + if (ttistable(t)) { /* `t' is a table? */ + Table *h = hvalue(t); + const TValue *res = luaH_get(h, key); /* do a primitive get */ + if (!ttisnil(res) || /* result is no nil? */ + (tm = fasttm(L, h->metatable, TM_INDEX)) == NULL) { /* or no TM? */ + setobj2s(L, val, res); + return; + } + /* else will try the tag method */ + } + else if (ttisnil(tm = luaT_gettmbyobj(L, t, TM_INDEX))) + luaG_typeerror(L, t, "index"); + if (ttisfunction(tm)) { + callTMres(L, val, tm, t, key); + return; + } + t = tm; /* else repeat with `tm' */ + } + luaG_runerror(L, "loop in gettable"); +} + + +void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { + int loop; + TValue temp; + for (loop = 0; loop < MAXTAGLOOP; loop++) { + const TValue *tm; + if (ttistable(t)) { /* `t' is a table? */ + Table *h = hvalue(t); + TValue *oldval = luaH_set(L, h, key); /* do a primitive set */ + if (!ttisnil(oldval) || /* result is no nil? */ + (tm = fasttm(L, h->metatable, TM_NEWINDEX)) == NULL) { /* or no TM? */ + setobj2t(L, oldval, val); + h->flags = 0; + luaC_barriert(L, h, val); + return; + } + /* else will try the tag method */ + } + else if (ttisnil(tm = luaT_gettmbyobj(L, t, TM_NEWINDEX))) + luaG_typeerror(L, t, "index"); + if (ttisfunction(tm)) { + callTM(L, tm, t, key, val); + return; + } + /* else repeat with `tm' */ + setobj(L, &temp, tm); /* avoid pointing inside table (may rehash) */ + t = &temp; + } + luaG_runerror(L, "loop in settable"); +} + + +static int call_binTM (lua_State *L, const TValue *p1, const TValue *p2, + StkId res, TMS event) { + const TValue *tm = luaT_gettmbyobj(L, p1, event); /* try first operand */ + if (ttisnil(tm)) + tm = luaT_gettmbyobj(L, p2, event); /* try second operand */ + if (ttisnil(tm)) return 0; + callTMres(L, res, tm, p1, p2); + return 1; +} + + +static const TValue *get_compTM (lua_State *L, Table *mt1, Table *mt2, + TMS event) { + const TValue *tm1 = fasttm(L, mt1, event); + const TValue *tm2; + if (tm1 == NULL) return NULL; /* no metamethod */ + if (mt1 == mt2) return tm1; /* same metatables => same metamethods */ + tm2 = fasttm(L, mt2, event); + if (tm2 == NULL) return NULL; /* no metamethod */ + if (luaO_rawequalObj(tm1, tm2)) /* same metamethods? */ + return tm1; + return NULL; +} + + +static int call_orderTM (lua_State *L, const TValue *p1, const TValue *p2, + TMS event) { + const TValue *tm1 = luaT_gettmbyobj(L, p1, event); + const TValue *tm2; + if (ttisnil(tm1)) return -1; /* no metamethod? */ + tm2 = luaT_gettmbyobj(L, p2, event); + if (!luaO_rawequalObj(tm1, tm2)) /* different metamethods? */ + return -1; + callTMres(L, L->top, tm1, p1, p2); + return !l_isfalse(L->top); +} + + +static int l_strcmp (const TString *ls, const TString *rs) { + const char *l = getstr(ls); + size_t ll = ls->tsv.len; + const char *r = getstr(rs); + size_t lr = rs->tsv.len; + for (;;) { + int temp = strcoll(l, r); + if (temp != 0) return temp; + else { /* strings are equal up to a `\0' */ + size_t len = strlen(l); /* index of first `\0' in both strings */ + if (len == lr) /* r is finished? */ + return (len == ll) ? 0 : 1; + else if (len == ll) /* l is finished? */ + return -1; /* l is smaller than r (because r is not finished) */ + /* both strings longer than `len'; go on comparing (after the `\0') */ + len++; + l += len; ll -= len; r += len; lr -= len; + } + } +} + + +int luaV_lessthan (lua_State *L, const TValue *l, const TValue *r) { + int res; + if (ttype(l) != ttype(r)) + return luaG_ordererror(L, l, r); + else if (ttisnumber(l)) + return luai_numlt(nvalue(l), nvalue(r)); + else if (ttisstring(l)) + return l_strcmp(rawtsvalue(l), rawtsvalue(r)) < 0; + else if ((res = call_orderTM(L, l, r, TM_LT)) != -1) + return res; + return luaG_ordererror(L, l, r); +} + + +static int lessequal (lua_State *L, const TValue *l, const TValue *r) { + int res; + if (ttype(l) != ttype(r)) + return luaG_ordererror(L, l, r); + else if (ttisnumber(l)) + return luai_numle(nvalue(l), nvalue(r)); + else if (ttisstring(l)) + return l_strcmp(rawtsvalue(l), rawtsvalue(r)) <= 0; + else if ((res = call_orderTM(L, l, r, TM_LE)) != -1) /* first try `le' */ + return res; + else if ((res = call_orderTM(L, r, l, TM_LT)) != -1) /* else try `lt' */ + return !res; + return luaG_ordererror(L, l, r); +} + + +int luaV_equalval (lua_State *L, const TValue *t1, const TValue *t2) { + const TValue *tm; + lua_assert(ttype(t1) == ttype(t2)); + switch (ttype(t1)) { + case LUA_TNIL: return 1; + case LUA_TNUMBER: return luai_numeq(nvalue(t1), nvalue(t2)); + case LUA_TBOOLEAN: return bvalue(t1) == bvalue(t2); /* true must be 1 !! */ + case LUA_TLIGHTUSERDATA: return pvalue(t1) == pvalue(t2); + case LUA_TUSERDATA: { + if (uvalue(t1) == uvalue(t2)) return 1; + tm = get_compTM(L, uvalue(t1)->metatable, uvalue(t2)->metatable, + TM_EQ); + break; /* will try TM */ + } + case LUA_TTABLE: { + if (hvalue(t1) == hvalue(t2)) return 1; + tm = get_compTM(L, hvalue(t1)->metatable, hvalue(t2)->metatable, TM_EQ); + break; /* will try TM */ + } + default: return gcvalue(t1) == gcvalue(t2); + } + if (tm == NULL) return 0; /* no TM? */ + callTMres(L, L->top, tm, t1, t2); /* call TM */ + return !l_isfalse(L->top); +} + + +void luaV_concat (lua_State *L, int total, int last) { + do { + StkId top = L->base + last + 1; + int n = 2; /* number of elements handled in this pass (at least 2) */ + if (!(ttisstring(top-2) || ttisnumber(top-2)) || !tostring(L, top-1)) { + if (!call_binTM(L, top-2, top-1, top-2, TM_CONCAT)) + luaG_concaterror(L, top-2, top-1); + } else if (tsvalue(top-1)->len == 0) /* second op is empty? */ + (void)tostring(L, top - 2); /* result is first op (as string) */ + else { + /* at least two string values; get as many as possible */ + size_t tl = tsvalue(top-1)->len; + char *buffer; + int i; + /* collect total length */ + for (n = 1; n < total && tostring(L, top-n-1); n++) { + size_t l = tsvalue(top-n-1)->len; + if (l >= MAX_SIZET - tl) luaG_runerror(L, "string length overflow"); + tl += l; + } + buffer = luaZ_openspace(L, &G(L)->buff, tl); + tl = 0; + for (i=n; i>0; i--) { /* concat all strings */ + size_t l = tsvalue(top-i)->len; + memcpy(buffer+tl, svalue(top-i), l); + tl += l; + } + setsvalue2s(L, top-n, luaS_newlstr(L, buffer, tl)); + } + total -= n-1; /* got `n' strings to create 1 new */ + last -= n-1; + } while (total > 1); /* repeat until only 1 result left */ +} + + +static void Arith (lua_State *L, StkId ra, const TValue *rb, + const TValue *rc, TMS op) { + TValue tempb, tempc; + const TValue *b, *c; + if ((b = luaV_tonumber(rb, &tempb)) != NULL && + (c = luaV_tonumber(rc, &tempc)) != NULL) { + lua_Number nb = nvalue(b), nc = nvalue(c); + switch (op) { + case TM_ADD: setnvalue(ra, luai_numadd(nb, nc)); break; + case TM_SUB: setnvalue(ra, luai_numsub(nb, nc)); break; + case TM_MUL: setnvalue(ra, luai_nummul(nb, nc)); break; + case TM_DIV: setnvalue(ra, luai_numdiv(nb, nc)); break; + case TM_MOD: setnvalue(ra, luai_nummod(nb, nc)); break; + case TM_POW: setnvalue(ra, luai_numpow(nb, nc)); break; + case TM_UNM: setnvalue(ra, luai_numunm(nb)); break; + default: lua_assert(0); break; + } + } + else if (!call_binTM(L, rb, rc, ra, op)) + luaG_aritherror(L, rb, rc); +} + + + +/* +** some macros for common tasks in `luaV_execute' +*/ + +#define runtime_check(L, c) { if (!(c)) break; } + +#define RA(i) (base+GETARG_A(i)) +/* to be used after possible stack reallocation */ +#define RB(i) check_exp(getBMode(GET_OPCODE(i)) == OpArgR, base+GETARG_B(i)) +#define RC(i) check_exp(getCMode(GET_OPCODE(i)) == OpArgR, base+GETARG_C(i)) +#define RKB(i) check_exp(getBMode(GET_OPCODE(i)) == OpArgK, \ + ISK(GETARG_B(i)) ? k+INDEXK(GETARG_B(i)) : base+GETARG_B(i)) +#define RKC(i) check_exp(getCMode(GET_OPCODE(i)) == OpArgK, \ + ISK(GETARG_C(i)) ? k+INDEXK(GETARG_C(i)) : base+GETARG_C(i)) +#define KBx(i) check_exp(getBMode(GET_OPCODE(i)) == OpArgK, k+GETARG_Bx(i)) + + +#define dojump(L,pc,i) {(pc) += (i); luai_threadyield(L);} + + +#define Protect(x) { L->savedpc = pc; {x;}; base = L->base; } + + +#define arith_op(op,tm) { \ + TValue *rb = RKB(i); \ + TValue *rc = RKC(i); \ + if (ttisnumber(rb) && ttisnumber(rc)) { \ + lua_Number nb = nvalue(rb), nc = nvalue(rc); \ + setnvalue(ra, op(nb, nc)); \ + } \ + else \ + Protect(Arith(L, ra, rb, rc, tm)); \ + } + + + +void luaV_execute (lua_State *L, int nexeccalls) { + LClosure *cl; + StkId base; + TValue *k; + const Instruction *pc; + reentry: /* entry point */ + lua_assert(isLua(L->ci)); + pc = L->savedpc; + cl = &clvalue(L->ci->func)->l; + base = L->base; + k = cl->p->k; + /* main loop of interpreter */ + for (;;) { + const Instruction i = *pc++; + StkId ra; + if ((L->hookmask & (LUA_MASKLINE | LUA_MASKCOUNT)) && + (--L->hookcount == 0 || L->hookmask & LUA_MASKLINE)) { + traceexec(L, pc); + if (L->status == LUA_YIELD) { /* did hook yield? */ + L->savedpc = pc - 1; + return; + } + base = L->base; + } + /* warning!! several calls may realloc the stack and invalidate `ra' */ + ra = RA(i); + lua_assert(base == L->base && L->base == L->ci->base); + lua_assert(base <= L->top && L->top <= L->stack + L->stacksize); + lua_assert(L->top == L->ci->top || luaG_checkopenop(i)); + switch (GET_OPCODE(i)) { + case OP_MOVE: { + setobjs2s(L, ra, RB(i)); + continue; + } + case OP_LOADK: { + setobj2s(L, ra, KBx(i)); + continue; + } + case OP_LOADBOOL: { + setbvalue(ra, GETARG_B(i)); + if (GETARG_C(i)) pc++; /* skip next instruction (if C) */ + continue; + } + case OP_LOADNIL: { + TValue *rb = RB(i); + do { + setnilvalue(rb--); + } while (rb >= ra); + continue; + } + case OP_GETUPVAL: { + int b = GETARG_B(i); + setobj2s(L, ra, cl->upvals[b]->v); + continue; + } + case OP_GETGLOBAL: { + TValue g; + TValue *rb = KBx(i); + sethvalue(L, &g, cl->env); + lua_assert(ttisstring(rb)); + Protect(luaV_gettable(L, &g, rb, ra)); + continue; + } + case OP_GETTABLE: { + Protect(luaV_gettable(L, RB(i), RKC(i), ra)); + continue; + } + case OP_SETGLOBAL: { + TValue g; + sethvalue(L, &g, cl->env); + lua_assert(ttisstring(KBx(i))); + Protect(luaV_settable(L, &g, KBx(i), ra)); + continue; + } + case OP_SETUPVAL: { + UpVal *uv = cl->upvals[GETARG_B(i)]; + setobj(L, uv->v, ra); + luaC_barrier(L, uv, ra); + continue; + } + case OP_SETTABLE: { + Protect(luaV_settable(L, ra, RKB(i), RKC(i))); + continue; + } + case OP_NEWTABLE: { + int b = GETARG_B(i); + int c = GETARG_C(i); + sethvalue(L, ra, luaH_new(L, luaO_fb2int(b), luaO_fb2int(c))); + Protect(luaC_checkGC(L)); + continue; + } + case OP_SELF: { + StkId rb = RB(i); + setobjs2s(L, ra+1, rb); + Protect(luaV_gettable(L, rb, RKC(i), ra)); + continue; + } + case OP_ADD: { + arith_op(luai_numadd, TM_ADD); + continue; + } + case OP_SUB: { + arith_op(luai_numsub, TM_SUB); + continue; + } + case OP_MUL: { + arith_op(luai_nummul, TM_MUL); + continue; + } + case OP_DIV: { + arith_op(luai_numdiv, TM_DIV); + continue; + } + case OP_MOD: { + arith_op(luai_nummod, TM_MOD); + continue; + } + case OP_POW: { + arith_op(luai_numpow, TM_POW); + continue; + } + case OP_UNM: { + TValue *rb = RB(i); + if (ttisnumber(rb)) { + lua_Number nb = nvalue(rb); + setnvalue(ra, luai_numunm(nb)); + } + else { + Protect(Arith(L, ra, rb, rb, TM_UNM)); + } + continue; + } + case OP_NOT: { + int res = l_isfalse(RB(i)); /* next assignment may change this value */ + setbvalue(ra, res); + continue; + } + case OP_LEN: { + const TValue *rb = RB(i); + switch (ttype(rb)) { + case LUA_TTABLE: { + setnvalue(ra, cast_num(luaH_getn(hvalue(rb)))); + break; + } + case LUA_TSTRING: { + setnvalue(ra, cast_num(tsvalue(rb)->len)); + break; + } + default: { /* try metamethod */ + Protect( + if (!call_binTM(L, rb, luaO_nilobject, ra, TM_LEN)) + luaG_typeerror(L, rb, "get length of"); + ) + } + } + continue; + } + case OP_CONCAT: { + int b = GETARG_B(i); + int c = GETARG_C(i); + Protect(luaV_concat(L, c-b+1, c); luaC_checkGC(L)); + setobjs2s(L, RA(i), base+b); + continue; + } + case OP_JMP: { + dojump(L, pc, GETARG_sBx(i)); + continue; + } + case OP_EQ: { + TValue *rb = RKB(i); + TValue *rc = RKC(i); + Protect( + if (equalobj(L, rb, rc) == GETARG_A(i)) + dojump(L, pc, GETARG_sBx(*pc)); + ) + pc++; + continue; + } + case OP_LT: { + Protect( + if (luaV_lessthan(L, RKB(i), RKC(i)) == GETARG_A(i)) + dojump(L, pc, GETARG_sBx(*pc)); + ) + pc++; + continue; + } + case OP_LE: { + Protect( + if (lessequal(L, RKB(i), RKC(i)) == GETARG_A(i)) + dojump(L, pc, GETARG_sBx(*pc)); + ) + pc++; + continue; + } + case OP_TEST: { + if (l_isfalse(ra) != GETARG_C(i)) + dojump(L, pc, GETARG_sBx(*pc)); + pc++; + continue; + } + case OP_TESTSET: { + TValue *rb = RB(i); + if (l_isfalse(rb) != GETARG_C(i)) { + setobjs2s(L, ra, rb); + dojump(L, pc, GETARG_sBx(*pc)); + } + pc++; + continue; + } + case OP_CALL: { + int b = GETARG_B(i); + int nresults = GETARG_C(i) - 1; + if (b != 0) L->top = ra+b; /* else previous instruction set top */ + L->savedpc = pc; + switch (luaD_precall(L, ra, nresults)) { + case PCRLUA: { + nexeccalls++; + goto reentry; /* restart luaV_execute over new Lua function */ + } + case PCRC: { + /* it was a C function (`precall' called it); adjust results */ + if (nresults >= 0) L->top = L->ci->top; + base = L->base; + continue; + } + default: { + return; /* yield */ + } + } + } + case OP_TAILCALL: { + int b = GETARG_B(i); + if (b != 0) L->top = ra+b; /* else previous instruction set top */ + L->savedpc = pc; + lua_assert(GETARG_C(i) - 1 == LUA_MULTRET); + switch (luaD_precall(L, ra, LUA_MULTRET)) { + case PCRLUA: { + /* tail call: put new frame in place of previous one */ + CallInfo *ci = L->ci - 1; /* previous frame */ + int aux; + StkId func = ci->func; + StkId pfunc = (ci+1)->func; /* previous function index */ + if (L->openupval) luaF_close(L, ci->base); + L->base = ci->base = ci->func + ((ci+1)->base - pfunc); + for (aux = 0; pfunc+aux < L->top; aux++) /* move frame down */ + setobjs2s(L, func+aux, pfunc+aux); + ci->top = L->top = func+aux; /* correct top */ + lua_assert(L->top == L->base + clvalue(func)->l.p->maxstacksize); + ci->savedpc = L->savedpc; + ci->tailcalls++; /* one more call lost */ + L->ci--; /* remove new frame */ + goto reentry; + } + case PCRC: { /* it was a C function (`precall' called it) */ + base = L->base; + continue; + } + default: { + return; /* yield */ + } + } + } + case OP_RETURN: { + int b = GETARG_B(i); + if (b != 0) L->top = ra+b-1; + if (L->openupval) luaF_close(L, base); + L->savedpc = pc; + b = luaD_poscall(L, ra); + if (--nexeccalls == 0) /* was previous function running `here'? */ + return; /* no: return */ + else { /* yes: continue its execution */ + if (b) L->top = L->ci->top; + lua_assert(isLua(L->ci)); + lua_assert(GET_OPCODE(*((L->ci)->savedpc - 1)) == OP_CALL); + goto reentry; + } + } + case OP_FORLOOP: { + lua_Number step = nvalue(ra+2); + lua_Number idx = luai_numadd(nvalue(ra), step); /* increment index */ + lua_Number limit = nvalue(ra+1); + if (luai_numlt(0, step) ? luai_numle(idx, limit) + : luai_numle(limit, idx)) { + dojump(L, pc, GETARG_sBx(i)); /* jump back */ + setnvalue(ra, idx); /* update internal index... */ + setnvalue(ra+3, idx); /* ...and external index */ + } + continue; + } + case OP_FORPREP: { + const TValue *init = ra; + const TValue *plimit = ra+1; + const TValue *pstep = ra+2; + L->savedpc = pc; /* next steps may throw errors */ + if (!tonumber(init, ra)) + luaG_runerror(L, LUA_QL("for") " initial value must be a number"); + else if (!tonumber(plimit, ra+1)) + luaG_runerror(L, LUA_QL("for") " limit must be a number"); + else if (!tonumber(pstep, ra+2)) + luaG_runerror(L, LUA_QL("for") " step must be a number"); + setnvalue(ra, luai_numsub(nvalue(ra), nvalue(pstep))); + dojump(L, pc, GETARG_sBx(i)); + continue; + } + case OP_TFORLOOP: { + StkId cb = ra + 3; /* call base */ + setobjs2s(L, cb+2, ra+2); + setobjs2s(L, cb+1, ra+1); + setobjs2s(L, cb, ra); + L->top = cb+3; /* func. + 2 args (state and index) */ + Protect(luaD_call(L, cb, GETARG_C(i))); + L->top = L->ci->top; + cb = RA(i) + 3; /* previous call may change the stack */ + if (!ttisnil(cb)) { /* continue loop? */ + setobjs2s(L, cb-1, cb); /* save control variable */ + dojump(L, pc, GETARG_sBx(*pc)); /* jump back */ + } + pc++; + continue; + } + case OP_SETLIST: { + int n = GETARG_B(i); + int c = GETARG_C(i); + int last; + Table *h; + if (n == 0) { + n = cast_int(L->top - ra) - 1; + L->top = L->ci->top; + } + if (c == 0) c = cast_int(*pc++); + runtime_check(L, ttistable(ra)); + h = hvalue(ra); + last = ((c-1)*LFIELDS_PER_FLUSH) + n; + if (last > h->sizearray) /* needs more space? */ + luaH_resizearray(L, h, last); /* pre-alloc it at once */ + for (; n > 0; n--) { + TValue *val = ra+n; + setobj2t(L, luaH_setnum(L, h, last--), val); + luaC_barriert(L, h, val); + } + continue; + } + case OP_CLOSE: { + luaF_close(L, ra); + continue; + } + case OP_CLOSURE: { + Proto *p; + Closure *ncl; + int nup, j; + p = cl->p->p[GETARG_Bx(i)]; + nup = p->nups; + ncl = luaF_newLclosure(L, nup, cl->env); + ncl->l.p = p; + for (j=0; jl.upvals[j] = cl->upvals[GETARG_B(*pc)]; + else { + lua_assert(GET_OPCODE(*pc) == OP_MOVE); + ncl->l.upvals[j] = luaF_findupval(L, base + GETARG_B(*pc)); + } + } + setclvalue(L, ra, ncl); + Protect(luaC_checkGC(L)); + continue; + } + case OP_VARARG: { + int b = GETARG_B(i) - 1; + int j; + CallInfo *ci = L->ci; + int n = cast_int(ci->base - ci->func) - cl->p->numparams - 1; + if (b == LUA_MULTRET) { + Protect(luaD_checkstack(L, n)); + ra = RA(i); /* previous call may change the stack */ + b = n; + L->top = ra + n; + } + for (j = 0; j < b; j++) { + if (j < n) { + setobjs2s(L, ra + j, ci->base - n + j); + } + else { + setnilvalue(ra + j); + } + } + continue; + } + } + } +} + diff --git a/src/lua/lvm.h b/src/lua/lvm.h new file mode 100644 index 0000000..bfe4f56 --- /dev/null +++ b/src/lua/lvm.h @@ -0,0 +1,36 @@ +/* +** $Id: lvm.h,v 2.5.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua virtual machine +** See Copyright Notice in lua.h +*/ + +#ifndef lvm_h +#define lvm_h + + +#include "ldo.h" +#include "lobject.h" +#include "ltm.h" + + +#define tostring(L,o) ((ttype(o) == LUA_TSTRING) || (luaV_tostring(L, o))) + +#define tonumber(o,n) (ttype(o) == LUA_TNUMBER || \ + (((o) = luaV_tonumber(o,n)) != NULL)) + +#define equalobj(L,o1,o2) \ + (ttype(o1) == ttype(o2) && luaV_equalval(L, o1, o2)) + + +LUAI_FUNC int luaV_lessthan (lua_State *L, const TValue *l, const TValue *r); +LUAI_FUNC int luaV_equalval (lua_State *L, const TValue *t1, const TValue *t2); +LUAI_FUNC const TValue *luaV_tonumber (const TValue *obj, TValue *n); +LUAI_FUNC int luaV_tostring (lua_State *L, StkId obj); +LUAI_FUNC void luaV_gettable (lua_State *L, const TValue *t, TValue *key, + StkId val); +LUAI_FUNC void luaV_settable (lua_State *L, const TValue *t, TValue *key, + StkId val); +LUAI_FUNC void luaV_execute (lua_State *L, int nexeccalls); +LUAI_FUNC void luaV_concat (lua_State *L, int total, int last); + +#endif diff --git a/src/lua/lzio.c b/src/lua/lzio.c new file mode 100644 index 0000000..293edd5 --- /dev/null +++ b/src/lua/lzio.c @@ -0,0 +1,82 @@ +/* +** $Id: lzio.c,v 1.31.1.1 2007/12/27 13:02:25 roberto Exp $ +** a generic input stream interface +** See Copyright Notice in lua.h +*/ + + +#include + +#define lzio_c +#define LUA_CORE + +#include "lua.h" + +#include "llimits.h" +#include "lmem.h" +#include "lstate.h" +#include "lzio.h" + + +int luaZ_fill (ZIO *z) { + size_t size; + lua_State *L = z->L; + const char *buff; + lua_unlock(L); + buff = z->reader(L, z->data, &size); + lua_lock(L); + if (buff == NULL || size == 0) return EOZ; + z->n = size - 1; + z->p = buff; + return char2int(*(z->p++)); +} + + +int luaZ_lookahead (ZIO *z) { + if (z->n == 0) { + if (luaZ_fill(z) == EOZ) + return EOZ; + else { + z->n++; /* luaZ_fill removed first byte; put back it */ + z->p--; + } + } + return char2int(*z->p); +} + + +void luaZ_init (lua_State *L, ZIO *z, lua_Reader reader, void *data) { + z->L = L; + z->reader = reader; + z->data = data; + z->n = 0; + z->p = NULL; +} + + +/* --------------------------------------------------------------- read --- */ +size_t luaZ_read (ZIO *z, void *b, size_t n) { + while (n) { + size_t m; + if (luaZ_lookahead(z) == EOZ) + return n; /* return number of missing bytes */ + m = (n <= z->n) ? n : z->n; /* min. between n and z->n */ + memcpy(b, z->p, m); + z->n -= m; + z->p += m; + b = (char *)b + m; + n -= m; + } + return 0; +} + +/* ------------------------------------------------------------------------ */ +char *luaZ_openspace (lua_State *L, Mbuffer *buff, size_t n) { + if (n > buff->buffsize) { + if (n < LUA_MINBUFFER) n = LUA_MINBUFFER; + luaZ_resizebuffer(L, buff, n); + } + return buff->buffer; +} + + diff --git a/src/lua/lzio.h b/src/lua/lzio.h new file mode 100644 index 0000000..51d695d --- /dev/null +++ b/src/lua/lzio.h @@ -0,0 +1,67 @@ +/* +** $Id: lzio.h,v 1.21.1.1 2007/12/27 13:02:25 roberto Exp $ +** Buffered streams +** See Copyright Notice in lua.h +*/ + + +#ifndef lzio_h +#define lzio_h + +#include "lua.h" + +#include "lmem.h" + + +#define EOZ (-1) /* end of stream */ + +typedef struct Zio ZIO; + +#define char2int(c) cast(int, cast(unsigned char, (c))) + +#define zgetc(z) (((z)->n--)>0 ? char2int(*(z)->p++) : luaZ_fill(z)) + +typedef struct Mbuffer { + char *buffer; + size_t n; + size_t buffsize; +} Mbuffer; + +#define luaZ_initbuffer(L, buff) ((buff)->buffer = NULL, (buff)->buffsize = 0) + +#define luaZ_buffer(buff) ((buff)->buffer) +#define luaZ_sizebuffer(buff) ((buff)->buffsize) +#define luaZ_bufflen(buff) ((buff)->n) + +#define luaZ_resetbuffer(buff) ((buff)->n = 0) + + +#define luaZ_resizebuffer(L, buff, size) \ + (luaM_reallocvector(L, (buff)->buffer, (buff)->buffsize, size, char), \ + (buff)->buffsize = size) + +#define luaZ_freebuffer(L, buff) luaZ_resizebuffer(L, buff, 0) + + +LUAI_FUNC char *luaZ_openspace (lua_State *L, Mbuffer *buff, size_t n); +LUAI_FUNC void luaZ_init (lua_State *L, ZIO *z, lua_Reader reader, + void *data); +LUAI_FUNC size_t luaZ_read (ZIO* z, void* b, size_t n); /* read next n bytes */ +LUAI_FUNC int luaZ_lookahead (ZIO *z); + + + +/* --------- Private Part ------------------ */ + +struct Zio { + size_t n; /* bytes still unread */ + const char *p; /* current position in buffer */ + lua_Reader reader; + void* data; /* additional data */ + lua_State *L; /* Lua state (for reader) */ +}; + + +LUAI_FUNC int luaZ_fill (ZIO *z); + +#endif diff --git a/src/luasandbox.c b/src/luasandbox.c index 1998bc2..0781e87 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -68,7 +68,6 @@ static void preload_modules(lua_State *lua) } -#ifndef LUA_JIT /** * Implementation of the memory allocator for the Lua state. * @@ -109,7 +108,6 @@ static void* memory_manager(void *ud, void *ptr, size_t osize, size_t nsize) } return nptr; } -#endif static size_t instruction_usage(lsb_lua_sandbox *lsb) @@ -456,11 +454,7 @@ lsb_lua_sandbox* lsb_create(void *parent, return NULL; } -#ifdef LUA_JIT - lsb->lua = luaL_newstate(); -#else lsb->lua = lua_newstate(memory_manager, lsb); -#endif if (logger) { lsb->logger = *logger; } @@ -551,10 +545,8 @@ lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) strcpy(lsb->state_file, state_file); } -#ifndef LUA_JIT size_t mem_limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif // load package module lua_pushcfunction(lsb->lua, luaopen_package); @@ -588,13 +580,7 @@ lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) } else { lua_sethook(lsb->lua, NULL, 0, 0); } -#ifdef LUA_JIT - // todo limit - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - (int)lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); -#else lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = mem_limit; -#endif lua_CFunction pf = lua_atpanic(lsb->lua, unprotected_panic); int jump = setjmp(g_jbuf); if (jump || luaL_dofile(lsb->lua, lsb->lua_file) != 0) { @@ -674,20 +660,6 @@ size_t lsb_usage(lsb_lua_sandbox *lsb, lsb_usage_type utype, if (!lsb || !lsb->lua || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { return 0; } -#ifdef LUA_JIT - if (lsb->lua && utype == LSB_UT_MEMORY) { - switch (ustat) { - case LSB_US_CURRENT: - return lua_gc(lsb->lua, LUA_GCCOUNT, 0) * 1024 - + lua_gc(lsb->lua, LUA_GCCOUNTB, 0); - case LSB_US_MAXIMUM: - return lua_gc(lsb->lua, LUA_GCMAXCOUNT, 0) * 1024 - + lua_gc(lsb->lua, LUA_GCMAXCOUNTB, 0); - default: - break; - } - } -#endif return lsb->usage[utype][ustat]; } diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 63d6d98..7007345 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -412,12 +412,8 @@ lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) data.fh = fh; // Clear the sandbox limits during preservation. -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); -#else // size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif // size_t cur_output_size = lsb->output.size; // size_t max_output_size = lsb->output.maxsize; lsb->output.maxsize = 0; @@ -460,15 +456,10 @@ lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) // destroyed if the user was collecting output between calls. /* // Restore the sandbox limits after preservation -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); -#else lua_gc(lsb->lua, LUA_GCCOLLECT, 0); lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; -#endif lsb->output.maxsize = max_output_size; lsb_clear_output_buffer(lsb->output); if (lsb->output.size > cur_output_size) { @@ -504,12 +495,8 @@ lsb_err_value restore_global_data(lsb_lua_sandbox *lsb) } // Clear the sandbox limits during restoration. -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); -#else size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif lua_sethook(lsb->lua, NULL, 0, 0); int err = luaL_dofile(lsb->lua, lsb->state_file); @@ -525,15 +512,10 @@ lsb_err_value restore_global_data(lsb_lua_sandbox *lsb) return LSB_ERR_LUA; } } -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); // reinstate limit -#else lua_gc(lsb->lua, LUA_GCCOLLECT, 0); lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; -#endif return NULL; } diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index b9176cb..6c922c4 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -6,13 +6,12 @@ add_library(luasandboxtest SHARED sandbox.c) set_target_properties(luasandboxtest PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) target_compile_definitions(luasandboxtest PRIVATE -Dluasandboxtest_EXPORTS) target_link_libraries(luasandboxtest luasandbox) -install(TARGETS luasandboxtest DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) +install(TARGETS luasandboxtest DESTINATION ${CMAKE_INSTALL_LIBDIR}) add_executable(test_generic_sandbox test_generic_sandbox.c) target_link_libraries(test_generic_sandbox luasandboxtest) -set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/ep_base/lib;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/test") +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/test") -add_test(NAME test_move_luasandbox_c_modules COMMAND cmake -E copy_directory ${CMAKE_BINARY_DIR}/ep_base/inst/lib/luasandbox/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_generic_sandbox COMMAND test_generic_sandbox) @@ -20,8 +19,5 @@ add_test(NAME test_generic_sandbox COMMAND test_generic_sandbox) if(WIN32) STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") set_tests_properties(test_generic_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) -elseif(APPLE) - STRING(REPLACE ";" ":" LIBRARY_PATHS "${LIBRARY_PATHS}") - set_tests_properties(test_generic_sandbox PROPERTIES ENVIRONMENT DYLD_LIBRARY_PATH=${LIBRARY_PATHS}) endif() diff --git a/src/test/lua/cjson.lua b/src/test/lua/cjson.lua deleted file mode 100644 index a5232e7..0000000 --- a/src/test/lua/cjson.lua +++ /dev/null @@ -1,31 +0,0 @@ -require "string" -require "table" -local cj = require "cjson" -local js = '["this is a test","this is a test","this is a test","this is a test","this is a test"]' - -function process() - assert(cj == cjson, "cjson not creating a global table") - - local value = cjson.decode("[ true, { \"foo\": \"bar\" } ]") - assert("bar" == value[2].foo, string.format("bar: %s", tostring(value[2].foo))) - - local null_json = '{"test" : 1, "null" : null}' - local value = cjson.decode(null_json) - assert(value.null == nil, "null not discarded") - - cjson.decode_null(true) - value = cjson.decode(null_json) - assert(type(value.null) == "userdata", "null discarded") - - local t = cjson.decode(js) - assert(#t == 5, "could not decode a JSON string bigger than the output buffer") - - local ok, json = pcall(cjson.encode, t) - assert(not ok, "could encode an array bigger than the the output buffer") - - assert(not cjson.new) - - assert(not cjson.encode_keep_buffer) - - return 0 -end diff --git a/src/test/lua/cjson_unlimited.lua b/src/test/lua/cjson_unlimited.lua deleted file mode 100644 index 08750cf..0000000 --- a/src/test/lua/cjson_unlimited.lua +++ /dev/null @@ -1,13 +0,0 @@ -require "cjson" - -function process() - local ls = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" - local t = {} - - for i=1, 1000 do - t[#t+1] = ls - end - write_output(cjson.encode(t)) - - return 0 -end diff --git a/src/test/lua/decoder.lua b/src/test/lua/decoder.lua deleted file mode 100644 index 53be314..0000000 --- a/src/test/lua/decoder.lua +++ /dev/null @@ -1,42 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require("lpeg") -local l = lpeg -l.locale(l) - -local space = l.space^1 - -local timestamp = l.Cg((l.R"09"^1 / "%0000000000"), "Timestamp") - -local severity = l.Cg( - l.Cs(l.P"debug" /"7") - + l.Cs(l.P"info" /"6") - + l.Cs(l.P"notice" /"5") - + l.Cs((l.P"warning" + "warn")/"4") - + l.Cs((l.P"error" + "err") /"3") - + l.Cs(l.P"crit" /"2") - + l.Cs(l.P"alert" /"1") - + l.Cs((l.P"emerg" + "panic") /"0") - , "Severity") - -local key = l.C(l.alpha^1) - -local value = l.C(l.R"!~"^1) - -local pair = space * l.Cg(key * "=" * value) - -local fields = l.Cg(l.Cf(l.Ct("") * pair^0, rawset), "Fields") - -local grammar = l.Ct(timestamp * space * severity * fields) - -function process () - local t = grammar:match("1376389920 debug id=2321 url=example.com item=1") - if t then - write_message(t) - else - return -1 - end - return 0 -end diff --git a/src/test/lua/lpeg.lua b/src/test/lua/lpeg.lua deleted file mode 100644 index d9553ae..0000000 --- a/src/test/lua/lpeg.lua +++ /dev/null @@ -1,18 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "lpeg" - --- csv grammar -local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' + - lpeg.C((1 - lpeg.S',\n"')^0) -local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1) - -function process () - local t = lpeg.match(record, '1,string with spaces,"quoted string, with comma and ""quoted"" text"') - assert(t[1] == "1", t[1]) - assert(t[2] == "string with spaces", t[2]) - assert(t[3] == 'quoted string, with comma and "quoted" text', t[3]) - return 0 -end diff --git a/src/test/lua/lpeg_cbufd.lua b/src/test/lua/lpeg_cbufd.lua deleted file mode 100644 index 7b0670e..0000000 --- a/src/test/lua/lpeg_cbufd.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "lpeg" - -local cbufd = require "lpeg.cbufd" - -function process(tc) - if tc == 0 then - local t = lpeg.match(cbufd.grammar, "header\n1\t2\t3\n2\tnan\t-4\n3\t-4.56\t5.67\n") - if not t then - return error("no match") - end - if t.header ~= "header" then error("header:" .. t.header) end - if t[1].time ~= 1e9 then return error("col 1 timestamp:" .. t[1].time) end - if t[1][1] ~= 2 then return error("col 1 val 1:" .. t[1][1]) end - if t[1][2] ~= 3 then return error("col 1 val 2:" .. t[1][2]) end - - if t[2].time ~= 2e9 then return error("col 2 timestamp:" .. t[2].time) end - if t[2][1] == t[2][1] then error("col 2 val 1:" .. t[2][1]) end - if t[2][2] ~= -4 then return error("col 2 val 2:" .. t[2][2]) end - - if t[3].time ~= 3e9 then error("col 3 timestamp:" .. t[3].time) end - if t[3][1] ~= -4.56 then error("col 3 val 1:" .. t[3][1]) end - if t[3][2] ~= 5.67 then error("col 3 val 2:" .. t[3][2]) end - end - - return 0 -end diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua deleted file mode 100644 index 2ef68e2..0000000 --- a/src/test/lua/lpeg_clf.lua +++ /dev/null @@ -1,509 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local ok, clf = pcall(require,"lpeg.common_log_format") -if not ok then print(clf) end -require "string" - -local combined_log = '127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"' - -local combined_result = { - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - remote_user = "-", - time = 1392050801000000000, - request = "GET / HTTP/1.1", - status = 304, - body_bytes_sent = {value = 0, representation = "B"}, - http_referer = "-", - http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" -} - -local function verify_result(fields, expected) - if not fields then - error("no result", 2) - end - - for k, v in pairs(expected) do - local f = fields[k] - if not f then - error(string.format("key: %s is not found in the result", k), 2) - end - - if type(v) == "table" then - if f.value ~= v.value then - error(string.format("key: %s value: %s received:%s", - k, - tostring(v.value), - tostring(f.value)), - 2) - end - if f.representation ~= v.representation then - error(string.format("key: %s representation: %s received: %s", - k, - tostring(v.representation), - tostring(f.representation)), - 2) - end - else - if f ~= v then - error(string.format("key: %s expected: %s received: %s", - k, - tostring(v), - tostring(f)), - 2) - end - end - end -end - -local function nginx_formats() - local tests = { -- format, input, result - {'"$arg_generic"' ,'"test \\"item\\""' ,'test \\"item\\"'} - ,{"$binary_remote_addr" ,"aaaa" ,"aaaa"} - ,{"$body_bytes_sent" ,"23" ,{23,"B"}} - ,{"$bytes_sent" ,"24" ,{24,"B"}} - ,{"$connection" ,"3" ,3} - ,{"$connection_requests" ,"0" ,0} - ,{"$content_length" ,"123" ,{123,"B"}} - ,{"$host" ,"example.com" ,{"example.com","hostname"}} - ,{"$hostname" ,"::1" ,{"::1","ipv6"}} - ,{"$https" ,"on" ,"on"} - ,{"$https" ,"" ,""} - ,{"$is_args" ,"?" ,"?"} - ,{"$is_args" ,"" ,""} - ,{"$msec" ,"1391794831.755" ,1391794831755000000} - ,{"$nginx_version" ,"1.1.1" ,"1.1.1"} - ,{"$pid" ,"99" ,99} - ,{"$pipe" ,"p" ,"p"} - ,{"$pipe" ,"." ,"."} - ,{"$proxy_protocol_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"$remote_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"$remote_port" ,"5435" ,5435} - ,{"$request_completion" ,"OK" ,"OK"} - ,{"$request_completion" ,"" ,""} - ,{"$request_length" ,"23" ,{23,"B"}} - ,{"$request_method" ,"GET" ,"GET"} - ,{"$request_time" ,"1.123" ,{1.123,"s"}} - ,{"$scheme" ,"http" ,"http"} - ,{"$scheme" ,"https" ,"https"} - ,{"$server_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"$server_name" ,"example.com" ,{"example.com","hostname"}} - ,{"$server_port" ,"5435" ,5435} - ,{"$server_protocol" ,"HTTP/1.0" ,"HTTP/1.0"} - ,{"$status" ,"200" ,200} - ,{"$tcpinfo_rtt" ,"200" ,200} - ,{"$tcpinfo_rttvar" ,"200" ,200} - ,{"$tcpinfo_snd_cwnd" ,"200" ,200} - ,{"$tcpinfo_rcv_space" ,"200" ,200} - ,{"$time_iso8601" ,"2014-02-10T08:46:41-08:00" ,1392050801000000000} - ,{"$time_local" ,"10/Feb/2014:08:46:41 -0800" ,1392050801000000000} - } - - for i, v in ipairs(tests) do - local grammar = clf.build_nginx_grammar(v[1]) - local fields = grammar:match(v[2]) - if not fields then - error(string.format("test: %s failed to match: %s", v[1], v[2])) - end - - local key - if i == 1 then - key = "arg_generic" - else - key = string.sub(v[1], 2) - end - if key == "msec" or key == "time_iso8601" or key == "time_local" then - key = "time" - end - - - if type(v[3]) == "table" then - if fields[key].value ~= v[3][1] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[3][1], fields[key].value)) - end - if fields[key].representation ~= v[3][2] then - error(string.format("test: %s expected representation: %s received: %s", v[1], v[3][2], fields[key].representation)) - end - else - if fields[key] ~= v[3] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[3], fields[key])) - end - end - end -end - -local function nginx_combined() - local grammar = clf.build_nginx_grammar('$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"') - local fields = grammar:match(combined_log) - verify_result(fields, combined_result) -end - -local function nginx_msec() - local grammar = clf.build_nginx_grammar('$msec $remote_addr "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"') - local log = '1391794831.755 127.0.0.1 "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" "-" 0.000' - local fields = grammar:match(log) - local result = { - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - time = 1391794831755000000, - request = "GET / HTTP/1.1", - status = 304, - body_bytes_sent = {value = 0, representation = "B"}, - http_referer = "-", - http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" - } - verify_result(fields, result) -end - -local function user_agent_normalization() - local user_agents = { - "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36" - ,"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" - ,"Mozilla/5.0 (Windows NT 6.1; rv:29.0) Gecko/20100101 Firefox/29.0" - ,"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:29.0) Gecko/20100101 Firefox/29.0" - ,"Mozilla/5.0 (Mobile; rv:29.0) Gecko/20100101 Firefox/29.0" - ,"curl/7.29.0" - ,"Apache-HttpClient/UNAVAILABLE (java 1.5)" - ,"Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405" - ,"Googlebot/2.1 (+http://www.google.com/bot.html)" - ,"" - ,"-" - ,"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; MSIE; )" -- https://bugzilla.mozilla.org/show_bug.cgi?id=1009280 - ,"Firefox/29.0 FxSync/1.31.0.20140327113732.desktop" -- https://github.com/mozilla-services/puppet-config/issues/312 - ,"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25" - ,"Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54" - ,"Opera/12.02 (Android 4.1; Linux; Opera Mobi/ADR-1111101157; U; en-US) Presto/2.9.201 Version/12.02" - ,"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14" - ,"Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10136" - ,"Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko" - ,"Firefox AndroidSync 1.20.0.1.0 (Firefox)" - ,"Firefox-iOS-Sync/1.0 (Firefox)" - } - local results = { - {"Chrome" , 32 , "Windows 8"} - ,{"MSIE" , 10 , "Windows 7"} - ,{"Firefox" , 29 , "Windows 7"} - ,{"Firefox" , 29 , "Macintosh"} - ,{"Firefox" , 29 , "FirefoxOS"} - ,{nil , nil , nil} - ,{nil , nil , nil} - ,{nil , nil , "iPad"} - ,{nil , nil , nil} - ,{nil , nil , nil} - ,{nil , nil , nil} - ,{"MSIE" , nil , "Windows 7"} - ,{"Firefox" , 29 , nil} - ,{"Safari" , 8536 , "iPad"} - ,{"Opera Mini" , 10 , nil} - ,{"Opera Mobi" , 12 , "Android"} - ,{"Opera" , 12 , "Windows Vista"} - ,{"Edge" , 12 , "Windows 10"} - ,{"MSIE" , 11 , "Windows 8.1"} - ,{"FxSync" , 1 , "Android"} - ,{"FxSync" , 1 , "iOS"} - } - - for i, v in ipairs(user_agents) do - local browser, version, os = clf.normalize_user_agent(v) - assert(results[i][1] == browser and results[i][2] == version and results[i][3] == os, - string.format("user_agent: %d browser: %s version: %s os: %s expected browser: %s version: %s os: %s", - i, tostring(browser), tostring(version), tostring(os), - tostring(results[i][1]), tostring(results[i][2]), tostring(results[i][3]))) - end -end - -local function apache_formats() - local tests = { -- format, key, input, result - {"%a", "remote_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"%A", "server_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"%B", "body_bytes_sent" ,"23" ,{23,"B"}} - ,{"%b", "body_bytes_sent" ,"23" ,{23,"B"}} - ,{"%b", "body_bytes_sent" ,"-" ,{0,"B"}} - ,{"%D", "request_time" ,"123" ,{123,"us"}} - ,{"%f", "request_filename" ,"test.txt" ,"test.txt"} - ,{"%h", "remote_addr" ,"example.com" ,{"example.com","hostname"}} - ,{"%H", "server_protocol" ,"HTTP/1.0" ,"HTTP/1.0"} - ,{"%k", "connection_requests" ,"0" ,0} - ,{"%l", "remote_user" ,"waldo" ,"waldo"} - ,{"%L", "request_log_id" ,"???" ,"???"} - ,{"%m", "request_method" ,"GET" ,"GET"} - ,{"%p", "server_port" ,"5435" ,5435} - ,{"%P", "pid" ,"99" ,99} - ,{"%q", "query_string" ,"query" ,"query"} - ,{"%r", "request" ,"request" ,"request"} - ,{"%R", "request_handler" ,"handler" ,"handler"} - ,{"%s", "status" ,"200" ,200} - ,{"%t", "time" ,"[10/Feb/2014:08:46:41 -0800]" ,1392050801000000000} - ,{"%T", "request_time" ,"1" ,{1,"s"}} - ,{"%u", "remote_user" ,"remote" ,"remote"} - ,{"%U", "uri" ,"uri" ,"uri"} - ,{"%v", "server_name" ,"example.com" ,{"example.com","hostname"}} - ,{"%V", "server_name" ,"example.com" ,{"example.com","hostname"}} - ,{'\\"%V\\"', "server_name" ,'"example.com"' ,{"example.com","hostname"}} - ,{"%X", "connection_status" ,"X" ,"X"} - ,{"%X", "connection_status" ,"+" ,"+"} - ,{"%X", "connection_status" ,"-" ,"-"} - ,{"%I", "request_length" ,"23" ,{23,"B"}} - ,{"%O", "response_length" ,"23" ,{23,"B"}} - ,{"%S", "total_length" ,"23" ,{23,"B"}} - - ,{"%>s" ,"status" ,"200" ,200} - ,{"%{c}a" ,"remote_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"%{Test-item}C" ,"cookie_Test-item" ,"item" ,"item"} - ,{"%{Test-item}e" ,"env_Test-item" ,"item" ,"item"} - ,{'"%{User-agent}i"' ,"http_user_agent" ,'"Mozilla/5.0 (comment)"' ,"Mozilla/5.0 (comment)"} - ,{'"%400,501{User-agent}i"' ,"http_user_agent" ,'"Mozilla/5.0 (comment)"' ,"Mozilla/5.0 (comment)"} - ,{'"%!400,501{User-agent}i"',"http_user_agent" ,'"Mozilla/5.0 (comment)"' ,"Mozilla/5.0 (comment)"} - ,{"%{Test-item}n" ,"module_Test-item" ,"item" ,"item"} - ,{"%{Test-item}o" ,"sent_http_test_item" ,"item" ,"item"} - ,{"%{canonical}p" ,"server_port" ,"99" ,99} - ,{"%{local}p" ,"server_port" ,"99" ,99} - ,{"%{remote}p" ,"remote_port" ,"99" ,99} - ,{"%{pid}P" ,"pid" ,"99" ,99} - ,{"%{tid}P" ,"tid" ,"99" ,99} - ,{"%{hextid}P" ,"tid" ,"63" ,99} - ,{"%{sec}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{"%{begin:sec}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{"%{end:sec}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{"%{msec}t" ,"time" ,"1392050801000" ,1392050801000000000} - ,{"%{usec}t" ,"time" ,"1392050801000000" ,1392050801000000000} - ,{"%{msec_frac}t" ,"sec_frac" ,"010" ,0.010} - ,{"%{usec_frac}t" ,"sec_frac" ,"010000" ,0.010} - ,{"%{%d/%b/%Y:%H:%M:%S %z}t","time" ,"10/Feb/2014:08:46:41 -0800" ,1392050801000000000} - ,{"%{%s}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{'"%r"' , "request" ,'"test \\"item\\""' ,'test \\"item\\"'} - ,{'"%{Test-item}o"' ,"sent_http_test_item" ,'"test \\"item\\""' ,'test \\"item\\"'} - ,{"%{SSL_PROTOCOL}x" ,"ssl_protocol" ,"TLSv1.2" ,"TLSv1.2"} - ,{'"%{SSL_CIPHER}x"' ,"ssl_cipher" ,'"ECDHE-RSA-AES256-SHA"' ,"ECDHE-RSA-AES256-SHA"} - } - - if os.date("%c"):find("^%d") then -- windows - tests[55][3] = "10/Feb/2014:08:46:41 PST" - end - - for i, v in ipairs(tests) do - local grammar = clf.build_apache_grammar(v[1]) - local fields = grammar:match(v[3]) - if not fields then - error(string.format("test: %s failed to match: %s", v[1], v[3])) - end - local key = v[2] - - if type(v[4]) == "table" then - if fields[key].value ~= v[4][1] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[4][1], fields[key].value)) - end - if fields[key].representation ~= v[4][2] then - error(string.format("test: %s expected representation: %s received: %s", v[1], v[4][2], fields[key].representation)) - end - else - if fields[key] ~= v[4] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[4], fields[key])) - end - end - end -end - -local function apache_custom() - -- test each of the unique grammars - local grammar = clf.build_apache_grammar('%% %a %B %b %D %f %k %p %P %s %t %T %v %X %I %O %S %A') - local log = '% 127.0.0.1 235 235 204 test.txt 2 80 1234 404 [20/Mar/2014:08:56:26 -0700] 0 example.com + 311 498 809 ::1' - local fields = grammar:match(log) - local result = { - server_addr = {value = "::1", representation = "ipv6"}, - pid = 1234, - total_length = {value = 809, representation = "B"}, - server_port = 80, - request_length = {value = 311, representation = "B"}, - connection_requests = 2, - connection_status = "+", - body_bytes_sent = {value = 235, representation = "B"}, - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - time = 1395330986000000000, - response_length = {value = 498, representation = "B"}, - request_time = {value = 0, representation = "s"}, - request_filename = "test.txt", - status = 404, - server_name = {value = "example.com", representation = "hostname"} - } - verify_result(fields, result) -end - -local function apache_clf() - local grammar = clf.build_apache_grammar('%h %l %u %t \"%r\" %>s %b') - local log = '127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0' - local fields = grammar:match(log) - local result = { - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - remote_user = "-", - body_bytes_sent = {value = 0, representation = "B"}, - time = 1392050801000000000, - request = "GET / HTTP/1.1", - status = 304, - } - verify_result(fields, result) -end - -local function apache_vhost_combined() - local grammar = clf.build_apache_grammar('%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"') - local log = '127.0.1.1:80 127.0.0.1 - - [20/Mar/2014:12:38:34 -0700] "GET / HTTP/1.1" 404 492 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0"' - local fields = grammar:match(log) - local result = { - remote_user = "-", - server_port = 80, - http_referer = "-", - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - time = 1395344314000000000, - response_length = {value = 492, representation = "B"}, - request = "GET / HTTP/1.1", - http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0", - status = 404, - server_name = {value = "127.0.1.1", representation = "ipv4"} - } - verify_result(fields, result) -end - -local function apache_combined() - local grammar = clf.build_apache_grammar('%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"') - local fields = grammar:match(combined_log) - verify_result(fields, combined_result) -end - -local function apache_referer() - local grammar = clf.build_apache_grammar('%{Referer}i -> %U') - local log = '- -> /' - local fields = grammar:match(log) - local result = { - uri = "/", - http_referer = "-" - } - verify_result(fields, result) -end - -local function nginx_error() - local log = '2014/03/01 11:29:39 [notice] 16842#0: using inherited sockets from "6;"' - local fields = clf.nginx_error_grammar:match(log) - local result = { - Pid = 16842, - Payload = 'using inherited sockets from "6;"', - Severity = 5, - Timestamp = 1393673379000000000 - } - verify_result(fields, result) - assert(fields.Fields.tid == 0, "expected a thread id of 0") -end - -local function nginx_error_connection() - -- optional connection - local log = '2014/03/01 11:29:39 [notice] 16842#0: *8878 using inherited sockets from "6;"' - local fields = clf.nginx_error_grammar:match(log) - local result = { - Pid = 16842, - Payload = 'using inherited sockets from "6;"', - Severity = 5, - Timestamp = 1393673379000000000 - } - verify_result(fields, result) - assert(fields.Fields.tid == 0, "invalid tid") - assert(fields.Fields.connection == 8878, "invalid connection") -end - -local function nginx_upstream() - local addrs = {"192.168.1.1:80", "192.168.1.2:80", "unix:/tmp/sock", "192.168.10.1:80", "192.168.10.2:80"} - local lengths = {1, 2, 3, 4, 5} - local times = {1.1, 1.2, 1.3, 1.4, "-"} - local statuses = {200, 201, 202, 203, "-"} - local uscs = "HIT" - local header = "header field" - - local grammar = clf.build_nginx_grammar('$upstream_addr $upstream_cache_status $upstream_response_length $upstream_response_time $upstream_status "$upstream_http_test" $upstream_cache_last_modified') - local log = string.format('%s, %s, %s : %s, %s %s %d, %d, %d : %d, %d %g, %g, %g : %g, %s %d, %d, %d : %d, %s "%s" Mon, 28 Sep 1970 06:00:00 GMT', - addrs[1], addrs[2], addrs[3], addrs[4], addrs[5], uscs, - lengths[1], lengths[2], lengths[3], lengths[4], lengths[5], - times[1], times[2], times[3], times[4], times[5], - statuses[1], statuses[2], statuses[3], statuses[4], statuses[5], header) - local fields = grammar:match(log) - if not fields then error(string.format("failed match: %s", log)) end - - if #fields.upstream_addr ~= #addrs then - error(string.format("#upstream_addr = %d", #fields.upstream_addr)) - end - for i, v in ipairs(addrs) do - if fields.upstream_addr[i] ~= v then - error(string.format("expected value: '%s' received: '%s'", v, fields.upstream_addr[i])) - end - end - - if #fields.upstream_response_length.value ~= #lengths then - error(string.format("#upstream_response_length = %d", #fields.upstream_response_length.value)) - end - if fields.upstream_response_length.representation ~= "B" then - error(string.format("upstream_response_length representation = '%s'", fields.upstream_response_length.representation)) - end - for i, v in ipairs(lengths) do - if fields.upstream_response_length.value[i] ~= v then - error(string.format("expected value: %d received: %d", v, fields.upstream_response_length.value[i])) - end - end - - if #fields.upstream_response_time.value ~= #times then - error(string.format("#upstream_response_time = %d", #fields.upstream_response_time.value)) - end - if fields.upstream_response_time.representation ~= "s" then - error(string.format("upstream_response_time representation = '%s'", fields.upstream_response_time.representation)) - end - for i, v in ipairs(times) do - if v == "-" then v = 0 end - if fields.upstream_response_time.value[i] ~= v then - error(string.format("expected value: %g received: %g", v, fields.upstream_response_time.value[i])) - end - end - - if #fields.upstream_status ~= #statuses then - error(string.format("#upstream_status = %d", #fields.upstream_status)) - end - for i, v in ipairs(statuses) do - if v == "-" then v = 0 end - if fields.upstream_status[i] ~= v then - error(string.format("expected value: %d received: %d", v, fields.upstream_status[i])) - end - end - - if fields.upstream_cache_status ~= uscs then - error(string.format("expected value: '%s' received: '%s'", uscs, fields.upstream_cache_status)) - end - if fields.upstream_http_test ~= header then - error(string.format("expected value: '%s' received: '%s'", header, fields.upstream_http_test)) - end - local usclm = 2.33496e16 - if fields.upstream_cache_last_modified ~= usclm then - error(string.format("expected value: '%s' received: '%s'", usclm, fields.upstream_cache_last_modified)) - end -end - -local function nginx_upstream_defaults() - local grammar = clf.build_nginx_grammar('$upstream_addr $upstream_cache_status $upstream_response_length $upstream_response_time $upstream_status "$upstream_http_test" $upstream_cache_last_modified') - local log = '- - - - - "-" -' - local fields = grammar:match(log) - if not fields then error(string.format("failed match: %s", log)) end -end - - -function process() - nginx_formats() - nginx_combined() - nginx_msec() - user_agent_normalization() - apache_formats() - apache_custom() - apache_clf() - apache_vhost_combined() - apache_combined() - apache_referer() - nginx_error() - nginx_error_connection() - nginx_upstream() - nginx_upstream_defaults() - - return 0 -end diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua deleted file mode 100644 index 7721706..0000000 --- a/src/test/lua/lpeg_date_time.lua +++ /dev/null @@ -1,149 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "string" - -local dt = require "lpeg.date_time" - -local function test_valid(grammar, tests, results) - for i,v in ipairs(tests) do - t = grammar:match(v) - if not t then - error(v) - else - ns = dt.time_to_ns(t) - if ns ~= results[i] then - error(string.format("test: %d %s expected: %g received: %g", i, v, results[i], ns), 2) - end - end - end -end - -local function rfc3339() - local tests = {"1999-05-05T23:23:59.217-07:00", - "1985-04-12T23:20:50.52Z", - "1996-12-19T16:39:57-08:00", - "1990-12-31T23:59:60Z", - "1990-12-31T15:59:60-08:00", - "1937-01-01T12:00:27.87+00:20"} - local results = {925971839217000064, - 482196050520000000, - 851042397000000000, - 662688000000000000, - 662688000000000000, - -1041337172130000000} - if os.date("%c"):find("^%d") then - results[6] = 0 -- windows will fail to convert this time - end - test_valid(dt.rfc3339, tests, results) -end - -local function rfc3339_invalid() - local tests = {"1985-04-12t23:20:50.52Z", - "1985-04-12T23:20:50.52z", - "1985-04-12", - "1999-05-05T23:23:59.217-0700"} - for i,v in ipairs(tests) do - if dt.rfc3339:match(v) then - error(v) - end - end -end - -local function clf() - local tests = {"10/Feb/2014:08:46:36 -0800"} - local results = {1392050796000000000} - test_valid(dt.clf_timestamp, tests, results) -end - -local function rfc3164() - local tests = {"Feb 10 16:46:36"} - local results = {os.time{year = os.date("%Y"), month = 2, day = 10, hour = 16, min = 46, sec = 36} * 1e9} - test_valid(dt.rfc3164_timestamp, tests, results) -end - -local function mysql() - local tests = {"20140210164636"} - local results = {1392050796000000000} - test_valid(dt.mysql_timestamp, tests, results) -end - -local function pgsql() - local tests = {"2014-02-10 16:46:36"} - local results = {1392050796000000000} - test_valid(dt.pgsql_timestamp, tests, results) -end - -local function strftime_all() - local formats - if os.date("%c"):find("^%d") then -- windows C89 support only - formats = {"%a", "%A", "%b", "%B","%c", "%d", - "%H", "%I", "%j", "%m", "%M", "%p", - "%S", "%U", "%w", "%W", "%x", - "%X", "%y", "%Y", "%z", "%Z", "%%", "test string"} - else - formats = {"%a", "%A", "%b", "%B","%c","%C", "%d", "%D", "%e", - "%F", "%g", "%G", "%h", "%H", "%I", "%j", "%k", "%l", "%m", - "%M", "%n", "%p", "%r", "%R", "%s", "%S", "%t", "%T", "%u", - "%U", "%V", "%w", "%W", "%x","%X", "%y", "%Y", "%z", "%Z", - "%%", "test string"} - end - - for i,v in ipairs(formats) do - local test = os.date(v) - local g = dt.build_strftime_grammar(v) - if not g:match(test) then - error(string.format("failed parsing: %s %s", v, test)) - end - end -end - -local function strftime_composite() - local formats = {"%c", "%D %T", "%D %r", "%d/%b/%Y:%H:%M:%S %z"} - local inputs = {"Mon Feb 10 16:46:36 2014", "02/10/14 16:46:36", "02/10/14 04:46:36 PM", "10/Feb/2014:16:46:36 +0000"} - if os.date("%c"):find("^%d") then -- windows %c is non-standard - inputs[1] = inputs[2] - end - local result = 1392050796000000000 - for i,v in ipairs(formats) do - local g = dt.build_strftime_grammar(v) - local t = g:match(inputs[i]) - if not t then - error(string.format("failed parsing: %s %s", v, inputs[i])) - end - if result ~= dt.time_to_ns(t) then - error(string.format("time conversion failed %s", inputs[i])) - end - end -end - -local function strftime_secfrac() - local g = dt.build_strftime_grammar("%d/%b/%Y:%H:%M:%S.%f") - local t = g:match("10/Feb/2014:16:46:36.124") - assert(t) - local ns = dt.time_to_ns(t) - assert(1392050796124000000 == ns, string.format("%d", ns)) -end - -local function strftime_invalid() - local r, g = pcall(dt.build_strftime_grammar, "%E") - if r then - error("allowed to build a grammar with an invalid specifier") - end -end - -function process() - rfc3339() - rfc3339_invalid() - clf() - rfc3164() - mysql() - pgsql() - strftime_all() - strftime_composite() - strftime_invalid() - strftime_secfrac() - - return 0 -end diff --git a/src/test/lua/lpeg_ip_address.lua b/src/test/lua/lpeg_ip_address.lua deleted file mode 100644 index e38a21b..0000000 --- a/src/test/lua/lpeg_ip_address.lua +++ /dev/null @@ -1,170 +0,0 @@ - --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "lpeg" - -local ip = require "lpeg.ip_address" - -local function ipv6() - local tests = {"2001:0db8:85a3:0000:0000:8a2e:0370:7334", - "2001:db8:85a3:0:0:8a2e:370:7334", - "2001:db8:85a3::8a2e:370:7334", - "2001:0db8:0000:0000:0000:0000:1428:57ab", - "2001:0db8:0000:0000:0000::1428:57ab", - "2001:0db8:0:0:0:0:1428:57ab", - "2001:0db8:0:0::1428:57ab", - "2001:0db8::1428:57ab", - "2001:db8::1428:57ab", - "0000:0000:0000:0000:0000:0000:0000:0001", - "::1", - "::ffff:12.34.56.78", - "::ffff:0c22:384e", - "2001:0db8:1234:0000:0000:0000:0000:0000", - "2001:0db8:1234:ffff:ffff:ffff:ffff:ffff", - "2001:db8:a::123", - "fe80::", - "::ffff:192.0.2.128", - "::ffff:c000:280", - "::"} - for i,v in ipairs(tests) do - if not ip.v6:match(v) then - error(v) - end - end -end - -local function ipv6_invalid() - local tests = { - "123", - "ldkfj", - "2001::FFD3::57ab", - "2001:db8:85a3::8a2e:37023:7334", - "2001:db8:85a3::8a2e:370k:7334", - "1:2:3:4:5:6:7:8:9", - "1::2::3", - "1:::3:4:5", - "1:2:3::4:5:6:7:8:9", - "::ffff:2.3.4", - "::ffff:257.1.2.3", - "1.2.3.4"} - local grammar = ip.v6 * lpeg.P(-1) -- make it a full match - for i,v in ipairs(tests) do - if grammar:match(v) then - error(v) - end - end -end - -local function ipv4() - local tests = { - "0.0.0.0", - "192.168.1.10", - "127.0.0.1", - "249.255.255.255", - "255.255.255.255", - "1.2.3.4"} - for i,v in ipairs(tests) do - if not ip.v4:match(v) then - error(v) - end - end -end - -local function ipv4_invalid() - local grammar = ip.v4 * lpeg.P(-1) -- make it a full match - local tests = { - "00.0.0.0", - "256.255.255.255", - "255.2550.255.255", - "1", - "1.2", - "1.2.3.", - "1.2.3.a", - "1.2.3.4.5", - "."} - for i,v in ipairs(tests) do - if grammar:match(v) then - error(v) - end - end -end - -local function hostname() - local tests = { - "localhost", - "example.com", - "127.org", - "example%20", - "UPPERCASE.COM", - "aA1-._~!$&'()*+,;=.com"} - for i,v in ipairs(tests) do - if not ip.hostname:match(v) then - error(v) - end - end -end - -local function hostname_invalid() - local grammar = ip.hostname * lpeg.P(-1) -- make it a full match - local tests = { - "example com", - "foo%AG", - "at@", - "colon:", - "lbracket[", - "rbracket]", - "hash#", - "quest?", - "slash/", - "lbrace{", - "rbrace}", - ""} - for i,v in ipairs(tests) do - if grammar:match(v) then - error(v) - end - end -end - -local function host() - local tests = { - "localhost", - "127.0.0.1", - "::1"} - for i,v in ipairs(tests) do - if not ip.host:match(v) then - error(v) - end - end -end - -local function host_field() - local tests = { - {"localhost", "hostname"}, - {"127.0.0.1", "ipv4"}, - {"::1", "ipv6"} - } - for i,v in ipairs(tests) do - local field = ip.host_field:match(v[1]) - if field then - assert(field.value == tests[i][1], field.value) - assert(field.representation == tests[i][2], field.representation) - else - error(v) - end - end -end - -function process(tc) - ipv6() - ipv6_invalid() - ipv4() - ipv4_invalid() - hostname() - hostname_invalid() - host() - host_field() - return 0 -end diff --git a/src/test/lua/lpeg_mysql.lua b/src/test/lua/lpeg_mysql.lua deleted file mode 100644 index 0b41ebb..0000000 --- a/src/test/lua/lpeg_mysql.lua +++ /dev/null @@ -1,289 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "string" -local mysql = require "lpeg.mysql" - -local slow_query_log = { -[[ -# Time: 140507 18:14:18 -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -use widget; -SET last_insert_id=999,insert_id=1000,timestamp=1399500744; -# administrator command: do something -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -SET last_insert_id=999,timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Time: 140507 18:14:18 -# User@Host: sync.rw[sync.rw] @ db01.example.com [127.0.0.1] -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Time: 140507 18:14:18 -# User@Host: sync_rw[sync_rw] @ db-01.example.com [127.0.0.1] -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]] -} - -local mariadb_slow_query_log = { -[[ -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Thread_id: 110804 Schema: weave0 QC_hit: No -# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Thread_id: 110804 Schema: weave0 QC_hit: No -# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Time: 140623 16:43:22 -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Thread_id: 110804 Schema: weave0 QC_hit: No -# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 -# Full_scan: Yes Full_join: Yes Tmp_table: Yes Tmp_table_on_disk: No -# Filesort: Yes Filesort_on_disk: No Merge_passes: 3 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]] -} - -local percona_slow_query_log = { -[[ -# User@Host: syncrw[syncrw] @ db-01.example.com [127.0.0.1] Id: 42 -# Schema: imdb Last_errno: 0 Killed: 0 -# Query_time: 7.725616 Lock_time: 0.000328 Rows_sent: 4 Rows_examined: 1543720 Rows_affected: 0 -# Bytes_sent: 272 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 -# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No -# Filesort: No Filesort_on_disk: No Merge_passes: 0 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Time: 130601 8:01:06 -# User@Host: syncrw[syncrw] @ db-01.example.com [127.0.0.1] Id: 42 -# Schema: imdb Last_errno: 0 Killed: 0 -# Query_time: 7.725616 Lock_time: 0.000328 Rows_sent: 4 Rows_examined: 1543720 Rows_affected: 0 Rows_read: 30 -# Bytes_sent: 272 Tmp_tables: 0 Tmp_disk_tables: 0 Tmp_table_sizes: 0 -# QC_Hit: No Full_scan: Yes Full_join: No Tmp_table: No Tmp_table_on_disk: No -# Filesort: No Filesort_on_disk: No Merge_passes: 0 -# InnoDB_IO_r_ops: 6415 InnoDB_IO_r_bytes: 105103360 InnoDB_IO_r_wait: 0.001279 -# InnoDB_rec_lock_wait: 0.000000 InnoDB_queue_wait: 0.000000 -# InnoDB_pages_distinct: 6430 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]] -} - -local fields = { - {"Query_time", 2.964652, "s"}, - {"Rows_examined", 9773}, - {"Lock_time", 0.000050, "s"}, - {"Rows_sent", 251} -} - -local mariadb_fields = { - {"Query_time", 1.178108, "s"}, - {"Rows_examined", 198}, - {"Lock_time", 0.000053, "s"}, - {"Rows_sent", 198}, - {"QC_hit", "No"}, - {"Thread_id", 110804}, - {"Schema", "weave0"} -} - -local mariadb_verbose_fields = { - {"Query_time", 1.178108, "s"}, - {"Rows_examined", 198}, - {"Lock_time", 0.000053, "s"}, - {"Rows_sent", 198}, - {"QC_hit", "No"}, - {"Thread_id", 110804}, - {"Schema", "weave0"}, - {"Full_scan", "Yes"}, - {"Full_join", "Yes"}, - {"Filesort", "Yes"}, - {"Filesort_on_disk", "No"}, - {"Tmp_table_on_disk", "No"}, - {"Tmp_table", "Yes"}, - {"Merge_passes", 3} -} - -local percona_fields = { - {"Schema", "imdb"}, - {"Last_errno", 0}, - {"Killed", 0}, - {"Query_time", 7.725616, "s"}, - {"Lock_time", 0.000328, "s"}, - {"Rows_sent", 4}, - {"Rows_examined", 1543720}, - {"Rows_affected", 0}, - {"Bytes_sent", 272, "B"}, - {"Tmp_tables", 0}, - {"Tmp_disk_tables", 0}, - {"Tmp_table_sizes", 0, "B"}, - {"QC_hit", "No"}, - {"Full_scan", "Yes"}, - {"Full_join", "No"}, - {"Tmp_table", "No"}, - {"Tmp_table_on_disk", "No"}, - {"Filesort", "No"}, - {"Filesort_on_disk", "No"}, - {"Merge_passes", 0} -} - -local percona_verbose_fields = { - {"Schema", "imdb"}, - {"Last_errno", 0}, - {"Killed", 0}, - {"Query_time", 7.725616, "s"}, - {"Lock_time", 0.000328, "s"}, - {"Rows_sent", 4}, - {"Rows_examined", 1543720}, - {"Rows_affected", 0}, - {"Rows_read", 30}, - {"Bytes_sent", 272, "B"}, - {"Tmp_tables", 0}, - {"Tmp_disk_tables", 0}, - {"Tmp_table_sizes", 0, "B"}, - {"QC_hit", "No"}, - {"Full_scan", "Yes"}, - {"Full_join", "No"}, - {"Tmp_table", "No"}, - {"Tmp_table_on_disk", "No"}, - {"Filesort", "No"}, - {"Filesort_on_disk", "No"}, - {"Merge_passes", 0}, - {"InnoDB_IO_r_ops", 6415}, - {"InnoDB_IO_r_bytes", 105103360, "B"}, - {"InnoDB_IO_r_wait", 0.001279, "s"}, - {"InnoDB_rec_lock_wait", 0, "s"}, - {"InnoDB_queue_wait", 0, "s"}, - {"InnoDB_pages_distinct", 6430} -} - -local function validate(fields, t) - if t.Timestamp ~= "1399500744000000000" then return error("Timestamp:" .. t.Timestamp) end - if t.Payload ~= "/* [queryName=FIND_ITEMS] */ SELECT *\nFROM widget\nWHERE id = 10;\n" then return error("Payload:" .. t.Payload) end - for i, v in ipairs(fields) do - if #v == 3 then - if t.Fields[v[1]].value ~= v[2] or t.Fields[v[1]].representation ~= v[3] then - if type(v[2]) == "string" then - return error(string.format("field:%s: %s (%s)", v[1], t.Fields[v[1]].value, t.Fields[v[1]].representation), 2) - else - return error(string.format("field:%s: %G (%s)", v[1], t.Fields[v[1]].value, t.Fields[v[1]].representation), 2) - end - end - else - print("Output:" .. v[1]) - if t.Fields[v[1]] ~= v[2] then return error(string.format("field:%s:", v[1]) .. t.Fields[v[1]], 2) end - end - end -end - -local function header() - local t = mysql.slow_query_grammar:match(slow_query_log[1]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end - validate(fields, t) -end - -local function standard() - local t = mysql.slow_query_grammar:match(slow_query_log[2]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end - validate(fields, t) -end - -local function short() - local t = mysql.short_slow_query_grammar:match(slow_query_log[3]) - if not t then return error("no match") end - validate(fields, t) -end - -local function mariadb_standard() - local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[1]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end - validate(mariadb_fields, t) -end - -local function mariadb_short() - local t = mysql.mariadb_short_slow_query_grammar:match(mariadb_slow_query_log[2]) - if not t then return error("no match") end - validate(mariadb_fields, t) -end - -local function mariadb_verbose() - local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[3]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end - validate(mariadb_verbose_fields, t) -end - -local function percona_standard() - local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[1]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end - validate(percona_fields, t) -end - -local function percona_verbose() - local t = mysql.percona_slow_query_grammar:match(percona_slow_query_log[2]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname: " .. t.Hostname) end - validate(percona_verbose_fields, t) -end - -function process(tc) - header() - standard() - short() - mariadb_standard() - mariadb_short() - mariadb_verbose() - percona_standard() - percona_verbose() - - return 0 -end diff --git a/src/test/lua/lpeg_postfix.lua b/src/test/lua/lpeg_postfix.lua deleted file mode 100644 index 0d3a561..0000000 --- a/src/test/lua/lpeg_postfix.lua +++ /dev/null @@ -1,1100 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require 'string' -require 'table' -local pf = require 'lpeg.postfix' - --- From https://github.com/whyscream/postfix-grok-patterns --- 2015-05-26 8fdd562d159845b97bf8d9c273e5357f3a143a94 --- Original under BSD-3-clause license -local tests = { - ["anvil_0001.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max cache size 1 at Oct 26 18:13:50", - { - postfix_anvil_cache_size = 1, - postfix_anvil_timestamp = "Oct 26 18:13:50" - } - }, - ["anvil_0002.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max connection count 1 for (smtpd:61.238.241.86) at Oct 26 18:13:50", - { - postfix_anvil_conn_count = 1, - postfix_anvil_timestamp = "Oct 26 18:13:50", - postfix_client_ip = "61.238.241.86", - postfix_service = "smtpd" - } - }, - ["anvil_0003.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max connection rate 1/60s for (smtpd:61.238.241.86) at Oct 26 18:13:50", - { - postfix_anvil_conn_period = "60s", - postfix_anvil_conn_rate = 1, - postfix_anvil_timestamp = "Oct 26 18:13:50", - postfix_client_ip = "61.238.241.86", - postfix_service = "smtpd" - } - }, - ["anvil_0004.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max connection count 1 for (smtpd:2604:8d00:0:1::3) at Oct 26 17:46:59", - { - postfix_anvil_conn_count = 1, - postfix_anvil_timestamp = "Oct 26 17:46:59", - postfix_client_ip = "2604:8d00:0:1::3", - postfix_service = "smtpd" - } - }, - ["anvil_0005.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max connection rate 1/60s for (smtpd:2604:8d00:0:1::3) at Oct 26 17:46:59", - { - postfix_anvil_conn_period = "60s", - postfix_anvil_conn_rate = 1, - postfix_anvil_timestamp = "Oct 26 17:46:59", - postfix_client_ip = "2604:8d00:0:1::3", - postfix_service = "smtpd" - } - }, - ["anvil_0006.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max connection count 1 for (127.0.0.1:2525:127.0.0.1) at Oct 26 19:46:00", - { - postfix_anvil_conn_count = 1, - postfix_anvil_timestamp = "Oct 26 19:46:00", - postfix_client_ip = "127.0.0.1", - postfix_service = "127.0.0.1:2525" - } - }, - ["anvil_0007.yaml"] = { - "POSTFIX_ANVIL", - "statistics: max connection rate 1/60s for (127.0.0.1:2525:127.0.0.1) at Oct 26 18:13:50", - { - postfix_anvil_conn_period = "60s", - postfix_anvil_conn_rate = 1, - postfix_anvil_timestamp = "Oct 26 18:13:50", - postfix_client_ip = "127.0.0.1", - postfix_service = "127.0.0.1:2525" - } - }, - ["bounce_0001.yaml"] = { - "POSTFIX_BOUNCE", - "17CCA8044: sender non-delivery notification: ADD2F8052", - { - postfix_bounce_queueid = "ADD2F8052", - postfix_queueid = "17CCA8044" - } - }, - ["bounce_0002.yaml"] = { - "POSTFIX_BOUNCE", - "65AFB8B5: sender delivery status notification: 1DF35917", - { - postfix_bounce_queueid = "1DF35917", - postfix_queueid = "65AFB8B5" - } - }, - ["bounce_0003.yaml"] = { - "POSTFIX_BOUNCE", - "264FE1A18: sender delay notification: 0A87A1A08", - { - postfix_bounce_queueid = "0A87A1A08", - postfix_queueid = "264FE1A18" - } - }, - ["cleanup_0001.yaml"] = { - "POSTFIX_CLEANUP", - "0F5383D: message-id=<544D2523.30609@rhsoft.net>", - { - postfix_keyvalue_data = "message-id=<544D2523.30609@rhsoft.net>", - postfix_queueid = "0F5383D" - } - }, - ["cleanup_0002.yaml"] = { - "POSTFIX_CLEANUP", - "4104E3D: message-id=<>", - { - postfix_keyvalue_data = "message-id=<>", - postfix_queueid = "4104E3D" - } - }, - ["cleanup_0003.yaml"] = { - "POSTFIX_CLEANUP", - "4104E3D: milter-header-redirect: header X-Spam-Status: Yes, score=39.3 required=5.0 tests=ADVANCE_FE from 061238241086.static.ctinets.com[61.238.241.86]; from= to= proto=ESMTP helo=: tom@example.com", - { - postfix_keyvalue_data = "from= to= proto=ESMTP helo=", - postfix_milter_data = "tom@example.com", - postfix_milter_message = "header X-Spam-Status: Yes, score=39.3 required=5.0 tests=ADVANCE_FE from 061238241086.static.ctinets.com[61.238.241.86]", - postfix_milter_result = "header-redirect", - postfix_queueid = "4104E3D" - } - }, - ["cleanup_0004.yaml"] = { - "POSTFIX_CLEANUP", - "4104E3D: milter-reject: END-OF-MESSAGE from 061238241086.static.ctinets.com[61.238.241.86]: 5.7.1 Blocked by SpamAssassin; from= to= proto=ESMTP helo=", - { - postfix_keyvalue_data = "from= to= proto=ESMTP helo=", - postfix_milter_message = "END-OF-MESSAGE from 061238241086.static.ctinets.com[61.238.241.86]: 5.7.1 Blocked by SpamAssassin", - postfix_milter_result = "reject", - postfix_queueid = "4104E3D" - } - }, - ["cleanup_0005.yaml"] = { - "POSTFIX_CLEANUP", - "warning: dict_ldap_get_values[1]: DN uid=mguiraud,ou=people,dc=neotion,dc=com not found, skipping ", - { - postfix_warning = "dict_ldap_get_values[1]: DN uid=mguiraud,ou=people,dc=neotion,dc=com not found, skipping " - } - }, - ["delays_0001.yaml"] = { - "POSTFIX_DELAYS", - "0.2/0.02/0.04/3.3", - { - postfix_delay_before_qmgr = 0.20000000000000001, - postfix_delay_conn_setup = 0.040000000000000001, - postfix_delay_in_qmgr = 0.02, - postfix_delay_transmission = 3.2999999999999998 - } - }, - ["discard_0001.yaml"] = { - "POSTFIX_DISCARD", - "25F2E5E061: to=, relay=none, delay=0.05, delays=0.05/0/0/0, dsn=2.0.0, status=sent (test.example.com)", - { - postfix_keyvalue_data = "to=, relay=none, delay=0.05, delays=0.05/0/0/0, dsn=2.0.0,", - postfix_queueid = "25F2E5E061", - postfix_status = "sent" - } - }, - ["dnsblog_0001.yaml"] = { - "POSTFIX_DNSBLOG", - "addr 61.238.241.86 listed by domain psbl.surriel.com as 127.0.0.2", - { - postfix_client_ip = "61.238.241.86", - postfix_dnsbl_domain = "psbl.surriel.com", - postfix_dnsbl_result = "127.0.0.2" - } - }, - ["dnsblog_0002.yaml"] = { - "POSTFIX_DNSBLOG", - "addr 2607:f8b0:4003:c01::23a listed by domain list.dnswl.org as 127.0.5.1", - { - postfix_client_ip = "2607:f8b0:4003:c01::23a", - postfix_dnsbl_domain = "list.dnswl.org", - postfix_dnsbl_result = "127.0.5.1" - } - }, - ["local_0001.yaml"] = { - "POSTFIX_LOCAL", - "2A22C263F6: to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0, status=sent (delivered to command: procmail -a \"$EXTENSION\")", - { - postfix_keyvalue_data = "to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0, status=sent (delivered to command: procmail -a \"$EXTENSION\")", - postfix_queueid = "2A22C263F6" - } - }, - ["local_0002.yaml"] = { - "POSTFIX_LOCAL", - "892A0205B6: to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1, status=bounced (unknown user: \"ghdsgfhdslfh\")", - { - postfix_keyvalue_data = "to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1, status=bounced (unknown user: \"ghdsgfhdslfh\")", - postfix_queueid = "892A0205B6" - } - }, - ["master_0001.yaml"] = { - "POSTFIX_MASTER", - "daemon started -- version 2.11.0, configuration /etc/postfix", - { - postfix_config_path = "/etc/postfix", - postfix_version = "2.11.0" - } - }, - ["master_0002.yaml"] = { - "POSTFIX_MASTER", - "terminating on signal 15", - { - postfix_termination_signal = 15 - } - }, - ["master_0003.yaml"] = { - "POSTFIX_MASTER", - "reload -- version 2.11.0, configuration /etc/postfix", - { - postfix_config_path = "/etc/postfix", - postfix_version = "2.11.0" - } - }, - ["pickup_0001.yaml"] = { - "POSTFIX_PICKUP", - "D2FAE20586: uid=0 from=", - { - postfix_queueid = "D2FAE20586", - -- from postfix_keyvalue_data = "uid=0 from=", - postfix_uid = 0, - postfix_from = 'fail2ban' - } - }, - ["pipe_0001.yaml"] = { - "POSTFIX_PIPE", - "0F5383D: to=, relay=dovecot, delay=4.3, delays=4.1/0.01/0/0.15, dsn=2.0.0, status=sent (delivered via dovecot service)", - { - postfix_keyvalue_data = "to=, relay=dovecot, delay=4.3, delays=4.1/0.01/0/0.15, dsn=2.0.0, status=sent", - postfix_pipe_service = "dovecot", - postfix_queueid = "0F5383D" - } - }, - ["pipe_0002.yaml"] = { - "POSTFIX_PIPE", - "153053D: to=, orig_to=, relay=dovecot, delay=3.4, delays=3.3/0.03/0/0.12, dsn=2.0.0, status=sent (delivered via dovecot service)", - { - postfix_keyvalue_data = "to=, orig_to=, relay=dovecot, delay=3.4, delays=3.3/0.03/0/0.12, dsn=2.0.0, status=sent", - postfix_pipe_service = "dovecot", - postfix_queueid = "153053D" - } - }, - ["pipe_0003.yaml"] = { - "POSTFIX_PIPE", - "95ECE24E0: to=, relay=dovecot, delay=0.12, delays=0.03/0/0/0.08, dsn=5.4.6, status=bounced (mail forwarding loop for tom@example.com)", - { - postfix_keyvalue_data = "to=, relay=dovecot, delay=0.12, delays=0.03/0/0/0.08, dsn=5.4.6, status=bounced", - postfix_queueid = "95ECE24E0", - postfix_to = "tom@example.com" - } - }, - ["postdrop_0001.yaml"] = { - "POSTFIX_POSTDROP", - "warning: uid=0: File too large", - { - postfix_warning = "uid=0: File too large" - } - }, - ["postscreen_0001.yaml"] = { - "POSTFIX_POSTSCREEN", - "cache btree:/var/lib/postfix-in/postscreen_cache full cleanup: retained=242 dropped=7 entries", - { - postfix_postscreen_cache_dropped = 7, - postfix_postscreen_cache_retained = 242 - } - }, - ["postscreen_0002.yaml"] = { - "POSTFIX_POSTSCREEN", - "CONNECT from [61.238.241.86]:53024 to [89.105.204.244]:25", - { - postfix_client_ip = "61.238.241.86", - postfix_client_port = 53024, - postfix_server_ip = "89.105.204.244", - postfix_server_port = 25 - } - }, - ["postscreen_0003.yaml"] = { - "POSTFIX_POSTSCREEN", - "DISCONNECT [216.81.72.72]:43421", - { - postfix_client_ip = "216.81.72.72", - postfix_client_port = 43421 - } - }, - ["postscreen_0004.yaml"] = { - "POSTFIX_POSTSCREEN", - "PASS OLD [61.238.241.86]:53024", - { - postfix_client_ip = "61.238.241.86", - postfix_client_port = 53024, - postfix_postscreen_access = "PASS OLD" - } - }, - ["postscreen_0005.yaml"] = { - "POSTFIX_POSTSCREEN", - "PASS OLD [2604:8d00:0:1::3]:52237", - { - postfix_client_ip = "2604:8d00:0:1::3", - postfix_client_port = 52237, - postfix_postscreen_access = "PASS OLD" - } - }, - ["postscreen_0006.yaml"] = { - "POSTFIX_POSTSCREEN", - "PASS NEW [2607:f8b0:4003:c01::23a]:36051", - { - postfix_client_ip = "2607:f8b0:4003:c01::23a", - postfix_client_port = 36051, - postfix_postscreen_access = "PASS NEW" - } - }, - ["postscreen_0007.yaml"] = { - "POSTFIX_POSTSCREEN", - "WHITELISTED [61.238.241.86]:53024", - { - postfix_client_ip = "61.238.241.86", - postfix_client_port = 53024, - postfix_postscreen_access = "WHITELISTED" - } - }, - ["postscreen_0008.yaml"] = { - "POSTFIX_POSTSCREEN", - "BLACKLISTED [61.238.241.86]:53024", - { - postfix_client_ip = "61.238.241.86", - postfix_client_port = 53024, - postfix_postscreen_access = "BLACKLISTED" - } - }, - ["postscreen_0009.yaml"] = { - "POSTFIX_POSTSCREEN", - "WHITELIST VETO [61.238.241.86]:53024", - { - postfix_client_ip = "61.238.241.86", - postfix_client_port = 53024, - postfix_postscreen_access = "WHITELIST VETO" - } - }, - ["postscreen_0010.yaml"] = { - "POSTFIX_POSTSCREEN", - "PREGREET 6 after 1.2 from [216.81.72.72]:43421: RSET\r\n", - { - postfix_client_ip = "216.81.72.72", - postfix_client_port = 43421, - postfix_postscreen_violation = "PREGREET", - postfix_postscreen_violation_time = 1.2 - } - }, - ["postscreen_0011.yaml"] = { - "POSTFIX_POSTSCREEN", - "DNSBL rank 3 for [111.73.45.149]:1038", - { - postfix_client_ip = "111.73.45.149", - postfix_client_port = 1038, - postfix_postscreen_dnsbl_rank = 3, - postfix_postscreen_violation = "DNSBL" - } - }, - ["postscreen_0012.yaml"] = { - "POSTFIX_POSTSCREEN", - "HANGUP after 63 from [111.73.45.149]:1038 in tests after SMTP handshake", - { - postfix_client_ip = "111.73.45.149", - postfix_client_port = 1038, - postfix_postscreen_violation = "HANGUP", - postfix_postscreen_violation_time = 63 - } - }, - ["postscreen_0013.yaml"] = { - "POSTFIX_POSTSCREEN", - "HANGUP after 0 from [66.55.85.58]:54017 in tests before SMTP handshake", - { - postfix_client_ip = "66.55.85.58", - postfix_client_port = 54017, - postfix_postscreen_violation = "HANGUP", - postfix_postscreen_violation_time = 0 - } - }, - ["postscreen_0014.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND TIME LIMIT from [182.246.250.243]:51799 after CONNECT", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51799, - postfix_postscreen_violation = "COMMAND TIME LIMIT", - postfix_smtp_stage = "CONNECT" - } - }, - ["postscreen_0015.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND TIME LIMIT from [182.246.250.243]:51799 after HELO", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51799, - postfix_postscreen_violation = "COMMAND TIME LIMIT", - postfix_smtp_stage = "HELO" - } - }, - ["postscreen_0016.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND TIME LIMIT from [182.246.250.243]:51799 after EHLO", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51799, - postfix_postscreen_violation = "COMMAND TIME LIMIT", - postfix_smtp_stage = "EHLO" - } - }, - ["postscreen_0017.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND TIME LIMIT from [182.246.250.243]:51799 after AUTH", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51799, - postfix_postscreen_violation = "COMMAND TIME LIMIT", - postfix_smtp_stage = "AUTH" - } - }, - ["postscreen_0018.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND TIME LIMIT from [182.246.250.243]:51799 after MAIL", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51799, - postfix_postscreen_violation = "COMMAND TIME LIMIT", - postfix_smtp_stage = "MAIL" - } - }, - ["postscreen_0019.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND TIME LIMIT from [182.246.250.243]:51796 after RCPT", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51796, - postfix_postscreen_violation = "COMMAND TIME LIMIT", - postfix_smtp_stage = "RCPT" - } - }, - ["postscreen_0020.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND COUNT LIMIT from [182.246.250.243]:51796 after RCPT", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51796, - postfix_postscreen_violation = "COMMAND COUNT LIMIT", - postfix_smtp_stage = "RCPT" - } - }, - ["postscreen_0021.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND LENGTH LIMIT from [182.246.250.243]:51796 after RCPT", - { - postfix_client_ip = "182.246.250.243", - postfix_client_port = 51796, - postfix_postscreen_violation = "COMMAND LENGTH LIMIT", - postfix_smtp_stage = "RCPT" - } - }, - ["postscreen_0022.yaml"] = { - "POSTFIX_POSTSCREEN", - "NON-SMTP COMMAND from [93.174.93.51]:45284 after CONNECT: GET http://ipv4scan.com/hello/check.txt HTTP/1.1", - { - postfix_client_ip = "93.174.93.51", - postfix_client_port = 45284, - postfix_postscreen_violation = "NON-SMTP COMMAND", - postfix_smtp_stage = "CONNECT" - } - }, - ["postscreen_0023.yaml"] = { - "POSTFIX_POSTSCREEN", - "NOQUEUE: reject: CONNECT from [1.2.3.4]:1337: too many connections", - { - postfix_client_ip = "1.2.3.4", - postfix_client_port = 1337, - postfix_postscreen_toobusy_data = "too many connections" - } - }, - ["postscreen_0024.yaml"] = { - "POSTFIX_POSTSCREEN", - "NOQUEUE: reject: CONNECT from [5.6.7.8]:1337: all server ports busy", - { - postfix_client_ip = "5.6.7.8", - postfix_client_port = 1337, - postfix_postscreen_toobusy_data = "all server ports busy" - } - }, - ["postscreen_0025.yaml"] = { - "POSTFIX_POSTSCREEN", - "COMMAND PIPELINING from [88.208.233.4]:42606 after .: \n", - { - postfix_client_ip = "88.208.233.4", - postfix_client_port = 42606, - postfix_postscreen_violation = "COMMAND PIPELINING", - postfix_smtp_stage = "." - } - }, - ["postscreen_0026.yaml"] = { - "POSTFIX_POSTSCREEN", - "BARE NEWLINE from [88.208.233.4]:42606 after .", - { - postfix_client_ip = "88.208.233.4", - postfix_client_port = 42606, - postfix_postscreen_violation = "BARE NEWLINE", - postfix_smtp_stage = "." - } - }, - ["postscreen_0027.yaml"] = { - "POSTFIX_POSTSCREEN", - "NOQUEUE: reject: RCPT from [182.98.255.184]:2413: 550 5.5.1 Protocol error; from=, to=, proto=SMTP, helo=", - { - postfix_action = "reject", - postfix_client_ip = "182.98.255.184", - postfix_client_port = 2413, - postfix_keyvalue_data = "from=, to=, proto=SMTP, helo=", - postfix_smtp_stage = "RCPT", - postfix_status_code = 550, - postfix_status_code_enhanced = "5.5.1", - postfix_status_message = "Protocol error" - } - }, - ["postscreen_0028.yaml"] = { - "POSTFIX_POSTSCREEN", - "NOQUEUE: reject: RCPT from [27.157.200.233]:4984: 550 5.7.1 Service unavailable; client [27.157.200.233] blocked using zen.spamhaus.org; from=, to=<4ECEA00F.9040306@example.com>, proto=ESMTP, helo=", - { - postfix_action = "reject", - postfix_client_ip = "27.157.200.233", - postfix_client_port = 4984, - postfix_keyvalue_data = "from=, to=<4ECEA00F.9040306@example.com>, proto=ESMTP, helo=", - postfix_smtp_stage = "RCPT", - postfix_status_code = 550, - postfix_status_code_enhanced = "5.7.1", - postfix_status_data = "27.157.200.233", - postfix_status_message = "blocked using zen.spamhaus.org" - } - }, - ["postscreen_0029.yaml"] = { - "POSTFIX_POSTSCREEN", - "warning: getpeername: Transport endpoint is not connected -- dropping this connection", - { - postfix_warning = "getpeername: Transport endpoint is not connected -- dropping this connection" - } - }, - ["qmgr_0001.yaml"] = { - "POSTFIX_QMGR", - "0F5383D: removed", - { - postfix_queueid = "0F5383D" - } - }, - ["qmgr_0002.yaml"] = { - "POSTFIX_QMGR", - "0F5383D: from=, size=5360, nrcpt=1 (queue active)", - { - postfix_keyvalue_data = "from=, size=5360, nrcpt=1", - postfix_queueid = "0F5383D" - } - }, - ["qmgr_0003.yaml"] = { - "POSTFIX_QMGR", - "warning: bounce_queue_lifetime is larger than maximal_queue_lifetime - adjusting bounce_queue_lifetime", - { - postfix_warning = "bounce_queue_lifetime is larger than maximal_queue_lifetime - adjusting bounce_queue_lifetime" - } - }, - ["relay_info_0001.yaml"] = { - "POSTFIX_RELAY_INFO", - "1.example.com.si[private/dovecot-lmtp]", - { - postfix_relay_hostname = "1.example.com.si", - postfix_relay_service = "private/dovecot-lmtp" - } - }, - ["scache_0001.yaml"] = { - "POSTFIX_SCACHE", - "statistics: domain lookup hits=0 miss=1 success=0%", - { - postfix_scache_hits = 0, - postfix_scache_miss = 1, - postfix_scache_success = 0 - } - }, - ["scache_0002.yaml"] = { - "POSTFIX_SCACHE", - "statistics: start interval Dec 6 21:20:35", - { - postfix_scache_timestamp = "Dec 6 21:20:35" - } - }, - ["scache_0003.yaml"] = { - "POSTFIX_SCACHE", - "statistics: address lookup hits=0 miss=1 success=0%", - { - postfix_scache_hits = 0, - postfix_scache_miss = 1, - postfix_scache_success = 0 - } - }, - ["scache_0004.yaml"] = { - "POSTFIX_SCACHE", - "statistics: max simultaneous domains=1 addresses=1 connection=1", - { - postfix_scache_addresses = 1, - postfix_scache_connection = 1, - postfix_scache_domains = 1 - } - }, - ["sendmail_0001.yaml"] = { - "POSTFIX_SENDMAIL", - "fatal: root(0): message file too big", - { - postfix_warning = "root(0): message file too big" - } - }, - ["smtp_0001.yaml"] = { - "POSTFIX_SMTP", - "7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)", - { - postfix_keyvalue_data = "to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0,", - postfix_queueid = "7EE668039", - postfix_smtp_response = "250 2.0.0 Ok: queued as 153053D", - postfix_status = "sent" - } - }, - ["smtp_0002.yaml"] = { - "POSTFIX_SMTP", - "0D9633D: to=, orig_to=, relay=mail1.example.net[10.74.103.11]:25, delay=11, delays=5.4/0.06/5.1/0.03, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as F12DF84004)", - { - postfix_keyvalue_data = "to=, orig_to=, relay=mail1.example.net[10.74.103.11]:25, delay=11, delays=5.4/0.06/5.1/0.03, dsn=2.0.0,", - postfix_queueid = "0D9633D", - postfix_smtp_response = "250 2.0.0 Ok: queued as F12DF84004", - postfix_status = "sent" - } - }, - ["smtp_0003.yaml"] = { - "POSTFIX_SMTP", - "Untrusted TLS connection established to mx4.hotmail.com[65.55.92.136]:25: TLSv1.2 with cipher ECDHE-RSA-AES256-SHA384 (256/256 bits)", - { - postfix_relay_hostname = "mx4.hotmail.com", - postfix_relay_ip = "65.55.92.136", - postfix_relay_port = 25, - postfix_tls_cipher = "ECDHE-RSA-AES256-SHA384", - postfix_tls_cipher_size = "256/256", - postfix_tls_version = "TLSv1.2" - } - }, - ["smtp_0004.yaml"] = { - "POSTFIX_SMTP", - "Untrusted TLS connection established to 127.0.0.1[127.0.0.1]:2525: TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)", - { - postfix_relay_hostname = "127.0.0.1", - postfix_relay_ip = "127.0.0.1", - postfix_relay_port = 2525, - postfix_tls_cipher = "AECDH-AES256-SHA", - postfix_tls_cipher_size = "256/256", - postfix_tls_version = "TLSv1.2" - } - }, - ["smtp_0005.yaml"] = { - "POSTFIX_SMTP", - "connect to intern.nl[194.109.6.98]:25: Connection timed out", - { - postfix_relay_hostname = "intern.nl", - postfix_relay_ip = "194.109.6.98", - postfix_relay_port = 25 - } - }, - ["smtp_0006.yaml"] = { - "POSTFIX_SMTP", - "B99FE3D: lost connection with mx3.hotmail.com[65.55.37.72] while receiving the initial server greeting", - { - postfix_queueid = "B99FE3D", - postfix_relay_hostname = "mx3.hotmail.com", - postfix_relay_ip = "65.55.37.72" - } - }, - ["smtp_0007.yaml"] = { - "POSTFIX_SMTP", - "connect to mail2.crabruzzo.it[2.115.207.21]:25: No route to host", - { - postfix_relay_hostname = "mail2.crabruzzo.it", - postfix_relay_ip = "2.115.207.21", - postfix_relay_port = 25 - } - }, - ["smtp_0015.yaml"] = { - "POSTFIX_SMTP", - "Trusted TLS connection established to gmail-smtp-in.l.google.com[74.125.136.26]:25: TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)", - { - postfix_relay_hostname = "gmail-smtp-in.l.google.com", - postfix_relay_ip = "74.125.136.26", - postfix_relay_port = 25, - postfix_tls_cipher = "ECDHE-RSA-AES128-GCM-SHA256", - postfix_tls_cipher_size = "128/128", - postfix_tls_version = "TLSv1.2" - } - }, - ["smtp_0016.yaml"] = { - "POSTFIX_SMTP", - "Verified TLS connection established to mail.sys4.de[2001:1578:400:111::7]:25: TLSv1 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)", - { - postfix_relay_hostname = "mail.sys4.de", - postfix_relay_ip = "2001:1578:400:111::7", - postfix_relay_port = 25, - postfix_tls_cipher = "ECDHE-RSA-AES256-SHA", - postfix_tls_cipher_size = "256/256", - postfix_tls_version = "TLSv1" - } - }, - ["smtp_0017.yaml"] = { - "POSTFIX_SMTP", - "AD6E41CDC05C: to=, relay=sub1.example.com[private/dovecot-lmtp], delay=0.15, delays=0.04/0/0.02/0.08, dsn=2.0.0, status=sent (250 2.0.0 qYyyBMsl9FRuZQAAA15QOA Saved)", - { - postfix_keyvalue_data = "to=, relay=sub1.example.com[private/dovecot-lmtp], delay=0.15, delays=0.04/0/0.02/0.08, dsn=2.0.0,", - postfix_queueid = "AD6E41CDC05C", - postfix_smtp_response = "250 2.0.0 qYyyBMsl9FRuZQAAA15QOA Saved", - postfix_status = "sent" - } - }, - ["smtp_0018.yaml"] = { - "POSTFIX_SMTP", - "4E74613FE76: to=, relay=gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b]:25, delay=1.9, delays=0.05/0/1.1/0.71, dsn=5.7.1, status=bounced (host gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b] said: 550-5.7.1 [2607:4100:3:25:100::2 12] Our system has detected that this 550-5.7.1 message is likely unsolicited mail. To reduce the amount of spam sent 550-5.7.1 to Gmail, this message has been blocked. Please visit 550-5.7.1 http://support.google.com/mail/bin/answer.py?hl=en&answer=188131 for 550 5.7.1 more information. qc12si8475009pab.211 - gsmtp (in reply to end of DATA command))", - { - postfix_keyvalue_data = "to=, relay=gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b]:25, delay=1.9, delays=0.05/0/1.1/0.71, dsn=5.7.1,", - postfix_queueid = "4E74613FE76", - postfix_smtp_response = "host gmail-smtp-in.l.google.com[2607:f8b0:400e:c04::1b] said: 550-5.7.1 [2607:4100:3:25:100::2 12] Our system has detected that this 550-5.7.1 message is likely unsolicited mail. To reduce the amount of spam sent 550-5.7.1 to Gmail, this message has been blocked. Please visit 550-5.7.1 http://support.google.com/mail/bin/answer.py?hl=en&answer=188131 for 550 5.7.1 more information. qc12si8475009pab.211 - gsmtp (in reply to end of DATA command)", - postfix_status = "bounced" - } - }, - ["smtp_0019.yaml"] = { - "POSTFIX_SMTP", - "warning: problem talking to service private/scache: Connection timed out", - { - postfix_warning = "problem talking to service private/scache: Connection timed out" - } - }, - ["smtpd_0001.yaml"] = { - "POSTFIX_SMTPD", - "connect from 061238241086.static.ctinets.com[61.238.241.86]", - { - postfix_client_hostname = "061238241086.static.ctinets.com", - postfix_client_ip = "61.238.241.86" - } - }, - ["smtpd_0002.yaml"] = { - "POSTFIX_SMTPD", - "disconnect from 061238241086.static.ctinets.com[61.238.241.86]", - { - postfix_client_hostname = "061238241086.static.ctinets.com", - postfix_client_ip = "61.238.241.86" - } - }, - ["smtpd_0003.yaml"] = { - "POSTFIX_SMTPD", - "4104E3D: client=061238241086.static.ctinets.com[61.238.241.86]", - { - postfix_keyvalue_data = "client=061238241086.static.ctinets.com[61.238.241.86]", - postfix_queueid = "4104E3D" - } - }, - ["smtpd_0004.yaml"] = { - "POSTFIX_SMTPD", - "NOQUEUE: reject: RCPT from 061238241086.static.ctinets.com[61.238.241.86]: 550 5.1.1 : Recipient address rejected: User unknown in virtual mailbox table; from= to= proto=ESMTP helo=", - { - postfix_action = "reject", - postfix_client_hostname = "061238241086.static.ctinets.com", - postfix_client_ip = "61.238.241.86", - postfix_keyvalue_data = "from= to= proto=ESMTP helo=", - postfix_smtp_stage = "RCPT", - postfix_status_code = 550, - postfix_status_code_enhanced = "5.1.1", - postfix_status_data = "toernooicommissie@example.com", - postfix_status_message = "Recipient address rejected: User unknown in virtual mailbox table" - } - }, - ["smtpd_0005.yaml"] = { - "POSTFIX_SMTPD", - "lost connection after STARTTLS from unknown[72.13.58.7]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtp_stage = "STARTTLS", - postfix_smtpd_lostconn_data = "lost connection" - } - }, - ["smtpd_0006.yaml"] = { - "POSTFIX_SMTPD", - "warning: hostname exemple.com does not resolve to address 185.14.29.32", - { - postfix_warning = "hostname exemple.com does not resolve to address 185.14.29.32" - } - }, - ["smtpd_0007.yaml"] = { - "POSTFIX_SMTPD", - "timeout after CONNECT from unknown[177.227.18.3]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "177.227.18.3", - postfix_smtp_stage = "CONNECT", - postfix_smtpd_lostconn_data = "timeout" - } - }, - ["smtpd_0008.yaml"] = { - "POSTFIX_POSTSCREEN", - "NOQUEUE: reject: RCPT from unknown[2001:980:cfb1:1:82f:f74e:a45c:3033]: 504 5.5.2 : Sender address rejected: need fully-qualified address; from= to= proto=SMTP helo=", - { - postfix_action = "reject", - postfix_client_hostname = "unknown", - postfix_client_ip = "2001:980:cfb1:1:82f:f74e:a45c:3033", - postfix_keyvalue_data = "from= to= proto=SMTP helo=", - postfix_smtp_stage = "RCPT", - postfix_status_code = 504, - postfix_status_code_enhanced = "5.5.2", - postfix_status_data = "aap@henk", - postfix_status_message = "Sender address rejected: need fully-qualified address" - } - }, - ["smtpd_0009.yaml"] = { - "POSTFIX_SMTPD", - "NOQUEUE: reject: RCPT from news.zihan-promo.com[192.36.205.58]: 554 5.7.1 Service unavailable; Helo command [news.zihan-promo.com] blocked using dbl.spamhaus.org; http://www.spamhaus.org/query/dbl?domain=zihan-promo.com; from= to= proto=ESMTP helo=", - { - postfix_action = "reject", - postfix_client_hostname = "news.zihan-promo.com", - postfix_client_ip = "192.36.205.58", - postfix_keyvalue_data = "from= to= proto=ESMTP helo=", - postfix_smtp_stage = "RCPT", - postfix_status_code = 554, - postfix_status_code_enhanced = "5.7.1", - postfix_status_data = "news.zihan-promo.com", - postfix_status_message = "blocked using dbl.spamhaus.org; http://www.spamhaus.org/query/dbl?domain=zihan-promo.com" - } - }, - ["smtpd_0010.yaml"] = { - "POSTFIX_SMTPD", - "Anonymous TLS connection established from julie.example.com[10.163.89.202]: TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)", - { - postfix_client_hostname = "julie.example.com", - postfix_client_ip = "10.163.89.202", - postfix_tls_cipher = "AECDH-AES256-SHA", - postfix_tls_cipher_size = "256/256", - postfix_tls_version = "TLSv1.2" - } - }, - ["smtpd_0011.yaml"] = { - "POSTFIX_SMTPD", - "lost connection after DATA (7774180 bytes) from unknown[72.13.58.7]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtp_stage = "DATA", - postfix_smtpd_lostconn_data = "lost connection" - } - }, - ["smtpd_0012.yaml"] = { - "POSTFIX_SMTPD", - "lost connection after UNKNOWN from unknown[2001:456:cfb1:1:f5d7:dead:beef:cafe]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "2001:456:cfb1:1:f5d7:dead:beef:cafe", - postfix_smtp_stage = "UNKNOWN", - postfix_smtpd_lostconn_data = "lost connection" - } - }, - ["smtpd_0013.yaml"] = { - "POSTFIX_SMTPD", - "lost connection after RSET from unknown[72.13.58.7]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtp_stage = "RSET", - postfix_smtpd_lostconn_data = "lost connection" - } - }, - ["smtpd_0014.yaml"] = { - "POSTFIX_SMTPD", - "timeout after END-OF-MESSAGE from unknown[72.13.58.7]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtp_stage = "END-OF-MESSAGE", - postfix_smtpd_lostconn_data = "timeout" - } - }, - ["smtpd_0015.yaml"] = { - "POSTFIX_SMTPD", - "improper command pipelining after EHLO from unknown[2001:968:9999:20:415c:cd2:da8e:d0cf]: MAIL FROM:<>\nRCPT TO:\nDATA\nSubject: pipeline test\n\nThis is a test\n.\nQUIT\n\r\n", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "2001:968:9999:20:415c:cd2:da8e:d0cf", - postfix_smtp_stage = "EHLO" - } - }, - ["smtpd_0016.yaml"] = { - "POSTFIX_SMTPD", - "lost connection after DATA (0 bytes) from unknown[72.13.58.7]", - { - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtp_stage = "DATA", - postfix_smtpd_lostconn_data = "lost connection" - } - }, - ["smtpd_0017.yaml"] = { - "POSTFIX_SMTPD", - "NOQUEUE: reject: VRFY from unknown[2001:968:9999:20:88b:9b7d:2a54:2bd2]: 454 4.7.1 : Relay access denied; to= proto=SMTP helo=", - { - postfix_action = "reject", - postfix_client_hostname = "unknown", - postfix_client_ip = "2001:968:9999:20:88b:9b7d:2a54:2bd2", - postfix_keyvalue_data = "to= proto=SMTP helo=", - postfix_smtp_stage = "VRFY", - postfix_status_code = 454, - postfix_status_code_enhanced = "4.7.1", - postfix_status_data = "aap@henk.com", - postfix_status_message = "Relay access denied" - } - }, - ["smtpd_0018.yaml"] = { - "POSTFIX_SMTPD", - "NOQUEUE: reject: VRFY from unknown[2001:968:9999:20:88b:9b7d:2a54:2bd2]: 550 5.1.1 : Recipient address rejected: User unknown in virtual mailbox table; to= proto=SMTP helo=", - { - postfix_action = "reject", - postfix_client_hostname = "unknown", - postfix_client_ip = "2001:968:9999:20:88b:9b7d:2a54:2bd2", - postfix_keyvalue_data = "to= proto=SMTP helo=", - postfix_smtp_stage = "VRFY", - postfix_status_code = 550, - postfix_status_code_enhanced = "5.1.1", - postfix_status_data = "does-not-exist@example.com", - postfix_status_message = "Recipient address rejected: User unknown in virtual mailbox table" - } - }, - ["smtpd_0019.yaml"] = { - "POSTFIX_SMTPD", - "SSL_accept error from unknown[72.13.58.7]: lost connection", - { - postfix_action = "SSL_accept error", - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtpd_lostconn_data = "lost connection" - } - }, - ["smtpd_0020.yaml"] = { - "POSTFIX_SMTPD", - "proxy-accept: END-OF-MESSAGE: 250 2.0.0 from MTA(smtp:[127.0.0.1]:10013): 250 2.0.0 Ok: queued as DF66E940333; from= to= proto=ESMTP helo=<[127.0.0.1]>", - { - postfix_keyvalue_data = "from= to= proto=ESMTP helo=<[127.0.0.1]>", - -- postfix_proxy_message = "250 2.0.0 from MTA(smtp:[127.0.0.1]:10013): 250 2.0.0 Ok: queued as DF66E940333", -- FIXME - postfix_proxy_result = "accept", - postfix_proxy_smtp_stage = "END-OF-MESSAGE", - postfix_proxy_status_code = 250, - postfix_proxy_status_code_enhanced = "2.0.0" - } - }, - ["smtpd_0021.yaml"] = { - "POSTFIX_SMTPD", - "proxy-reject: END-OF-MESSAGE: ; from= to= proto=ESMTP helo=<[192.168.0.16]>", - { - postfix_keyvalue_data = "from= to= proto=ESMTP helo=<[192.168.0.16]>", - -- postfix_proxy_message = "", -- FIXME - postfix_proxy_result = "reject", - postfix_proxy_smtp_stage = "END-OF-MESSAGE" - } - }, - ["smtpd_0022.yaml"] = { - "POSTFIX_SMTPD", - "proxy-reject: END-OF-MESSAGE: 554 5.7.0 Reject, contact postmaster@example.com, id=31619-02 - spam; from=<> to= proto=SMTP helo=<17.33.12.53>", - { - postfix_keyvalue_data = "from=<> to= proto=SMTP helo=<17.33.12.53>", - -- postfix_proxy_message = "554 5.7.0 Reject, contact postmaster@example.com, id=31619-02 - spam", -- FIXME - postfix_proxy_result = "reject", - postfix_proxy_smtp_stage = "END-OF-MESSAGE", - postfix_proxy_status_code = 554, - postfix_proxy_status_code_enhanced = "5.7.0" - } - }, - ["smtpd_0023.yaml"] = { - "POSTFIX_SMTPD", - "SSL_accept error from unknown[72.13.58.7]: Connection timed out", - { - postfix_action = "SSL_accept error", - postfix_client_hostname = "unknown", - postfix_client_ip = "72.13.58.7", - postfix_smtpd_lostconn_data = "Connection timed out" - } - }, - ["tlsmgr_0001.yaml"] = { - "POSTFIX_TLSMGR", - "warning: request to update table btree:/var/spool/postfix/smtpd_scache in non-postfix directory /var/spool/postfix", - { - postfix_warning = "request to update table btree:/var/spool/postfix/smtpd_scache in non-postfix directory /var/spool/postfix" - } - }, - ["tlsproxy_0001.yaml"] = { - "POSTFIX_TLSPROXY", - "DISCONNECT [82.118.17.142]:51637", - { - postfix_client_ip = "82.118.17.142", - postfix_client_port = 51637 - } - }, - ["tlsproxy_0002.yaml"] = { - "POSTFIX_TLSPROXY", - "CONNECT from [82.118.17.142]:51637", - { - postfix_client_ip = "82.118.17.142", - postfix_client_port = 51637 - } - }, - ["trivial_rewrite_0001.yaml"] = { - "POSTFIX_TRIVIAL_REWRITE", - "warning: virtual_alias_domains lookup failure", - { - postfix_warning = "virtual_alias_domains lookup failure" - } - }, - ["trivial_rewrite_0002.yaml"] = { - "POSTFIX_TRIVIAL_REWRITE", - "warning: dict_ldap_lookup: Search error -1: Can't contact LDAP server ", - { - postfix_warning = "dict_ldap_lookup: Search error -1: Can't contact LDAP server " - } - }, - -- testsuite additions, not in postfix-grok-patterns - ["keyvalue_data_0001.yaml"] = { - "POSTFIX_KEYVALUE_DATA", - "from=<>, to=, relay=dovecot, delay=4.3, delays=4.1/0.01/0/0.15, dsn=2.0.0, status=sent", - { - from = '', - to = 'tom@example.com', - relay = 'dovecot', - delay = '4.3', - delays = '4.1/0.01/0/0.15', - dsn = '2.0.0', - status='sent' - } - } -} - -function process_one(filename, programmname, message, expect, extract_keyvalue_data) - local ret = pf.postfix_match(programmname, message, extract_keyvalue_data) - if not ret then - error(string.format("%s: failed match", filename)) - elseif type(ret) ~= 'table' then - error(string.format("%s: incorrect type %s: %s", filename, type(ret), ret)) - else - for k,v in pairs(expect) do - if not ret[k] then - error(string.format("%s: missing key '%s'", filename, k)) - else - if type(ret[k]) == 'table' then - if ret[k]['value'] ~= v then - error(string.format("%s: data mismatch for '%s': '%s' != '%s'", filename, k, ret[k]['value'], v)) - end - elseif ret[k] ~= v then - error(string.format("%s: data mismatch for '%s': '%s' != '%s'", filename, k, ret[k], v)) - end - end - end - end -end - - -function process() - for filename, test in pairs(tests) do - local pattern = test[1] - local data = test[2] - local results = test[3] - local programmname = string.gsub(string.lower(pattern), '_', '/', 1) - process_one(filename, programmname, data, results) - end - -- test with extract_keyvalue_data - local message = '25F2E5E061: to=, relay=none, delay=0.05, delays=0.05/0/0/0, dsn=2.0.0, status=sent (test.example.com)' - local expect = { - postfix_queueid = "25F2E5E061", - postfix_to = 'george.desantis@test.example.com', - postfix_relay = 'none', - postfix_delay = 0.05, - postfix_delay_before_qmgr = 0.05, - postfix_delay_in_qmgr = 0, - postfix_delay_conn_setup = 0, - postfix_delay_transmission = 0, - postfix_dsn = '2.0.0', - postfix_status = 'sent' - } - process_one("discard_0001.yaml+extract_keyvalue_data", 'mta-in/discard', message, expect, true) - return 0 -end diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua deleted file mode 100644 index 0fcc219..0000000 --- a/src/test/lua/lpeg_syslog.lua +++ /dev/null @@ -1,143 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local syslog = require "lpeg.syslog" - -local function traditional_file_format() - local grammar = syslog.build_rsyslog_grammar('%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n') - local log = 'Feb 9 14:17:01 trink-x230 CRON[20758]: (root) CMD ( cd / && run-parts --report /etc/cron.hourly)\n' - local fields = grammar:match(log) - assert(fields.msg == "(root) CMD ( cd / && run-parts --report /etc/cron.hourly)", fields.msg) - local ts = os.time{year = os.date("%Y"), month = 2, day = 9, hour = 14, min = 17, sec = 01} * 1e9 - assert(fields.timestamp == ts, fields.timestamp) - assert(fields.syslogtag.pid == 20758, fields.syslogtag.pid) - assert(fields.syslogtag.programname == "CRON", fields.syslogtag.programname) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function file_format() - local grammar = syslog.build_rsyslog_grammar('%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n') - local log = '2014-02-10T08:21:59.111537-08:00 trink-x230 kernel: Kernel logging (proc) stopped.\n' - local fields = grammar:match(log) - assert(fields.msg == "Kernel logging (proc) stopped.", fields.msg) - assert(fields.timestamp == 1392049319111537000, fields.timestamp) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function forward_format() - local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%') - local log = '<6>2014-02-10T08:33:15.407935-08:00 trink-x230 kernel: imklog 5.8.6, log source = /proc/kmsg started.' - local fields = grammar:match(log) - assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) - assert(fields.timestamp == 1392049995407935000, fields.timestamp) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 0, fields.pri.facility) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function traditional_forward_format() - local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%') - local log = '<6>Feb 10 08:38:47 trink-x230 kernel: imklog 5.8.6, log source = /proc/kmsg started.' - local fields = grammar:match(log) - assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) - local ts = os.time{year = os.date("%Y"), month = 2, day = 10, hour = 8, min = 38, sec = 47} * 1e9 - assert(fields.timestamp == ts, fields.timestamp) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 0, fields.pri.facility) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function kitchen_sink() - local grammar = syslog.build_rsyslog_grammar("%HOSTNAME% %SOURCE% %FROMHOST% %FROMHOST-IP% %SYSLOGTAG% %PROGRAMNAME% %PRI% %PRI-TEXT% %IUT% %SYSLOGFACILITY% %SYSLOGFACILITY-TEXT% %SYSLOGPRIORITY% %SYSLOGPRIORITY-TEXT% %TIMEGENERATED% %TIMEREPORTED:::date-mysql% %TIMESTAMP:::date-rfc3339% %PROTOCOL-VERSION% %STRUCTURED-DATA% %APP-NAME% %PROCID% %MSGID% %$NOW% %$YEAR% %$MONTH% %$DAY% %$HOUR% %$HHOUR% %$QHOUR% %$MINUTE% %MSG%\n") - local log = 'trink-x230 trink-x230 trink-x230 127.0.0.1 kernel: kernel 6 kern.info<6> 1 0 kern 6 info Feb 10 09:20:53 20140210092053 2014-02-10T09:20:53.559934-08:00 0 - kernel - 2014-02-10 2014 02 10 09 00 01 20 imklog 5.8.6, log source = /proc/kmsg started.\n' - local fields = grammar:match(log) - assert(fields.hostname == "trink-x230", fields.hostname) - assert(fields.source == "trink-x230", fields.source) - assert(fields.fromhost == "trink-x230", fields.fromhost) - assert(fields["fromhost-ip"] == "127.0.0.1", fields["fromhost-ip"]) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.programname == "kernel", fields.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 0, fields.pri.facility) - assert(fields["pri-text"] == 0, fields["pri-text"]) - assert(fields.iut == "1", fields.iut) - assert(fields.syslogfacility == 0, fields.syslogfacility) - assert(fields["syslogfacility-text"] == 0, fields["syslogfacility-text"]) - assert(fields.syslogpriority == 6, fields.syslogpriority) - assert(fields["syslogpriority-text"] == 6, fields["syslogpriority-text"]) - local ts = os.time{year = os.date("%Y"), month = 2, day = 10, hour = 9, min = 20, sec = 53} * 1e9 - assert(fields.timegenerated == ts, fields.timegenerated) - -- time reported is mapped to timestamp - assert(fields.timestamp == 1392052853559934000, fields.timestamp) - assert(fields["protocol-version"] == "0", fields["protocol-version"]) - assert(type(fields["structured-data"]) == "table", type(fields["structured-data"])) - assert(fields["app-name"] == "kernel", fields["app-name"]) - assert(fields.procid == "", fields.procid) - assert(fields.msgid == "-", fields.msgid) - assert(fields["$now"] == "2014-02-10", fields["$now"]) - assert(fields["$year"] == "2014") - assert(fields["$month"] == "02") - assert(fields["$day"] == "10") - assert(fields["$hour"] == "09") - assert(fields["$hhour"] == "00") - assert(fields["$qhour"] == "01") - assert(fields["$minute"] == "20") - assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) -end - -local function no_colon_in_tag() - local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%') - local log = '<166>2014-06-26T23:13:23-07:00 example.com at java.net.SocketInputStream.socketRead0(Native Method)' - local fields = grammar:match(log) - assert(fields.msg == "at java.net.SocketInputStream.socketRead0(Native Method)", fields.msg) - assert(fields.timestamp == 1403849603000000000, fields.timestamp) - assert(fields.syslogtag.programname == "", fields.syslogtag.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 20, fields.pri.facility) - assert(fields.hostname == "example.com", fields.hostname) -end - -local function structured_data() - local grammar = syslog.build_rsyslog_grammar("%STRUCTURED-DATA%") - local log = '[origin@123 name1="first value" name2="sec\\ond\\\\v[a\\]l\\"ue"]' - local fields = grammar:match(log) - local sd = fields["structured-data"] - assert(sd["_name1"] == "first value", sd["_name1"]) - assert(sd["_name2"] == "sec\\ond\\v[a]l\"ue", sd["_name2"]) - assert(sd["id"] == "origin@123", sd["id"]) - - log = '[origin@123 name="value\\\\"]' - fields = grammar:match(log) - sd = fields["structured-data"] - assert(sd["_name"] == "value\\", sd["_name"]) - assert(sd["id"] == "origin@123", sd["id"]) - - log = '[origin@123 name="value\\\""]' - fields = grammar:match(log) - sd = fields["structured-data"] - assert(sd["_name"] == 'value"', sd["_name"]) - assert(sd["id"] == "origin@123", sd["id"]) - - log = '[origin@123 name=""]' - fields = grammar:match(log) - sd = fields["structured-data"] - assert(sd["_name"] == "", sd["_name"]) - assert(sd["id"] == "origin@123", sd["id"]) -end - -function process() - traditional_file_format() - file_format() - forward_format() - traditional_forward_format() - kitchen_sink() - no_colon_in_tag() - structured_data() - - return 0 -end - diff --git a/src/test/lua/lpeg_syslog_message.lua b/src/test/lua/lpeg_syslog_message.lua deleted file mode 100644 index 75c6bde..0000000 --- a/src/test/lua/lpeg_syslog_message.lua +++ /dev/null @@ -1,354 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local syslog_message = require "lpeg.syslog_message" - -local function CRON() - local grammar = syslog_message.get_prog_grammar('CRON') - local log - local fields - - log = '(root) CMD ( cd / && run-parts --report /etc/cron.hourly)' - fields = grammar:match(log) - assert(fields.cron_username == 'root', fields.cron_username) - assert(fields.cron_event == 'CMD', fields.cron_event) - assert(fields.cron_detail == ' cd / && run-parts --report /etc/cron.hourly', fields.cron_detail) - -end - -local function crontab() - local grammar = syslog_message.get_prog_grammar('crontab') - local log - local fields - - log = '(root) LIST (root)' - fields = grammar:match(log) - assert(fields.cron_username == 'root', fields.cron_username) - assert(fields.cron_event == 'LIST', fields.cron_event) - assert(fields.cron_detail == 'root', fields.cron_detail) -end - -local function dhclient() - local grammar = syslog_message.get_prog_grammar('dhclient') - local log - local fields - - log = 'DHCPDISCOVER on eth0 to 10.20.30.42 port 67 interval 18000' - fields = grammar:match(log) - assert(fields.dhcp_type == 'DHCPDISCOVER', fields.dhcp_type) - assert(fields.dhcp_client_interface == 'eth0', fields.dhcdhcp_client_interfacep_source) - assert(fields.dhcp_server_addr.value == '10.20.30.42', fields.dhcp_server_addr) - assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) - assert(fields.dhcp_server_port == 67, fields.dhcp_server_port) - assert(fields.dhcp_client_interval_seconds == 18000, fields.dhcp_client_interval_seconds) - - - log = 'DHCPREQUEST on eth0 to 10.20.30.42 port 67' - fields = grammar:match(log) - assert(fields.dhcp_type == 'DHCPREQUEST', fields.dhcp_type) - assert(fields.dhcp_client_interface == 'eth0', fields.dhcdhcp_client_interfacep_source) - assert(fields.dhcp_server_addr.value == '10.20.30.42', fields.dhcp_server_addr) - assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) - assert(fields.dhcp_server_port == 67, fields.dhcp_server_port) - - log = 'DHCPACK from 10.20.30.42' - fields = grammar:match(log) - assert(fields.dhcp_type == 'DHCPACK', fields.dhcp_type) - assert(fields.dhcp_server_addr.value == '10.20.30.42', fields.dhcp_server_addr) - assert(fields.dhcp_server_addr.representation == 'ipv4', fields.dhcp_server_addr) - - log = 'bound to 10.20.30.40 -- renewal in 20346 seconds.' - fields = grammar:match(log) - assert(fields.dhcp_client_addr.value == '10.20.30.40', fields.dhcp_client_addr) - assert(fields.dhcp_client_addr.representation == 'ipv4', fields.dhcp_client_addr) - assert(fields.dhcp_client_renewal_seconds == 20346, fields.dhcp_client_renewal_seconds) -end - -local function dhcpd() - local grammar = syslog_message.get_prog_grammar('dhcpd') - local log - local fields - - log = 'DHCPINFORM from 192.168.0.45 via 192.168.0.1' - fields = grammar:match(log) - assert(fields.dhcp_type == 'DHCPINFORM', fields.dhcp_type) - assert(fields.dhcp_client_addr.value == '192.168.0.45', fields.dhcp_client_addr) - assert(fields.dhcp_client_addr.representation == 'ipv4', fields.dhcp_client_addr) - assert(fields.dhcp_source == '192.168.0.1', fields.dhcp_source) - - log = 'DHCPDISCOVER from aa:bb:cc:dd:ee:ff via 10.2.3.4: unknown network segment' - fields = grammar:match(log) - assert(fields.dhcp_type == 'DHCPDISCOVER', fields.dhcp_type) - assert(fields.dhcp_client_hw_addr == 'aa:bb:cc:dd:ee:ff', fields.dhcp_client_hw_addr) - assert(fields.dhcp_source == '10.2.3.4', fields.dhcp_source) - assert(fields.dhcp_message == 'unknown network segment', fields.dhcp_message) - - log = 'DHCPACK to 192.168.2.3 (aa:bb:cc:dd:ee:ff) via vlan42' - fields = grammar:match(log) - assert(fields.dhcp_type == 'DHCPACK', fields.dhcp_type) - assert(fields.dhcp_client_hw_addr == 'aa:bb:cc:dd:ee:ff', fields.dhcp_client_hw_addr) - assert(fields.dhcp_source == 'vlan42', fields.dhcp_source) -end - -local function login() - local grammar = syslog_message.get_prog_grammar('login') - local log - local fields - - log = "FAILED LOGIN (1) on '/dev/tty1' FOR 'root', Authentication failure" - fields = grammar:match(log) - assert(fields.failcount == 1, fields.failcount) - assert(fields.tty == '/dev/tty1', fields.tty) - assert(fields.user == 'root', fields.user) - assert(fields.pam_error == 'Authentication failure', fields.pam_error) -end - -local function named() - local grammar = syslog_message.get_prog_grammar('named') - local log - local fields - - log = "lame server resolving '40.30.20.10.in-addr.arpa' (in '30.20.10.in-addr.arpa'?): 10.11.12.13#53" - fields = grammar:match(log) - assert(fields.dns_error == 'lame server', fields.dns_error) - assert(fields.dns_name == '40.30.20.10.in-addr.arpa', fields.dns_name) - assert(fields.dns_domain == '30.20.10.in-addr.arpa', fields.dns_domain) - assert(fields.dns_addr.value == '10.11.12.13', fields.dns_addr.value) - assert(fields.dns_addr.representation == 'ipv4', fields.dns_addr.representation) - assert(fields.dns_port == 53, fields.dns_port) - - log = "host unreachable resolving 'ipv6.example.org/AAAA/IN': 2001:503:231d::2:30#53" - fields = grammar:match(log) - assert(fields.dns_error == 'host unreachable', fields.dns_error) - assert(fields.dns_name == 'ipv6.example.org', fields.dns_name) - assert(fields.dns_type == 'AAAA', fields.dns_type) - assert(fields.dns_class == 'IN', fields.dns_class) - assert(fields.dns_addr.value == '2001:503:231d::2:30', fields.dns_addr.value) - assert(fields.dns_addr.representation == 'ipv6', fields.dns_addr.representation) - assert(fields.dns_port == 53, fields.dns_port) - - log = "DNS format error from 134.170.107.24#53 resolving cid-ff58b408a75804a8.users.storage.live.com/AAAA for client 10.2.3.4#60466: Name storage.live.com (SOA) not subdomain of zone users.storage.live.com -- invalid response" - fields = grammar:match(log) - assert(fields.dns_error == 'DNS format error', fields.dns_error) - assert(fields.dns_addr.value == '134.170.107.24', fields.dns_addr.value) - assert(fields.dns_addr.representation == 'ipv4', fields.dns_addr.representation) - assert(fields.dns_port == 53, fields.dns_port) - assert(fields.dns_name == 'cid-ff58b408a75804a8.users.storage.live.com', fields.dns_name) - assert(fields.dns_type == 'AAAA', fields.dns_type) - assert(fields.dns_client_addr.value == '10.2.3.4', fields.dns_client_addr.value) - assert(fields.dns_client_addr.representation == 'ipv4', fields.dns_client_addr.representation) - assert(fields.dns_client_port == 60466, fields.dns_client_port) - assert(fields.dns_message == "Name storage.live.com (SOA) not subdomain of zone users.storage.live.com -- invalid response", fields.dns_message) - - log = 'DNS format error from 184.105.66.196#53 resolving ns-os1.qq.com/AAAA: Name qq.com (SOA) not subdomain of zone ns-os1.qq.com -- invalid response' - fields = grammar:match(log) - assert(fields.dns_error == 'DNS format error', fields.dns_error) - assert(fields.dns_addr.value == '184.105.66.196', fields.dns_addr.value) - assert(fields.dns_addr.representation == 'ipv4', fields.dns_addr.representation) - assert(fields.dns_port == 53, fields.dns_port) - assert(fields.dns_name == 'ns-os1.qq.com', fields.dns_name) - assert(fields.dns_type == 'AAAA', fields.dns_type) - assert(fields.dns_message == "Name qq.com (SOA) not subdomain of zone ns-os1.qq.com -- invalid response", fields.dns_message) - - log = "client 10.8.6.1#17069/key trusty (pc.example.org): view internal: transfer of 'pc.example.org/IN': IXFR started: TSIG trusty (serial 12 -> 14)" - fields = grammar:match(log) - assert(fields.dns_client_addr.value == '10.8.6.1', fields.dns_client_addr.value) - assert(fields.dns_client_addr.representation == 'ipv4', fields.dns_client_addr.representation) - assert(fields.dns_client_port == 17069, fields.dns_client_port) - assert(fields.dns_client_signer == 'trusty', fields.dns_client_signer) - assert(fields.dns_name == 'pc.example.org', fields.dns_name) - --assert(fields.dns_view == 'internal', fields.dns_view) - assert(fields.dns_message == "transfer of 'pc.example.org/IN': IXFR started: TSIG trusty (serial 12 -> 14)", fields.dns_message) - - log = "success resolving 'ns1.example.com/AAAA' (in 'example.com'?) after disabling EDNS" - fields = grammar:match(log) - assert(fields.dns_name == 'ns1.example.com', fields.dns_name) - assert(fields.dns_type == 'AAAA', fields.dns_type) - assert(fields.dns_domain == 'example.com', fields.dns_domain) - assert(fields.dns_message == "disabling EDNS", fields.dns_message) - - log = "zone 12.11.10.in-addr.arpa/IN/internal: sending notifies (serial 42)" - fields = grammar:match(log) - assert(fields.dns_domain == '12.11.10.in-addr.arpa', fields.dns_domain) - assert(fields.dns_class == 'IN', fields.dns_class) - assert(fields.dns_view == 'internal', fields.dns_view) - assert(fields.dns_message == "sending notifies", fields.dns_message) - assert(fields.dns_serial == 42, fields.dns_serial) - - log = "clients-per-query decreased to 22" - fields = grammar:match(log) - assert(fields.dns_clients_per_query == 22, fields.dns_clients_per_query) -end - -local function puppet_agent() - local grammar = syslog_message.get_prog_grammar('puppet-agent') - local log - local fields - - log = '(/Stage[main]/Nantes::Profile::Heka_base/Exec[setcap CAP_NET_BIND_SERVICE=+eip /usr/bin/hekad]/returns) executed successfully' - fields = grammar:match(log) - assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/Exec[setcap CAP_NET_BIND_SERVICE=+eip /usr/bin/hekad]', fields.puppet_resource_path) - assert(fields.puppet_parameter == 'returns', fields.puppet_parameter) - assert(fields.puppet_change == 'executed successfully', fields.puppet_change) - - log = "(/Stage[main]/Logstash::Service/Logstash::Service::Init[logstash]/Service[logstash]/ensure) ensure changed 'stopped' to 'running'" - fields = grammar:match(log) - assert(fields.puppet_resource_path == '/Stage[main]/Logstash::Service/Logstash::Service::Init[logstash]/Service[logstash]', fields.puppet_resource_path) - assert(fields.puppet_parameter == 'ensure', fields.puppet_parameter) - assert(fields.puppet_old_value == 'stopped', fields.puppet_old_value) - assert(fields.puppet_new_value == 'running', fields.puppet_new_value) - - log = '(/Stage[main]/Elasticsearch::Config/File[/var/log/elasticsearch/test.log]/owner) current_value root, should be elasticsearch (noop)' - fields = grammar:match(log) - assert(fields.puppet_resource_path == '/Stage[main]/Elasticsearch::Config/File[/var/log/elasticsearch/test.log]', fields.puppet_resource_path) - assert(fields.puppet_parameter == 'owner', fields.puppet_parameter) - assert(fields.puppet_current_value == 'root', fields.puppet_current_value) - assert(fields.puppet_should_value == 'elasticsearch', fields.puppet_should_value) - assert(fields.puppet_noop == true, fields.puppet_noop) - - log = '(/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_local_syslog.toml]/content)' - fields = grammar:match(log) - assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_local_syslog.toml]', fields.puppet_resource_path) - assert(fields.puppet_parameter == 'content', fields.puppet_parameter) - - log = '(/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]) Scheduling refresh of Service[heka]' - fields = grammar:match(log) - assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]', fields.puppet_resource_path) - assert(fields.puppet_callback == 'refresh', fields.puppet_callback) - assert(fields.puppet_callback_target == 'Service[heka]', fields.puppet_callback_target) - - log = "(Class[Logstash::Service]) Would have triggered 'refresh' from 1 events" - fields = grammar:match(log) - assert(fields.puppet_resource_path == 'Class[Logstash::Service]', fields.puppet_resource_path) - assert(fields.puppet_msg == 'Would have triggered', fields.puppet_msg) - assert(fields.puppet_callback == 'refresh', fields.puppet_callback) - assert(fields.puppet_events_count == 1, fields.puppet_events_count) - - log = '(/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]) Filebucketed /etc/heka/conf.d/10_net_syslog.toml to main with sum 70358a826b06f61f36bdc6aecaa3db14' - fields = grammar:match(log) - assert(fields.puppet_resource_path == '/Stage[main]/Nantes::Profile::Heka_base/File[/etc/heka/conf.d/10_net_syslog.toml]', fields.puppet_resource_path) - assert(fields.puppet_file_path == '/etc/heka/conf.d/10_net_syslog.toml', fields.puppet_file_path) - assert(fields.puppet_bucket == 'main', fields.puppet_bucket) - assert(fields.puppet_file_sum == '70358a826b06f61f36bdc6aecaa3db14', fields.puppet_file_sum) - - log = 'Finished catalog run in 7.11 seconds' - fields = grammar:match(log) - assert(fields.puppet_msg == 'Finished catalog run', fields.puppet_msg) - assert(fields.puppet_benchmark_seconds == 7.11, fields.puppet_benchmark_seconds) -end - -local function sshd() - local grammar = syslog_message.get_prog_grammar('sshd') - local log - local fields - - log = 'Accepted publickey for sathieu from 10.11.12.13 port 4242 ssh2' - fields = grammar:match(log) - assert(fields.sshd_authmsg == 'Accepted', fields.sshd_authmsg) - assert(fields.sshd_method == 'publickey', fields.sshd_method) - assert(fields.remote_user == 'sathieu', fields.remote_user) - assert(fields.remote_addr.value == '10.11.12.13', fields.remote_addr) - assert(fields.remote_port == 4242, fields.remote_port) - - log = "Failed password for invalid user administrator from 10.20.30.40 port 4242 ssh2" - fields = grammar:match(log) - assert(fields.sshd_authmsg == 'Failed', fields.sshd_authmsg) - assert(fields.sshd_method == 'password', fields.sshd_method) - assert(fields.remote_user == 'administrator', fields.remote_user) - assert(fields.remote_addr.value == '10.20.30.40', fields.remote_addr) - assert(fields.remote_port == 4242, fields.remote_port) - - log = "Received disconnect from 10.2.3.4: 11: The user disconnected the application [preauth]" - fields = grammar:match(log) - assert(fields.remote_addr.value == '10.2.3.4', fields.remote_addr) - assert(fields.disconnect_reason == 11, fields.disconnect_reason) - assert(fields.disconnect_msg == 'The user disconnected the application [preauth]', fields.disconnect_msg) -end - -local function sudo() - local grammar = syslog_message.get_prog_grammar('sudo') - local log - local fields - - log = "usrnagios : TTY=unknown ; PWD=/home/usrnagios ; USER=root ; COMMAND=/usr/bin/ctdb -Y status" - fields = grammar:match(log) - assert(fields.sudo_message == 'usrnagios', fields.sudo_message) - assert(fields.sudo_tty == 'unknown', fields.sudo_tty) - assert(fields.sudo_pwd == '/home/usrnagios', fields.sudo_pwd) - assert(fields.sudo_user == 'root', fields.sudo_user) - assert(fields.sudo_command == '/usr/bin/ctdb -Y status', fields.sudo_command) -end - -local function systemd_logind() - local grammar = syslog_message.get_prog_grammar('systemd-logind') - local log - local fields - - log = "New session 42 of user sathieu." - fields = grammar:match(log) - assert(fields.sd_message == 'SESSION_START', fields.sd_message) - assert(fields.session_id == 42, fields.session_id) - assert(fields.user_id == 'sathieu', fields.user_id) - - log = "Removed session 42." - fields = grammar:match(log) - assert(fields.sd_message == 'SESSION_STOP', fields.sd_message) - assert(fields.session_id == 42, fields.session_id) -end - -local function pam() - local grammar = syslog_message.get_wildcard_grammar('PAM') - local log - local fields - - log = "pam_unix(login:session): session opened for user sathieu by LOGIN(uid=1000)" - fields = grammar:match(log) - assert(fields.pam_module == 'pam_unix', fields.pam_module) - assert(fields.pam_service == 'login', fields.pam_service) - assert(fields.pam_type == 'session', fields.pam_type) - assert(fields.pam_action == 'session opened', fields.pam_action) - assert(fields.user_name == 'sathieu', fields.user_name) - assert(fields.uid == 1000, fields.uid) - - log = "pam_unix(login:auth): authentication failure; logname=LOGIN uid=0 euid=0 tty=/dev/tty1 ruser= rhost= user=root" - fields = grammar:match(log) - assert(fields.pam_module == 'pam_unix', fields.pam_module) - assert(fields.pam_service == 'login', fields.pam_service) - assert(fields.pam_type == 'auth', fields.pam_type) - assert(fields.pam_action == 'authentication failure', fields.pam_action) - assert(fields.logname == 'LOGIN', fields.logname) - assert(fields.uid == 0, fields.uid) - assert(fields.euid == 0, fields.euid) - assert(fields.tty == '/dev/tty1', fields.tty) - assert(fields.ruser == '', fields.ruser) - assert(fields.rhost == '', fields.rhost) - assert(fields.user == 'root', fields.user) - - log = "pam_unix(login:session): session closed for user sathieu" - fields = grammar:match(log) - assert(fields.pam_module == 'pam_unix', fields.pam_module) - assert(fields.pam_service == 'login', fields.pam_service) - assert(fields.pam_type == 'session', fields.pam_type) - assert(fields.pam_action == 'session closed', fields.pam_action) - assert(fields.user_name == 'sathieu', fields.user_name) -end - -function process() - CRON() - crontab() - dhclient() - dhcpd() - login() - named() - puppet_agent() - sshd() - sudo() - systemd_logind() - - -- wildcard grammars - pam() - - return 0 -end - diff --git a/src/test/lua/no_external_modules.lua b/src/test/lua/no_external_modules.lua new file mode 100644 index 0000000..c446b69 --- /dev/null +++ b/src/test/lua/no_external_modules.lua @@ -0,0 +1,8 @@ +require "foo" + +function process(tc) + return 0 +end + +function report() +end diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 6af9162..43de543 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -2,7 +2,14 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. + function process(tc) - write_output(1.2, " string ", nil, " ", true, " ", false) + if tc == 0 then -- lua types + write_output(1.2, " string ", nil, " ", true, " ", false) + elseif tc == 1 then -- user data + require "ud" + local udv = ud.new("foo") + write_output(udv) + end return 0 end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 4186638..401470e 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -2,9 +2,9 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "cjson" -require "circular_buffer" -require "lpeg" +require "io" + + function process(tc) if tc == 0 then -- error internal reference output({}) @@ -15,16 +15,7 @@ function process(tc) end output(escape) elseif tc == 2 then -- unsupported userdata - write_output(lpeg.P"") - elseif tc == 3 then -- overflow - local cb = circular_buffer.new(1000, 1, 60); - write_output(cb) - elseif tc == 4 then -- overflow cjson encode buffer - local t = {} - for i=1, 10 do - t[#t+1] = "this is a test" - end - cjson.encode(t) + write_output(io.stdin) end return 0 end diff --git a/src/test/lua/sax.lua b/src/test/lua/sax.lua deleted file mode 100644 index 99985b4..0000000 --- a/src/test/lua/sax.lua +++ /dev/null @@ -1,53 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "sax" - -local SAX_N = 9 -local SAX_W = 3 -local SAX_C = 5 - -local data = {16,18,12,24,11,15,16,12,17} -local nan_data = {0/0,0/0,0/0,0/0,0/0,0/0,0/0,0/0,0/0} -local partial_inf_data = {16,0/0,12,1/0,11,15,-1/0,12,17} - -unused = sax.window.new(SAX_N, SAX_W, SAX_C) -win = sax.window.new(SAX_N, SAX_W, SAX_C) -nan_win = sax.window.new(SAX_N, SAX_W, SAX_C) -partial_inf_win = sax.window.new(SAX_N, SAX_W, SAX_C) -mm = sax.window.new(100, 5, 8) -mm_tail = sax.window.new(100, 5, 8) - -word = sax.word.new(data, SAX_W, SAX_C) -nan_word = sax.word.new("###", SAX_C) -w1 = word - -assert(tostring(win) == "###") - -function process() - for i,v in ipairs(data) do - win:add(v) - end - - for i,v in ipairs(nan_data) do - nan_win:add(v) - end - - for i,v in ipairs(partial_inf_data) do - partial_inf_win:add(v) - end - - for i=1, 1000000 do - mm:add(i-1) - end - - for i=999001, 1000000 do - mm_tail:add(i-1) - end - return 0 -end - -function report(tc) - write_output(win, " ", word, " ", mm, " ", mm_tail) -end diff --git a/src/test/lua/sax_benchmark.lua b/src/test/lua/sax_benchmark.lua deleted file mode 100644 index 31b2c5a..0000000 --- a/src/test/lua/sax_benchmark.lua +++ /dev/null @@ -1,24 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "sax" - -local data = {16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, - 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, - 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, - 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13, - 16,18,12,24,11,15,16,12,17,13,16,18,12,24,11,15,16,12,17,13} - -win = sax.window.new(100, 5, 8) -win:add(data) - -function process(ts) - win:add(ts) - return 0 -end - -function report(tc) - write_output(tostring(win:get_word())) -end - diff --git a/src/test/lua/serialize.lua b/src/test/lua/serialize.lua index 0eac4df..d856be2 100644 --- a/src/test/lua/serialize.lua +++ b/src/test/lua/serialize.lua @@ -2,14 +2,13 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "circular_buffer" -util = require "lsb.util" +require "ud" count = 0 rate = 0.12345678 rates = {99.1,98,97,92.002,91.10001,key="val"} kvp = {a="foo", b="bar", r=rates} -nested = {arg1=1, arg2=2, nested={n1="one",n2="two"}, empty = nil, cb = circular_buffer.new(2,6,1)} +nested = {arg1=1, arg2=2, nested={n1="one",n2="two"}, empty = nil, ud = ud.new("ud1")} _G["key with spaces"] = "kws" boolean = true empty = nil @@ -39,12 +38,7 @@ cycleb = {type="cycle b"} cyclea["b"] = cycleb cycleb["a"] = cyclea -data = circular_buffer.new(3,3,1) - -delta = circular_buffer.new(2,1,1) -delta:add(0, 1, 2) -delta:annotate(0, 1, "info", "annotation\"\t\b\r\n\240 end") -delta:annotate(1e9, 1, "alert", "boom") +data = ud.new("ud2") dataRef = data @@ -55,3 +49,4 @@ end function timer_event(ns) end + diff --git a/src/test/lua/struct.lua b/src/test/lua/struct.lua deleted file mode 100644 index 3f41da8..0000000 --- a/src/test/lua/struct.lua +++ /dev/null @@ -1,11 +0,0 @@ -require "struct" - -local fmt = " b end) do - assert(abc[cnt] == k) - cnt = cnt - 1 - end -end - - -function process () - behead_array() - pairs_by_key() - return 0 -end diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index 0f9a1f7..36640f0 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -1,16 +1,6 @@ if _PRESERVATION_VERSION and _PRESERVATION_VERSION ~= 0 then return end _G["ninf"] = -1/0 -if _G["dataRef"] == nil then _G["dataRef"] = circular_buffer.new(3, 3, 1) end -_G["dataRef"]:set_header(1, "Column_1", "count", "sum") -_G["dataRef"]:set_header(2, "Column_2", "count", "sum") -_G["dataRef"]:set_header(3, "Column_3", "count", "sum") -_G["dataRef"]:fromstring("2 2 nan nan nan nan nan nan nan nan nan") -if _G["delta"] == nil then _G["delta"] = circular_buffer.new(2, 1, 1) end -_G["delta"]:set_header(1, "Column_1", "count", "sum") -_G["delta"]:fromstring("1 1 2 nan 0 2") -_G["delta"]:annotate(1e+09, 1, "alert", "boom", true) -_G["delta"]:annotate(0, 1, "info", "annotation\"\t\b\r\n end", true) -_G["data"] = _G["dataRef"] +if _G["dataRef"] == nil then _G["dataRef"] = ud.new('ud2') end _G["rate"] = 0.12345678 _G["kvp"] = {} _G["kvp"]["a"] = "foo" @@ -22,11 +12,6 @@ _G["kvp"]["r"][4] = 92.002 _G["kvp"]["r"][5] = 91.10001 _G["kvp"]["r"]["key"] = "val" _G["kvp"]["b"] = "bar" -_G["cycleb"] = {} -_G["cycleb"]["a"] = {} -_G["cycleb"]["a"]["b"] = _G["cycleb"] -_G["cycleb"]["a"]["type"] = "cycle a" -_G["cycleb"]["type"] = "cycle b" _G["uuids"] = {} _G["uuids"][1] = {} _G["uuids"][1]["type"] = "test" @@ -35,6 +20,12 @@ _G["uuids"][2] = {} _G["uuids"][2]["type"] = "test1" _G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" _G["inf"] = 1/0 +_G["data"] = _G["dataRef"] +_G["cycleb"] = {} +_G["cycleb"]["a"] = {} +_G["cycleb"]["a"]["b"] = _G["cycleb"] +_G["cycleb"]["a"]["type"] = "cycle a" +_G["cycleb"]["type"] = "cycle b" _G["cyclea"] = _G["cycleb"]["a"] _G["large_key"] = {} _G["large_key"]["aaaaaaaaaaaaaaaaaaa"] = {} @@ -55,6 +46,7 @@ _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccc _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["ccccccccccccccccccc"]["ddddddddddddddddddd"]["eeeeeeeeeeeeeeeeeee"]["fffffffffffffffffff"]["BD48B609-8922-4E59-A358-C242075CE086"] = 6 _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["BD48B609-8922-4E59-A358-C242075CE082"] = 2 _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["BD48B609-8922-4E59-A358-C242075CE081"] = 1 +_G["count"] = 0 _G["boolean"] = true _G["key with spaces"] = "kws" _G["nested"] = {} @@ -63,15 +55,7 @@ _G["nested"]["nested"] = {} _G["nested"]["nested"]["n2"] = "two" _G["nested"]["nested"]["n1"] = "one" _G["nested"]["arg1"] = 1 -if _G["nested"]["cb"] == nil then _G["nested"]["cb"] = circular_buffer.new(2, 6, 1) end -_G["nested"]["cb"]:set_header(1, "Column_1", "count", "sum") -_G["nested"]["cb"]:set_header(2, "Column_2", "count", "sum") -_G["nested"]["cb"]:set_header(3, "Column_3", "count", "sum") -_G["nested"]["cb"]:set_header(4, "Column_4", "count", "sum") -_G["nested"]["cb"]:set_header(5, "Column_5", "count", "sum") -_G["nested"]["cb"]:set_header(6, "Column_6", "count", "sum") -_G["nested"]["cb"]:fromstring("1 1 nan nan nan nan nan nan nan nan nan nan nan nan") +if _G["nested"]["ud"] == nil then _G["nested"]["ud"] = ud.new('ud1') end _G["rates"] = _G["kvp"]["r"] -_G["count"] = 0 _G["_VERSION"] = "Lua 5.1" _G["nan"] = 0/0 diff --git a/src/test/sandbox.c b/src/test/sandbox.c index 82975bf..cd9df9c 100644 --- a/src/test/sandbox.c +++ b/src/test/sandbox.c @@ -52,7 +52,7 @@ int lsb_test_write_output(lua_State *lua) } -int lsb_test_process(lsb_lua_sandbox *lsb, double ts) +int lsb_test_process(lsb_lua_sandbox *lsb, double tc) { static const char *func_name = "process"; lua_State *lua = lsb_get_lua(lsb); @@ -60,7 +60,7 @@ int lsb_test_process(lsb_lua_sandbox *lsb, double ts) if (lsb_pcall_setup(lsb, func_name)) return 1; - lua_pushnumber(lua, ts); + lua_pushnumber(lua, tc); if (lua_pcall(lua, 1, 2, 0) != 0) { char err[LSB_ERROR_SIZE]; const char *em = lua_tostring(lua, -1); diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c index eb1841a..b898a75 100644 --- a/src/test/test_generic_sandbox.c +++ b/src/test/test_generic_sandbox.c @@ -18,6 +18,8 @@ #include "luasandbox/test/mu_test.h" #include "luasandbox/test/sandbox.h" #include "luasandbox/util/util.h" +#include "luasandbox_output.h" +#include "luasandbox_serialize.h" char *e = NULL; static char print_out[2048] = { 0 }; @@ -29,6 +31,78 @@ static char print_out[2048] = { 0 }; #define MODULE_PATH "path = 'modules/?.lua';cpath = 'modules/?.so'\n" #endif +static const char *mozsvc_test_ud = "mozsvc.test_ud"; + +typedef struct test_ud { + char name[10]; +} test_ud; + + +static int ud_new(lua_State *lua) +{ + size_t len; + const char *name = luaL_checklstring(lua, 1, &len); + test_ud *ud = lua_newuserdata(lua, sizeof(test_ud)); + if (len < 10) { + strcpy(ud->name, name); + } else { + memcpy(ud->name, name, 9); + ud->name[9] = 0; + } + luaL_getmetatable(lua, mozsvc_test_ud); + lua_setmetatable(lua, -2); + return 1; +} + + +static int ud_serialize(lua_State *lua) +{ + lsb_output_buffer *ob = lua_touserdata(lua, -1); + const char *key = lua_touserdata(lua, -2); + test_ud *ud = lua_touserdata(lua, -3); + if (!(ob && key && ud)) {return 1;} + return lsb_outputf(ob, "if %s == nil then %s = ud.new('%s') end\n", key, key, + ud->name) == NULL ? 0 : 1; +} + + +static int ud_output(lua_State *lua) +{ + lsb_output_buffer *ob = lua_touserdata(lua, -1); + test_ud *ud = lua_touserdata(lua, -2); + if (!(ob && ud)) {return 1; } + return lsb_outputf(ob, "%s", ud->name) == NULL ? 0 : 1; +} + + +static const struct luaL_reg testudlib_f[] = +{ + { "new", ud_new }, { NULL, NULL } +}; + + +static int luaopen_ud(lua_State *lua) +{ + lua_newtable(lua); + lsb_add_serialize_function(lua, ud_serialize); + lsb_add_output_function(lua, ud_output); + lua_replace(lua, LUA_ENVIRONINDEX); + luaL_newmetatable(lua, mozsvc_test_ud); + luaL_register(lua, "ud", testudlib_f); + return 1; +} + + +static void add_ud_module(lsb_lua_sandbox *sb) +{ + lua_State *lua = lsb_get_lua(sb); + luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); + lua_pushstring(lua, "ud"); + lua_pushcfunction(lua, luaopen_ud); + lua_rawset(lua, -3); + lua_pop(lua, 1); // remove the preloaded table +} + static const char *test_cfg = "userflag = true\n" @@ -187,14 +261,14 @@ static char* test_init_error() e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s\n", e); - sb = lsb_create(NULL, "lua/lpeg_date_time.lua", "path = '';cpath = ''", NULL); + sb = lsb_create(NULL, "lua/no_external_modules.lua", NULL, NULL); mu_assert(sb, "lsb_create() received: NULL"); // disabled external modules ret = lsb_init(sb, NULL); mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); - const char *expected = "lua/lpeg_date_time.lua:7: module 'lpeg.date_time' not " - "found:"; + const char *expected = "no 'path' configuration was specified for the " + "sandbox; external modules have been disabled"; mu_assert(strcmp(lsb_get_error(sb), expected) == 0, "lsb_get_error() received: %s", lsb_get_error(sb)); @@ -364,11 +438,13 @@ static char* test_output() { const char *outputs[] = { "1.2 string nil true false", + "foo", NULL }; lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); @@ -398,8 +474,6 @@ static char* test_output_errors() "process() lua/output_errors.lua:10: bad argument #1 to 'output' (unsupported type)" , "process() lua/output_errors.lua:16: output_limit exceeded" , "process() lua/output_errors.lua:18: bad argument #1 to 'write_output' (unknown userdata type)" - , "process() lua/output_errors.lua:21: output_limit exceeded" - , "process() lua/output_errors.lua:27: strbuf output_limit exceeded" , NULL }; @@ -427,48 +501,6 @@ static char* test_output_errors() } -static char* test_cjson() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cjson.lua", - MODULE_PATH "output_limit = 64", NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - - int result = lsb_test_process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_cjson_unlimited() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/cjson_unlimited.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &lsb_test_write_output, "write_output"); - - int result = lsb_test_process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - mu_assert(lsb_test_output_len == 103001, "received %d bytes", (int)lsb_test_output_len); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_errors() { const char *tests[] = { @@ -532,60 +564,6 @@ static char* test_errors() } -static char* test_lpeg() -{ - const char *tests[] = { - "lua/lpeg.lua" - , "lua/lpeg_clf.lua" - , "lua/lpeg_cbufd.lua" - , "lua/lpeg_date_time.lua" - , "lua/lpeg_ip_address.lua" - , "lua/lpeg_mysql.lua" - , "lua/lpeg_postfix.lua" - , "lua/lpeg_syslog.lua" - , "lua/lpeg_syslog_message.lua" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lsb_lua_sandbox *sb = lsb_create(NULL, tests[i], test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - - int result = lsb_test_process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - - return NULL; -} - - -static char* test_util() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/util_test.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - - int result = lsb_test_process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_serialize() { const char *output_file = "serialize.preserve"; @@ -593,17 +571,14 @@ static char* test_serialize() remove(output_file); lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); lsb_err_value ret = lsb_init(sb, output_file); mu_assert(!ret, "lsb_init() received: %s", ret); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); -#ifdef LUA_JIT - char *expected = lsb_read_file("output/serialize.data"); -#else char *expected = lsb_read_file("output/serialize.lua51.data"); -#endif char *actual = lsb_read_file(output_file); mu_assert(strcmp(expected, actual) == 0, "serialization mismatch"); free(expected); @@ -688,24 +663,6 @@ static char* test_serialize_failure() } -static char* test_struct() -{ - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/struct.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - - int result = lsb_test_process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_sandbox_config() { lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sandbox_config.lua", test_cfg, NULL); @@ -721,50 +678,6 @@ static char* test_sandbox_config() } -static char* test_sax() -{ - const char *output_file = "sax.preserve"; - const char *word = "CDC CDC ABEGH ABEGH"; - - remove(output_file); - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sax.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &lsb_test_write_output, "write_output"); - - int result = lsb_test_report(sb, 0); - mu_assert(result == 0, "report() received: %d %s", result, lsb_get_error(sb)); - mu_assert(strcmp("### CDC ##### #####", lsb_test_output) == 0, "received: %s", lsb_test_output); - - result = lsb_test_process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = lsb_test_report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(word, lsb_test_output) == 0, "received: %s", lsb_test_output); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/sax.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - - ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &lsb_test_write_output, "write_output"); - - lsb_test_report(sb, 0); - mu_assert(strcmp(word, lsb_test_output) == 0, "received: %s", lsb_test_output); - - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - static char* test_print() { const char *tests[] = @@ -848,6 +761,21 @@ static char* test_print_lsb_test_logger() } +static char* test_serialize_binary() +{ + size_t size = 512; + lsb_output_buffer b; + lsb_err_value ret = lsb_init_output_buffer(&b, size); + mu_assert(ret == NULL, "received: %s", lsb_err_string(ret)); + lsb_serialize_binary(&b, "a\r\n\\\"", 5); + mu_assert(b.pos == 9, "received %d", (int)b.pos); + mu_assert(memcmp(b.buf, "a\\r\\n\\\\\\\"", 5) == 0, "received %.*s", 9, + b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + static char* benchmark_counter() { int iter = 10000000; @@ -880,9 +808,10 @@ static char* benchmark_serialize() remove(output_file); lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); lsb_err_value ret = lsb_init(sb, output_file); - mu_assert(!ret, "lsb_init() received: %s", ret); + mu_assert(!ret, "lsb_init() received: %s %s", ret, lsb_get_error(sb)); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); } @@ -902,6 +831,7 @@ static char* benchmark_deserialize() for (int x = 0; x < iter; ++x) { lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); lsb_err_value ret = lsb_init(sb, "output/serialize.data"); mu_assert(!ret, "lsb_init() received: %s", ret); @@ -942,34 +872,6 @@ static char* benchmark_lua_types_output() } -static char* benchmark_sax_add() -{ - int iter = 1000000; - - lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sax_benchmark.lua", test_cfg, NULL); - mu_assert(sb, "lsb_create() received: NULL"); - lsb_err_value ret = lsb_init(sb, NULL); - mu_assert(!ret, "lsb_init() received: %s", ret); - lsb_add_function(sb, &lsb_test_write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == lsb_test_process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - lsb_test_report(sb, 0); - mu_assert(strcmp("ABEGH", lsb_test_output) == 0, "received: %s", lsb_test_output); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_sax_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_sax_add() %g seconds\n", ((double)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - static char* all_tests() { mu_run_test(test_api_assertion); @@ -984,26 +886,20 @@ static char* all_tests() mu_run_test(test_simple_error); mu_run_test(test_output); mu_run_test(test_output_errors); - mu_run_test(test_cjson); - mu_run_test(test_cjson_unlimited); mu_run_test(test_errors); - mu_run_test(test_lpeg); - mu_run_test(test_util); mu_run_test(test_serialize); mu_run_test(test_restore); mu_run_test(test_serialize_failure); - mu_run_test(test_struct); mu_run_test(test_sandbox_config); - mu_run_test(test_sax); mu_run_test(test_print); mu_run_test(test_print_disabled); mu_run_test(test_print_lsb_test_logger); + mu_run_test(test_serialize_binary); mu_run_test(benchmark_counter); mu_run_test(benchmark_serialize); mu_run_test(benchmark_deserialize); mu_run_test(benchmark_lua_types_output); - mu_run_test(benchmark_sax_add); return NULL; } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 939cf87..e0df018 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -23,5 +23,5 @@ if(LIBM_LIBRARY) target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) endif() -install(TARGETS luasandboxutil DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT core) +install(TARGETS luasandboxutil DESTINATION ${CMAKE_INSTALL_LIBDIR}) add_subdirectory(test) diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index c6b4069..b79f732 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -26,6 +26,7 @@ static char* test_stub() return NULL; } + static char* test_init_small_buf() { size_t size = 512; From 8f6db36078d681a91f2596d5488353c3266b24e9 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 12 Jul 2016 06:57:00 -0700 Subject: [PATCH 170/235] Fix OSX build errors --- include/luasandbox/test/sandbox.h | 2 +- src/luasandbox_defines.h | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/include/luasandbox/test/sandbox.h b/include/luasandbox/test/sandbox.h index ea962d0..8465fa0 100644 --- a/include/luasandbox/test/sandbox.h +++ b/include/luasandbox/test/sandbox.h @@ -87,4 +87,4 @@ LSB_TEST_EXPORT int lsb_test_write_output(lua_State *lua); } #endif -#endif \ No newline at end of file +#endif diff --git a/src/luasandbox_defines.h b/src/luasandbox_defines.h index d4315c9..bee13dd 100644 --- a/src/luasandbox_defines.h +++ b/src/luasandbox_defines.h @@ -13,6 +13,8 @@ #define snprintf _snprintf #elif __linux #define CLOSE_ON_EXEC "e" +#else +#define CLOSE_ON_EXEC "" #endif #ifdef _MSC_VER From 766157637debb197cd5e036d910c68ab2d4a3f52 Mon Sep 17 00:00:00 2001 From: Michael Trinkala Date: Tue, 12 Jul 2016 08:29:29 -0700 Subject: [PATCH 171/235] Fix Windows build errors --- src/CMakeLists.txt | 1 - src/heka/message.c | 1 + src/heka/sandbox.c | 4 +++- src/lua/lgc.c | 2 +- src/lua/ltable.c | 4 ++-- src/luasandbox.c | 1 + src/luasandbox_defines.h | 4 +++- src/luasandbox_output.c | 1 + src/luasandbox_serialize.c | 1 + src/test/sandbox.c | 1 + src/util/util.c | 2 +- 11 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 367bafa..beff13e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,7 +4,6 @@ if(MSVC) add_definitions( - -DLUA_WIN -DLUA_BUILD_AS_DLL -D_CRT_SECURE_NO_WARNINGS ) diff --git a/src/heka/message.c b/src/heka/message.c index d5e24b3..00a724f 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -23,6 +23,7 @@ #include "luasandbox/util/heka_message.h" #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" +#include "../luasandbox_defines.h" static void set_missing_headers(lua_State *lua, int idx, lsb_heka_sandbox *hsb) diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 86112c8..e564c33 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -114,7 +114,9 @@ static int inject_message_input(lua_State *lua) lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); lua_pop(lua, 1); // remove this ptr - if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); + if (!lsb) { + return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); + } const char *scp = NULL; double ncp = NAN; diff --git a/src/lua/lgc.c b/src/lua/lgc.c index e909c79..95bbdfb 100644 --- a/src/lua/lgc.c +++ b/src/lua/lgc.c @@ -285,7 +285,7 @@ static l_mem propagatemark (global_State *g) { if (traversetable(g, h)) /* table is weak? */ black2gray(o); /* keep it gray */ return sizeof(Table) + sizeof(TValue) * h->sizearray + - sizeof(Node) * sizenode(h); + sizeof(Node) * (size_t)sizenode(h); } case LUA_TFUNCTION: { Closure *cl = gco2cl(o); diff --git a/src/lua/ltable.c b/src/lua/ltable.c index ec84f4f..18e7294 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -320,7 +320,7 @@ static void resize (lua_State *L, Table *t, int nasize, int nhsize) { setobjt2t(L, luaH_set(L, t, key2tval(old)), gval(old)); } if (nold != dummynode) - luaM_freearray(L, nold, twoto(oldhsize), Node); /* free old array */ + luaM_freearray(L, nold, (size_t)twoto(oldhsize), Node); /* free old array */ } @@ -373,7 +373,7 @@ Table *luaH_new (lua_State *L, int narray, int nhash) { void luaH_free (lua_State *L, Table *t) { if (t->node != dummynode) - luaM_freearray(L, t->node, sizenode(t), Node); + luaM_freearray(L, t->node, (size_t)sizenode(t), Node); luaM_freearray(L, t->array, t->sizearray, TValue); luaM_free(L, t); } diff --git a/src/luasandbox.c b/src/luasandbox.c index 0781e87..be972d1 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -6,6 +6,7 @@ /** @brief Lua sandboxed implementation @file */ +#define LUA_LIB #include "luasandbox.h" #include diff --git a/src/luasandbox_defines.h b/src/luasandbox_defines.h index bee13dd..578081d 100644 --- a/src/luasandbox_defines.h +++ b/src/luasandbox_defines.h @@ -11,7 +11,9 @@ #ifdef _WIN32 #define snprintf _snprintf -#elif __linux +#endif + +#if __linux #define CLOSE_ON_EXEC "e" #else #define CLOSE_ON_EXEC "" diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c index c2e2ff6..93c18ec 100644 --- a/src/luasandbox_output.c +++ b/src/luasandbox_output.c @@ -6,6 +6,7 @@ /** @brief Lua sandbox output buffer implementation @file */ +#define LUA_LIB #include "luasandbox_output.h" #include diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 7007345..350e1f4 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -6,6 +6,7 @@ /** @brief Sandbox serialization implementation @file */ +#define LUA_LIB #include "luasandbox_serialize.h" #include diff --git a/src/test/sandbox.c b/src/test/sandbox.c index cd9df9c..a3608ab 100644 --- a/src/test/sandbox.c +++ b/src/test/sandbox.c @@ -16,6 +16,7 @@ #include "luasandbox/lauxlib.h" #include "luasandbox/lua.h" #include "luasandbox_output.h" +#include "../luasandbox_defines.h" const char *lsb_test_output = NULL; size_t lsb_test_output_len = 0; diff --git a/src/util/util.c b/src/util/util.c index 4599ee4..d14643a 100644 --- a/src/util/util.c +++ b/src/util/util.c @@ -130,7 +130,7 @@ long long lsb_get_timestamp() unsigned long long t = ft.dwHighDateTime; t <<= 32; t |= ft.dwLowDateTime; - t -= 16444736000000000ULL; // convert from Jan 1 1601 to Jan 1 1970 + t -= 116444736000000000ULL; // convert from Jan 1 1601 to Jan 1 1970 return t * 100LL; #else struct timeval tv; From 153bd4565a0bf90d3d223e9fc1e6430a1911194f Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 12 Jul 2016 08:53:48 -0700 Subject: [PATCH 172/235] Bump the version to 1.0.0 - fix doc typos --- CMakeLists.txt | 2 +- gen_gh_pages.lua | 3 +++ include/luasandbox.h | 4 ++-- include/luasandbox/heka/sandbox.h | 4 ++-- include/luasandbox/util/output_buffer.h | 8 ++++---- 5 files changed, 12 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eab13fc..4eb1e00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 0.24.0 LANGUAGES C) +project(luasandbox VERSION 1.0.0 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index dec91ca..331fe84 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -117,6 +117,9 @@ local function output_menu(before, after, paths, version)

              +
              ]]) diff --git a/include/luasandbox.h b/include/luasandbox.h index f651d11..47e140e 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -148,8 +148,8 @@ LSB_EXPORT char* lsb_destroy(lsb_lua_sandbox *lsb); * Retrieve the sandbox usage statistics. * * @param lsb Pointer to the sandbox. - * @param lsb_usage_type Type of statistic to retrieve i.e. memory. - * @param lsb_usage_stat Type of statistic to retrieve i.e. current. + * @param utype Type of statistic to retrieve i.e. memory. + * @param ustat Type of statistic to retrieve i.e. current. * * @return size_t Count or number of bytes depending on the statistic. */ diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index b6a38fa..439c439 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -204,7 +204,7 @@ int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb * defaults) * @param logger Struct for error reporting/debug printing (NULL to disable) - * @param upc checkpoint_updated callback when using batch or async output + * @param ucp checkpoint_updated callback when using batch or async output * * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL */ @@ -261,7 +261,7 @@ lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb); * @param err Reason for termination */ LSB_HEKA_EXPORT void -lsb_heka_terminate_sandbox(lsb_heka_sandbox *lsb, const char *err); +lsb_heka_terminate_sandbox(lsb_heka_sandbox *hsb, const char *err); /** * Frees all memory associated with the sandbox; hsb cannont be used after this diff --git a/include/luasandbox/util/output_buffer.h b/include/luasandbox/util/output_buffer.h index 2adac33..6a325f4 100644 --- a/include/luasandbox/util/output_buffer.h +++ b/include/luasandbox/util/output_buffer.h @@ -94,23 +94,23 @@ lsb_outputs(lsb_output_buffer *b, const char *str, size_t len); * More efficient output of a double to a string. NaN/Inf check and then calls * lsb_outputfd. * - * @param output Pointer the output buffer. + * @param b Pointer the output buffer. * @param d Double value to convert to a string. * * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT lsb_err_value lsb_outputd(lsb_output_buffer *output, double d); +LSB_UTIL_EXPORT lsb_err_value lsb_outputd(lsb_output_buffer *b, double d); /** * More efficient output of a double to a string; no NaN or Inf outputs. * - * @param output Pointer the output buffer. + * @param b Pointer the output buffer. * @param d Double value to convert to a string. * * @return lsb_err_value NULL on success error message on failure */ -LSB_UTIL_EXPORT lsb_err_value lsb_outputfd(lsb_output_buffer *ob, double d); +LSB_UTIL_EXPORT lsb_err_value lsb_outputfd(lsb_output_buffer *b, double d); #ifdef __cplusplus } From e704dc6c572ed04a2efb0ac148a018d248b341f7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 13 Jul 2016 20:22:48 -0700 Subject: [PATCH 173/235] Add error functions to the math library --- CMakeLists.txt | 2 +- docs/sandbox.md | 4 ++++ src/lua/lmathlib.c | 14 ++++++++++++++ src/test/lua/sandbox_config.lua | 2 ++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4eb1e00..5860ad8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.0.0 LANGUAGES C) +project(luasandbox VERSION 1.0.1 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/sandbox.md b/docs/sandbox.md index 4e72119..54bacc7 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -49,6 +49,10 @@ By default only the base library is loaded additional libraries must be loaded w The following modules have been modified, as described, for use in the sandbox. - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) - The require() function has been modified to not expose any of the package table to the sandbox. + - [math](http://www.lua.org/manual/5.1/manual.html#5.6) + - Added Functions + - erf(x) - Returns the error function value for x. + - erfc(x)- Returns the complementary error function value for x. - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - The local timezone is set to UTC in all sandboxes. diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c index 441fbf7..0ff1526 100644 --- a/src/lua/lmathlib.c +++ b/src/lua/lmathlib.c @@ -212,6 +212,18 @@ static int math_randomseed (lua_State *L) { } +static int math_erf (lua_State *L) { + lua_pushnumber(L, erf(luaL_checknumber(L, 1))); + return 1; +} + + +static int math_erfc (lua_State *L) { + lua_pushnumber(L, erfc(luaL_checknumber(L, 1))); + return 1; +} + + static const luaL_Reg mathlib[] = { {"abs", math_abs}, {"acos", math_acos}, @@ -222,6 +234,8 @@ static const luaL_Reg mathlib[] = { {"cosh", math_cosh}, {"cos", math_cos}, {"deg", math_deg}, + {"erf", math_erf}, + {"erfc", math_erfc}, {"exp", math_exp}, {"floor", math_floor}, {"fmod", math_fmod}, diff --git a/src/test/lua/sandbox_config.lua b/src/test/lua/sandbox_config.lua index 7c71b79..c54d1cc 100644 --- a/src/test/lua/sandbox_config.lua +++ b/src/test/lua/sandbox_config.lua @@ -66,6 +66,8 @@ local tests = { cos=1, cosh=1, deg=1, + erf=1, + erfc=1, exp=1, floor=1, fmod=1, From 02f882640890b8c092e91cc65976dfd4c35f18c6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 14 Jul 2016 11:45:42 -0700 Subject: [PATCH 174/235] Change the luasandboxConfig.cmake installation path to share - centos x64 could not locate it in its lib64 installation path - update the DEB package dependencies --- CMakeLists.txt | 4 ++-- cmake/mozsvc.cmake | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5860ad8..9f19556 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.0.1 LANGUAGES C) +project(luasandbox VERSION 1.0.2 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) @@ -17,7 +17,7 @@ if(WIN32) set(INSTALL_CMAKE_DIR cmake) set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) else() - set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_LIBDIR}/${PROJECT_NAME}/cmake") + set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}") endif() set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index b4b2ce7..3927d64 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -45,6 +45,7 @@ endif() set(CPACK_PACKAGE_VENDOR "Mozilla Services") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE.txt") set(CPACK_STRIP_FILES TRUE) +set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) include(CPack) include(CTest) From fe9d26eb6d5f7e44efaa1d0353098e4f9897f3aa Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 19 Jul 2016 13:28:14 -0700 Subject: [PATCH 175/235] Fix up the build/install on RedHat - Move the cmake configuration file to a directory we own - prevent possible RPM install directory permission conflicts - remove the indentation causing a build warning -> error on FC24 --- CMakeLists.txt | 2 +- src/lua/ltablib.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f19556..f0524ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ if(WIN32) set(INSTALL_CMAKE_DIR cmake) set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) else() - set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_DATAROOTDIR}/cmake/${PROJECT_NAME}") + set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake") endif() set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") diff --git a/src/lua/ltablib.c b/src/lua/ltablib.c index b6d9cb4..14eca90 100644 --- a/src/lua/ltablib.c +++ b/src/lua/ltablib.c @@ -16,7 +16,7 @@ #include "lualib.h" -#define aux_getn(L,n) (luaL_checktype(L, n, LUA_TTABLE), luaL_getn(L, n)) +#define aux_getn(L,n) (luaL_checktype(L, n, LUA_TTABLE), luaL_getn(L, n)) static int foreachi (lua_State *L) { @@ -137,7 +137,7 @@ static void addfield (lua_State *L, luaL_Buffer *b, int i) { if (!lua_isstring(L, -1)) luaL_error(L, "invalid value (%s) at index %d in table for " LUA_QL("concat"), luaL_typename(L, -1), i); - luaL_addvalue(b); + luaL_addvalue(b); } From 2062baa849564362fc5b6a707a436899144a80d9 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 21 Jul 2016 14:52:46 -0700 Subject: [PATCH 176/235] lsb_terminate should not close the lua_State This avoids some race conditions when the sandbox is used by a threaded host, instead everything is cleaned up in destroy. --- CMakeLists.txt | 2 +- cmake/mozsvc.cmake | 6 +++++- include/luasandbox.h | 2 +- src/luasandbox.c | 19 +++++++++---------- src/luasandbox_serialize.c | 2 +- src/test/test_generic_sandbox.c | 6 +++++- src/util/test/test_util.c | 2 +- 7 files changed, 23 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f0524ca..41c8a25 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.0.2 LANGUAGES C) +project(luasandbox VERSION 1.0.3 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 3927d64..cf9c0ea 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -44,7 +44,11 @@ endif() set(CPACK_PACKAGE_VENDOR "Mozilla Services") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE.txt") -set(CPACK_STRIP_FILES TRUE) +if(CMAKE_BUILD_TYPE MATCHES "^[Dd][Ee][Bb][Uu][Gg]$") + set(CPACK_STRIP_FILES FALSE) +else() + set(CPACK_STRIP_FILES TRUE) +endif() set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) include(CPack) diff --git a/include/luasandbox.h b/include/luasandbox.h index 47e140e..1bc3221 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -242,7 +242,7 @@ lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name); LSB_EXPORT void lsb_pcall_teardown(lsb_lua_sandbox *lsb); /** - * Shutdown the sandbox due to a fatal error. + * Change the sandbox state to LSB_TERMINATED due to a fatal error. * * @param lsb Pointer to the sandbox. * @param err Reason for termination diff --git a/src/luasandbox.c b/src/luasandbox.c index be972d1..7e3181e 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -646,7 +646,12 @@ char* lsb_destroy(lsb_lua_sandbox *lsb) strcpy(err, lsb->error_message); } } - lsb_terminate(lsb, NULL); + + if (lsb->lua) { + lua_close(lsb->lua); + lsb->lua = NULL; + } + lsb_free_output_buffer(&lsb->output); free(lsb->state_file); free(lsb->lua_file); @@ -658,7 +663,7 @@ char* lsb_destroy(lsb_lua_sandbox *lsb) size_t lsb_usage(lsb_lua_sandbox *lsb, lsb_usage_type utype, lsb_usage_stat ustat) { - if (!lsb || !lsb->lua || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { + if (!lsb || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { return 0; } return lsb->usage[utype][ustat]; @@ -712,7 +717,7 @@ void lsb_add_function(lsb_lua_sandbox *lsb, lua_CFunction func, const char *func_name) { if (!lsb || !func || !func_name) return; - if (!lsb->lua) return; + if (lsb->state == LSB_TERMINATED) return; lua_pushcfunction(lsb->lua, func); lua_setglobal(lsb->lua, func_name); @@ -722,7 +727,7 @@ void lsb_add_function(lsb_lua_sandbox *lsb, lua_CFunction func, lsb_err_value lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name) { if (!lsb || !func_name) return LSB_ERR_UTIL_NULL; - if (!lsb->lua) return LSB_ERR_TERMINATED; + if (lsb->state == LSB_TERMINATED) return LSB_ERR_TERMINATED; if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, @@ -765,11 +770,5 @@ void lsb_terminate(lsb_lua_sandbox *lsb, const char *err) strncpy(lsb->error_message, err, LSB_ERROR_SIZE); lsb->error_message[LSB_ERROR_SIZE - 1] = 0; } - - if (lsb->lua) { - lua_close(lsb->lua); - lsb->lua = NULL; - } - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = 0; lsb->state = LSB_TERMINATED; } diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 350e1f4..2dc1d93 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -369,7 +369,7 @@ serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) { - if (!lsb->lua || !lsb->state_file) { + if (!lsb->lua || !lsb->state_file || lsb->state == LSB_TERMINATED) { return NULL; } lua_sethook(lsb->lua, NULL, 0, 0); diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c index b898a75..773f3ce 100644 --- a/src/test/test_generic_sandbox.c +++ b/src/test/test_generic_sandbox.c @@ -170,6 +170,8 @@ static char* test_api_assertion() lsb_add_function(NULL, NULL, NULL); lsb_pcall_teardown(NULL); lsb_terminate(NULL, NULL); + lsb_terminate(sb, NULL); + lsb_add_function(sb, lsb_test_write_output, "write_output"); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); @@ -317,8 +319,10 @@ static char* test_usage_error() mu_assert(sb, "lsb_create() received: NULL"); lsb_terminate(sb, "forced termination"); + lsb_state s = lsb_get_state(sb); + mu_assert(s == LSB_TERMINATED, "lsb_get_state() received: %d", s); u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "Terminated memory usage received: %" PRIuSIZE, u); + mu_assert(u > 0, "Terminated memory usage received: 0"); e = lsb_destroy(sb); mu_assert(!e, "lsb_destroy() received: %s", e); diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c index 01f96e5..f258e2f 100644 --- a/src/util/test/test_util.c +++ b/src/util/test/test_util.c @@ -103,7 +103,7 @@ static char* benchmark_lsb_get_timestamp() printf("benchmark_lsb_get_timestamp(%d) - clock %g seconds\n", iter, (double)t / CLOCKS_PER_SEC / iter); long long delta = lsb_get_timestamp() / 1000000000LL - time(NULL); - mu_assert(delta > 0 ? delta < 1 : delta > -1, "delta %lld", delta); + mu_assert(delta > 0 ? delta <= 1 : delta >= -1, "delta %lld", delta); return NULL; } From 370032de280eb78153d904b1b27c253ad63a7174 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 7 Aug 2016 15:21:18 -0700 Subject: [PATCH 177/235] lsb_outputs should not assume the string is NUL terminated --- CMakeLists.txt | 2 +- src/util/output_buffer.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41c8a25..536d8a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.0.3 LANGUAGES C) +project(luasandbox VERSION 1.0.4 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index 54f56a8..3bede46 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -151,8 +151,9 @@ lsb_err_value lsb_outputs(lsb_output_buffer *b, const char *str, size_t len) lsb_err_value ret = lsb_expand_output_buffer(b, len + 1); if (ret) return ret; - memcpy(b->buf + b->pos, str, len + 1); + memcpy(b->buf + b->pos, str, len); b->pos += len; + b->buf[b->pos] = 0; return ret; } From 42df2af0c439a3100840c8f1eaea154bcc46e7f9 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 11 Aug 2016 09:44:43 -0700 Subject: [PATCH 178/235] Message matcher updates - add Lua string unescaping support to the lsb_heka_cat message matcher - fix the >= string comparison in the matcher for strings with embedded NULs --- CMakeLists.txt | 2 +- docs/util/message_matcher.md | 1 + include/luasandbox/util/string.h | 23 ++++- src/cli/lsb_heka_cat.c | 20 +++- src/util/heka_message_matcher.c | 2 +- src/util/string.c | 68 ++++++++++++++ src/util/test/CMakeLists.txt | 5 + src/util/test/test_heka_message_matcher.c | 4 +- src/util/test/test_string.c | 107 ++++++++++++++++++++++ 9 files changed, 224 insertions(+), 8 deletions(-) create mode 100644 src/util/test/test_string.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 536d8a9..79c85a8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.0.4 LANGUAGES C) +project(luasandbox VERSION 1.0.5 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/util/message_matcher.md b/docs/util/message_matcher.md index 2dc0fe2..5f49d12 100644 --- a/docs/util/message_matcher.md +++ b/docs/util/message_matcher.md @@ -82,3 +82,4 @@ All message variables must be on the left hand side of the relational comparison ## Additional Restrictions * Message matchers are restricted to 128 relational comparisons +* A NUL character '\0' is not allowed in a matcher string diff --git a/include/luasandbox/util/string.h b/include/luasandbox/util/string.h index 2d334d8..e4e5070 100644 --- a/include/luasandbox/util/string.h +++ b/include/luasandbox/util/string.h @@ -19,17 +19,36 @@ typedef struct lsb_const_string #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif /** - * Initializes the struct to zero + * Initializes the struct to zero. * * @param s Pointer to the struct * */ LSB_UTIL_EXPORT void lsb_init_const_string(lsb_const_string *s); + +/** + * Lua string unescape. The returned string is always NUL terminated, but can + * contain other NULs in its body. + * + * @param d Pointer to the destination array where the content is to be + * unescaped. + * @param s C string to be unescaped + * @param dlen The length of the destination array (must be 1 byte larger than + * the source string (for inclusion of the NUL terminator). After + * successful conversion the final length of the escaped string is + * written back to this value as it may not equal strlen(d). + * + * @return char* A pointer to d or NULL on error. + */ +LSB_UTIL_EXPORT +char* lsb_lua_string_unescape(char *d, const char *s, size_t *dlen); + #ifdef __cplusplus } #endif diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 0f7d37d..9f12c10 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -245,7 +245,18 @@ static int find_header(const char *cur, int clen, const char *prev, int num) } char flag; - int hend = i + 2 + hlen; + int hend = i + 2; + if (hend < clen) { + flag = cur[hend]; + } else { + flag = prev[hend - clen]; + } + + if (flag != 0x08) { + continue; + } + + hend += hlen; if (hend < clen) { flag = cur[hend]; } else { @@ -290,7 +301,7 @@ static void move_to_offset(FILE *fh, int num) int loc = find_header(cur, consume, prev, num); if (loc >= 0) { - if (fseek(fh, -(consume - loc) , SEEK_CUR)) { + if (fseek(fh, -(consume - loc), SEEK_CUR)) { log_cb(NULL, NULL, 0, "fseek failed (find position)"); } return; @@ -376,7 +387,10 @@ int main(int argc, char **argv) return EXIT_FAILURE; } - lsb_message_matcher *mm = lsb_create_message_matcher(matcher); + char ms[strlen(matcher) + 1]; + size_t len = sizeof(ms); + lsb_message_matcher *mm = lsb_create_message_matcher( + lsb_lua_string_unescape(ms, matcher, &len)); if (!mm) { log_cb(NULL, NULL, 0, "invalid message matcher: %s", matcher); return EXIT_FAILURE; diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c index 4b6dcb6..a28b266 100644 --- a/src/util/heka_message_matcher.c +++ b/src/util/heka_message_matcher.c @@ -49,7 +49,7 @@ static bool string_test(match_node *mn, lsb_const_string *val) { int cmp = strncmp(val->s, mn->value.s, val->len); if (cmp == 0) { - return val->len == mn->value_len; + return val->len >= mn->value_len; } return cmp > 0; } diff --git a/src/util/string.c b/src/util/string.c index 848e74c..edfe5b0 100644 --- a/src/util/string.c +++ b/src/util/string.c @@ -6,6 +6,10 @@ /** String functions @file */ +#include +#include +#include + #include "luasandbox/util/string.h" void lsb_init_const_string(lsb_const_string *s) @@ -13,3 +17,67 @@ void lsb_init_const_string(lsb_const_string *s) s->s = NULL; s->len = 0; } + + +char* lsb_lua_string_unescape(char *d, const char *s, size_t *dlen) +{ + if (!s || !d || !dlen || *dlen <= strlen(s)) { + return NULL; + } + + int x = 0; + int y = 0; + while (s[x]) { + switch (s[x]) { + case '\\': + ++x; + switch (s[x]) { + case 'a': + d[y++] = '\a'; break; + case 'b': + d[y++] = '\b'; break; + case 'f': + d[y++] = '\f'; break; + case 'n': + d[y++] = '\n'; break; + case 'r': + d[y++] = '\r'; break; + case 't': + d[y++] = '\t'; break; + case 'v': + d[y++] = '\v'; break; + default: + if (!isdigit(s[x])) { + switch (s[x]) { + case '"': + case '\'': + case '?': + case '\\': + break; + default: + return NULL; + } + d[y++] = s[x]; + } else { /* \xxx */ + int n = 0; + int c = 0; + do { + c = 10 * c + (s[x++] - '0'); + } while (++n < 3 && isdigit(s[x])); + if (c > UCHAR_MAX) return NULL; + d[y++] = (char)c; + --x; + } + } + ++x; + break; + + default: + d[y++] = s[x++]; + break; + } + } + d[y] = 0; + *dlen = y; + return d; +} diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt index c247612..1ac3bf0 100644 --- a/src/util/test/CMakeLists.txt +++ b/src/util/test/CMakeLists.txt @@ -34,6 +34,10 @@ add_executable(test_heka_message_matcher test_heka_message_matcher.c) target_link_libraries(test_heka_message_matcher luasandboxutil) add_test(NAME test_heka_message_matcher COMMAND test_heka_message_matcher) +add_executable(test_string test_string.c) +target_link_libraries(test_string luasandboxutil) +add_test(NAME test_string COMMAND test_string) + if(WIN32) set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src/util") set_tests_properties(test_input_buffer PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) @@ -44,4 +48,5 @@ if(WIN32) set_tests_properties(test_util PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) set_tests_properties(test_heka_message PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) set_tests_properties(test_heka_message_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_string PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) endif() diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index cfbda04..286de36 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -18,7 +18,7 @@ // {"Logger":"GoSpec","Uuid":"xxx","Pid":32157,"Severity":6,"EnvVersion":"0.8","Fields":[{""value":["bar"],"name":"foo","value_type":0},{"value":[64],"name":"number","value_type":2},{"value":["data"],"name":"bytes","value_type":1},{"value":[999,1024],"name":"int","value_type":2},{"value":[99.9],"name":"double","value_type":3},{"value":[true],"name":"bool","value_type":4},{"value":["alternate"],"name":"foo","value_type":0},{"value":["name=test;type=web;"],"name":"Payload","value_type":0},{"representation":"date-time","value":["Mon Jan 02 15:04:05 -0700 2006"],"name":"Timestamp","value_type":0},{"value":[0],"name":"zero","value_type":2},{"value":["43"],"name":"string","value_type":0}],"Payload":"Test Payload","Timestamp":1.428773426113e+18,"Hostname":"trink-x230","Type":"TEST"} -char pb[] = "\x0a\x10\x27\x88\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x0c\x54\x65\x73\x74\x20\x50\x61\x79\x6c\x6f\x61\x64\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; +char pb[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x0c\x54\x65\x73\x74\x20\x50\x61\x79\x6c\x6f\x61\x64\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; size_t pblen = sizeof(pb); @@ -96,6 +96,8 @@ static char* test_true_matcher() , "Fields[foo] <= 'barx'" , "Fields[foo] < 'barx'" , "Fields[foo] >= 'bar'" + , "Uuid > '#'" + , "Uuid >= '#'" , "Fields[foo] > 'baq'" , "Fields[foo] != 'bara'" , "Fields[bytes] == 'data'" diff --git a/src/util/test/test_string.c b/src/util/test/test_string.c new file mode 100644 index 0000000..3459cfd --- /dev/null +++ b/src/util/test/test_string.c @@ -0,0 +1,107 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include + +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/string.h" + +static char* test_stub() +{ + return NULL; +} + +typedef struct { + char *s; + char *r; + size_t len; +} testcase; + + +static char* test_success() +{ + testcase tests[] = { + {"TRUE" ,"TRUE" ,4}, + {"\\a" ,"\a" ,1}, + {"\\b" ,"\b" ,1}, + {"\\f" ,"\f" ,1}, + {"\\n" ,"\n" ,1}, + {"\\r" ,"\r" ,1}, + {"\\t" ,"\t" ,1}, + {"\\v" ,"\v" ,1}, + {"\\\"" ,"\"" ,1}, + {"\\'" ,"'" ,1}, + {"\\\\" ,"\\" ,1}, + {"\\?" ,"?" ,1}, + {"\\1" ,"\1" ,1}, + {"\\33" ,"!" ,1}, + {"\\109" ,"m" ,1}, + {"+\\99\\1009+" ,"+cd9+" ,5}, + {"1\\000M" ,"1\x00M" ,3}, + }; + + for (unsigned i = 0; i < sizeof tests / sizeof(testcase); ++i){ + testcase *tc = &tests[i]; + char d[strlen(tc->s) + 1]; + size_t len = sizeof(d); + char *us = lsb_lua_string_unescape(d, tc->s, &len); + mu_assert(us, "test: %d valid string: %s", i, tc->s); + mu_assert(len == tc->len, "test: %d string: %s expected len: %" PRIuSIZE + " received: %" PRIuSIZE, i, tc->s, tc->len, len); + mu_assert(memcmp(us, tc->r, len) == 0, "test: %d memcmp string: %s", i, + tc->s); + } + return NULL; +} + + +static char* test_failure() +{ + testcase tests[] = { + {"\\p" ,NULL ,2}, // invalid escape char + {"\\999" ,NULL ,4}, // invalid char value + {"foo" ,NULL ,2}, // destination too small + {NULL ,NULL ,2}, // NULL source + }; + + for (unsigned i = 0; i < sizeof tests / sizeof(testcase); ++i){ + testcase *tc = &tests[i]; + char d[tc->len + 1]; + size_t len = sizeof(d); + mu_assert(!lsb_lua_string_unescape(d, tc->s, &len), + "test: %d invalid string: %s", i, tc->s); + } + mu_assert(!lsb_lua_string_unescape(NULL, tests[0].s, &tests[0].len), + "NULL destination"); + + char d[3]; + mu_assert(!lsb_lua_string_unescape(d, tests[0].s, NULL), "NULL length"); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_success); + mu_run_test(test_failure); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} From d486fef4c252896c83b472498f0258e784b6b19c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 23 Aug 2016 11:14:21 -0700 Subject: [PATCH 179/235] Issue #153 Add a clean stop API to Heka input sandboxes --- CMakeLists.txt | 2 +- README.md | 2 +- docs/heka/index.md | 13 +++++------ docs/heka/input.md | 12 ++++++++++ docs/index.md | 2 +- include/luasandbox.h | 16 +++++++++++-- include/luasandbox/heka/sandbox.h | 21 ++++++++++++++---- src/heka/sandbox.c | 28 +++++++++++++++++++++++ src/heka/test/lua/input.lua | 5 +++++ src/heka/test/test_heka_sandbox.c | 37 ++++++++++++++++++++++++++++--- src/luasandbox.c | 15 ++++++++----- src/util/heka_message.c | 7 +----- 12 files changed, 129 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79c85a8..de512bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.0.5 LANGUAGES C) +project(luasandbox VERSION 1.1.0 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/README.md b/README.md index 2eed7a0..b20b722 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ environment including functionality like global data preservation/restoration on shutdown/startup, output collection in textual or binary formats and an array of parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) -These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/trink/hindsight). +These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/mozilla-services/hindsight). The goal was to decouple the Heka/Hindsight functionality from any particular infrastructure and make it embeddable into any tool or language. diff --git a/docs/heka/index.md b/docs/heka/index.md index 62ec350..2a898b7 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -2,14 +2,12 @@ ## Overview -This document describes the pending 1.0 release of the Heka sandbox API built on -the [Generic Sandbox Interface](../sandbox.html). The 1.0 release is a refined +This document describes the 1.0 release of the Heka sandbox API built on the +[Generic Sandbox Interface](../sandbox.html). The 1.0 release is a refined implementation of its predecessor which was developed in [Heka](https://github.com/mozilla-services/heka). The goal is to decople it from -Go and make it easily embeddable in any language. We are in the process of -porting it back to the Heka Go implementation but a new design of that -infrastructure has been created to replace it called -[Hindsight](https://github.com/trink/hindsight). +Go and make it easily embeddable in any language. The Go version of Heka has +been deprecated and replaced by [Hindsight](https://github.com/mozilla-services/hindsight). ## Sandbox API Changes from the Go Heka Sandbox @@ -39,7 +37,8 @@ There are a few intentional changes between tho original Heka sandbox and this v #### Input Sandbox -1. A [create_stream_reader](input.html#create_stream_reader) function was added. +1. [create_stream_reader](input.html#create_stream_reader) function was added. +1. [is_running](input.html#is_running) function was added. #### Output Sandbox diff --git a/docs/heka/input.md b/docs/heka/input.md index 0785f6b..ddd0631 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -41,6 +41,18 @@ Provides access to the sandbox configuration variables. *Return* * value (string, number, bool, table) +### is_running + +Provides a synchronization point for collecting statistics and communicating +shutdown status. + +*Arguments* +* none + +*Return* +* running (boolean) - true if a sandbox state is LSB_RUNNING, false if not or + throws an error if the request to the host fails. + ### decode_message Converts a Heka protobuf encoded message string into a Lua table. diff --git a/docs/index.md b/docs/index.md index f0340dd..ef1bc66 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,7 +13,7 @@ environment including functionality like global data preservation/restoration on shutdown/startup, output collection in textual or binary formats and an array of parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) -These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/trink/hindsight). +These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/mozilla-services/hindsight). The goal was to decouple the Heka/Hindsight functionality from any particular infrastructure and make it embeddable into any tool or language. diff --git a/include/luasandbox.h b/include/luasandbox.h index 1bc3221..70b49de 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -40,7 +40,8 @@ typedef enum { LSB_UNKNOWN = 0, LSB_RUNNING = 1, - LSB_TERMINATED = 2 + LSB_TERMINATED = 2, + LSB_STOP = 3 } lsb_state; typedef enum { @@ -123,9 +124,20 @@ lsb_create(void *parent, const char *lua_file, const char *cfg, LSB_EXPORT lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file); +/** + * Changes the sandbox state to LSB_STOP to allow for a clean exit. This call is + * not thread safe. + * + * @param lsb sandbox to clean stop + * + * @return + * + */ +LSB_EXPORT void lsb_stop_sandbox_clean(lsb_lua_sandbox *lsb); + /** * Aborts the running sandbox from a different thread of execution. A "shutting - * down" termination message is generated. + * down" Lua error message is generated. * * @param lsb sandbox to abort * diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 439c439..8df9fd3 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -72,8 +72,11 @@ LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; /** * inject_message callback function provided by the host. Only one (or neither) * of the checkpoint values will be set in a call. Numeric checkpoints can have - * a measurable performance benefit but are not always applicable so both option - * are provided to support various types of input plugins. + * a measurable performance benefit but are not always applicable so both + * options are provided to support various types of input plugins. This function + * can be called with a NULL pb pointer by the 'is_running' API; it should be + * treated as a no-op and used as a synchronization point to collect statistics + * and communicate shutdown status. * * @param parent Opaque pointer the host object owning this sandbox * @param pb Pointer to a Heka protobuf encoded message being injected. @@ -240,13 +243,23 @@ int lsb_heka_pm_output(lsb_heka_sandbox *hsb, lsb_heka_message *msg, void *sequence_id, bool profile); +/** + * Requests a long running input sandbox to stop. This call is not thread safe. + * + * @param hsb Heka sandbox to cleanly stop + * + * @return + * + */ +LSB_HEKA_EXPORT void +lsb_heka_stop_sandbox_clean(lsb_heka_sandbox *hsb); /** * Aborts the running sandbox from a different thread of execution. A "shutting * down" termination message is generated. Used to abort long runnning sandboxes * such as an input sandbox. * - * @param hsb Heka sandbox to abort + * @param hsb Heka sandbox to forcibly stop * * @return * @@ -338,7 +351,7 @@ LSB_HEKA_EXPORT const lsb_heka_message* lsb_heka_get_message(lsb_heka_sandbox *hsb); /** - * Retrieve the sandbox tyye. + * Retrieve the sandbox type. * * * @param hsb Heka sandbox * diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index e564c33..8d5973c 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -38,6 +38,25 @@ static const char *lsb_heka_message_matcher = "lsb.heka_message_matcher"; int heka_create_stream_reader(lua_State *lua); + +static int is_running(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); + } + // call inject_message with a NULL message/checkpoint (special case + // synchronization point) + if (hsb->cb.iim(hsb->parent, NULL, 0, NAN, NULL) != 0) { + return luaL_error(lua, "%s() failed: rejected by the callback", __func__); + } + lua_pushboolean(lua, lsb_heka_is_running(hsb)); + return 1; +} + + static int read_message(lua_State *lua) { lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); @@ -109,6 +128,7 @@ static int mm_create(lua_State *lua) return 1; } + static int inject_message_input(lua_State *lua) { lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); @@ -457,6 +477,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, // inject_payload is intentionally excluded from input plugins // you can construct whatever you need with inject_message lsb_add_function(hsb->lsb, heka_create_stream_reader, "create_stream_reader"); + lsb_add_function(hsb->lsb, is_running, "is_running"); if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { @@ -760,12 +781,19 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, } +void lsb_heka_stop_sandbox_clean(lsb_heka_sandbox *hsb) +{ + lsb_stop_sandbox_clean(hsb->lsb); +} + + void lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb) { lsb_stop_sandbox(hsb->lsb); } + void lsb_heka_terminate_sandbox(lsb_heka_sandbox *hsb, const char *err) { lsb_terminate(hsb->lsb, err); diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index a94f632..3e7156c 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -6,6 +6,7 @@ assert(read_config) assert(decode_message) assert(inject_message) assert(create_stream_reader) +assert(is_running) assert(not read_message) assert(not encode_message) assert(not update_checkpoint) @@ -34,6 +35,10 @@ function process_message(cp) return 0, "string" elseif cp == 7 then error(nil) + elseif cp == 8 then + assert(is_running(), "running") + elseif cp == 9 then + assert(not is_running(), "not running") end return 0, "no cp" end diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 66ff1ff..74b6f9b 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -38,6 +38,10 @@ static lsb_logger logger = { .context = NULL, .cb = dlog }; static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, const char *cp_string) { + if (!pb) { + return 0; + } + static int cnt = 0; struct im_result { const char *pb; @@ -328,7 +332,7 @@ static char* test_timer_event() lsb_heka_get_error(hsb)); mu_assert(1 == lsb_heka_timer_event(hsb, 2, false), "err: %s", lsb_heka_get_error(hsb)); - mu_assert(false == lsb_heka_is_running(hsb), "running"); + mu_assert(false == lsb_heka_is_running(hsb), "not running"); stats = lsb_heka_get_stats(hsb); mu_assert(0 == stats.im_cnt, "received %llu", stats.im_cnt); @@ -353,6 +357,32 @@ static char* test_timer_event() } +static char* test_clean_stop_input() +{ + static const char *state_file = "stop.data"; + remove(state_file); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + + mu_assert(true == lsb_heka_is_running(hsb), "running"); + int rv = lsb_heka_pm_input(hsb, 8, NULL, true); + const char *err = lsb_heka_get_error(hsb); + mu_assert(rv == 0, "error: %s expected: %d received: %d", err, 0, rv); + mu_assert(true == lsb_heka_is_running(hsb), "running"); + + lsb_heka_stop_sandbox_clean(hsb); + rv = lsb_heka_pm_input(hsb, 9, NULL, true); + err = lsb_heka_get_error(hsb); + mu_assert(rv == 0, "error: %s expected: %d received: %d", err, 0, rv); + mu_assert(false == lsb_heka_is_running(hsb), "not running"); + + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "received %s", e); + return NULL; +} + + static char* test_stop_input() { static const char *state_file = "stop.data"; @@ -416,10 +446,10 @@ static char* test_pm_error() }; struct pm_result results[] = { - { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:26: boom" }, + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:27: boom" }, { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, - { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:32: aaaaaaaaaaaaaaaaaaaaaaaaa" + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:33: aaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, // >max error message @@ -657,6 +687,7 @@ static char* all_tests() mu_run_test(test_create_analysis_sandbox); mu_run_test(test_create_output_sandbox); mu_run_test(test_timer_event); + mu_run_test(test_clean_stop_input); mu_run_test(test_stop_input); mu_run_test(test_pm_input); mu_run_test(test_pm_error); diff --git a/src/luasandbox.c b/src/luasandbox.c index 7e3181e..a4ce909 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -25,12 +25,6 @@ #include "luasandbox_impl.h" #include "luasandbox_serialize.h" -#ifdef _MSC_VER -#ifndef __func__ -#define __func__ __FUNCTION__ -#endif -#endif - lsb_err_id LSB_ERR_INIT = "already initialized"; lsb_err_id LSB_ERR_LUA = "lua error"; // use lsb_get_error for details lsb_err_id LSB_ERR_TERMINATED = "sandbox already terminated"; @@ -619,6 +613,15 @@ static void stop_hook(lua_State *lua, lua_Debug *ar) } +void lsb_stop_sandbox_clean(lsb_lua_sandbox *lsb) +{ + if (!lsb) { + return; + } + lsb->state = LSB_STOP; +} + + void lsb_stop_sandbox(lsb_lua_sandbox *lsb) { if (!lsb) { diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 8fd6afe..5b0fde1 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -13,15 +13,10 @@ #include #include +#include "../luasandbox_defines.h" #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" -#ifdef _MSC_VER -#ifndef __func__ -#define __func__ __FUNCTION__ -#endif -#endif - static size_t decode_header(char *buf, size_t len, size_t max_message_size) { if (*buf != 0x08) { From eeb152c1e81dc442f7ee156c0181c8d3da042472 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Mon, 12 Sep 2016 13:40:16 +0200 Subject: [PATCH 180/235] Add instructions on how to make a TGZ,DEB,RPM,ZIP --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b20b722..bee7ec8 100644 --- a/README.md +++ b/README.md @@ -59,3 +59,4 @@ infrastructure and make it embeddable into any tool or language. nmake ctest + cpack -G TGZ # (DEB|RPM|ZIP) From b8f2516a83958a921d5f59b040cee2a0e3253c03 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 26 Sep 2016 06:47:38 -0700 Subject: [PATCH 181/235] Refactor the Heka message creation into a util function --- CMakeLists.txt | 2 +- include/luasandbox/util/heka_message.h | 12 ++++++++++++ src/cli/lsb_heka_cat.c | 8 +++----- src/heka/message.c | 18 +++++++----------- src/util/heka_message.c | 11 +++++++++++ src/util/test/test_heka_message.c | 19 +++++++++++++++++++ 6 files changed, 53 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de512bf..77d85fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.1.0 LANGUAGES C) +project(luasandbox VERSION 1.1.1 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index 6d7e60f..5f71471 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -20,6 +20,7 @@ #define LSB_UUID_SIZE 16 #define LSB_UUID_STR_SIZE 36 #define LSB_HDR_FRAME_SIZE 3 +#define LSB_MIN_HDR_SIZE 14 #define LSB_MAX_HDR_SIZE (255 + LSB_HDR_FRAME_SIZE) #define LSB_UUID "Uuid" @@ -205,6 +206,17 @@ LSB_UTIL_EXPORT bool lsb_read_heka_field(const lsb_heka_message *m, LSB_UTIL_EXPORT lsb_err_value lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len); +/** + * Writes the Heka framing header to the specified buffer. + * + * @param buf Buffer to write the header to must be at least LSB_MIN_HDR_SIZE + * size. + * @param len Length of the message to encode into the header + * + * @return LSB_UTIL_EXPORT size_t + */ +LSB_UTIL_EXPORT size_t lsb_write_heka_header(char *buf, size_t len); + #ifdef __cplusplus } #endif diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 9f12c10..05b47ef 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -193,11 +193,9 @@ static void output_text(lsb_heka_message *msg) static void output_heka(lsb_heka_message *msg) { - static char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f - int hlen = lsb_pb_output_varint(header + 3, msg->raw.len) + 1; - header[1] = (char)hlen; - header[hlen + 2] = '\x1f'; - if (fwrite(header, hlen + 3, 1, stdout) != 1) { + static char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, msg->raw.len); + if (fwrite(header, hlen, 1, stdout) != 1) { log_cb(NULL, NULL, 0, "error outputting header"); exit(1); } diff --git a/src/heka/message.c b/src/heka/message.c index 00a724f..f909fbe 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -849,14 +849,12 @@ int heka_encode_message(lua_State *lua) lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len; if (framed) { - char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f - int hlen = lsb_pb_output_varint(header + 3, len) + 1; - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len + hlen + LSB_HDR_FRAME_SIZE; - header[1] = (char)hlen; - header[hlen + 2] = '\x1f'; + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, len); + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len + hlen; luaL_Buffer b; luaL_buffinit(lua, &b); - luaL_addlstring(&b, header, hlen + LSB_HDR_FRAME_SIZE); + luaL_addlstring(&b, header, hlen); luaL_addlstring(&b, output, len); luaL_pushresult(&b); } else { @@ -986,13 +984,11 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) lua_pushlstring(lua, m->raw.s, m->raw.len); } else if (strcmp(field, "framed") == 0) { { - char header[14] = "\x1e\x00\x08"; // up to 10 varint bytes and a \x1f - int hlen = lsb_pb_output_varint(header + 3, m->raw.len) + 1; - header[1] = (char)hlen; - header[hlen + 2] = '\x1f'; + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, m->raw.len); luaL_Buffer b; luaL_buffinit(lua, &b); - luaL_addlstring(&b, header, hlen + 3); + luaL_addlstring(&b, header, hlen); luaL_addlstring(&b, m->raw.s, m->raw.len); luaL_pushresult(&b); } diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 5b0fde1..552681e 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -564,3 +564,14 @@ lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) } return NULL; } + + +size_t lsb_write_heka_header(char *buf, size_t len) +{ + int hlen = lsb_pb_output_varint(buf + 3, len) + 1; + buf[hlen + 2] = '\x1f'; + buf[0] = '\x1e'; + buf[1] = (char)hlen; + buf[2] = '\x08'; + return LSB_HDR_FRAME_SIZE + hlen; +} diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 02ff853..2a5aa3e 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -334,6 +334,24 @@ static char* test_write_heka_uuid() } +static char* test_write_heka_header() +{ + + char header[LSB_MIN_HDR_SIZE]; + size_t hlen; + + for (unsigned i = 0; i < 7; ++i) { + hlen = lsb_write_heka_header(header, 1LL << (i * 8)); + mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); + } + hlen = lsb_write_heka_header(header, 1LL << 56); + mu_assert(hlen == 13, "received %" PRIuSIZE, hlen); + hlen = lsb_write_heka_header(header, 1LL << 63); + mu_assert(hlen == 14, "received %" PRIuSIZE, hlen); + return NULL; +} + + static char* all_tests() { mu_run_test(test_stub); @@ -344,6 +362,7 @@ static char* all_tests() mu_run_test(test_find_message); mu_run_test(test_read_heka_field); mu_run_test(test_write_heka_uuid); + mu_run_test(test_write_heka_header); return NULL; } From f3857696229d3e5f654ff04d0aa87779e71203fb Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Tue, 27 Sep 2016 15:59:02 +0200 Subject: [PATCH 182/235] Drop git requirement (#158) --- CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index de512bf..f3b9851 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,6 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake DESTINATION ${INSTALL_CMAKE_DIR}) find_library(LIBM_LIBRARY m) -find_package(Git REQUIRED) include_directories("${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}/include/luasandbox" ) install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) From dae2fcd8c0ea278765ac6deacfadbd648567d912 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 27 Sep 2016 13:58:26 -0700 Subject: [PATCH 183/235] Improve lsb_heka_cat Fields array output --- CMakeLists.txt | 2 +- src/cli/lsb_heka_cat.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 77d85fa..ca1e755 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.1.1 LANGUAGES C) +project(luasandbox VERSION 1.2.0 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 05b47ef..81eb328 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -121,6 +121,7 @@ static void output_text(lsb_heka_message *msg) p = read_string(wiretype, p, e, &cs); if (p) { fprintf(stdout, "%.*s", (int)cs.len, cs.s); + if (p < e) fprintf(stdout, "|"); } } } @@ -145,6 +146,7 @@ static void output_text(lsb_heka_message *msg) fprintf(stdout, "\\x%02hhx", (unsigned char)cs.s[i]); } } + if (p < e) fprintf(stdout, "|"); } } } @@ -156,6 +158,7 @@ static void output_text(lsb_heka_message *msg) p = lsb_pb_read_varint(p, e, &ll); if (p) { fprintf(stdout, "%lld", ll); + if (p < e) fprintf(stdout, "|"); } } } @@ -165,9 +168,7 @@ static void output_text(lsb_heka_message *msg) double d; for (int i = 0; p <= (e - sizeof(double)); p += sizeof(double), ++i) { memcpy(&d, p, sizeof(double)); - if (i > 0) { - fprintf(stdout, ","); - } + if (i > 0) fprintf(stdout, "|"); fprintf(stdout, "%.17g", d); } } @@ -179,6 +180,7 @@ static void output_text(lsb_heka_message *msg) p = lsb_pb_read_varint(p, e, &ll); if (p) { fprintf(stdout, "%s", ll == 0 ? "false" : "true"); + if (p < e) fprintf(stdout, "|"); } } } From 103307f11f341088d6bc5ea2faca73ced73e5414 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 28 Sep 2016 20:37:49 -0700 Subject: [PATCH 184/235] Log a warning when messages are discarded due to the max_message_size --- src/util/heka_message.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 552681e..64398e6 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -17,7 +17,10 @@ #include "luasandbox/util/output_buffer.h" #include "luasandbox/util/protobuf.h" -static size_t decode_header(char *buf, size_t len, size_t max_message_size) +static size_t decode_header(char *buf, + size_t len, + size_t max_message_size, + lsb_logger *logger) { if (*buf != 0x08) { return 0; @@ -29,6 +32,12 @@ static size_t decode_header(char *buf, size_t len, size_t max_message_size) if (lsb_pb_read_varint(p + 1, buf + len, &vi)) { if (vi > 0 && vi <= (long long)max_message_size) { return (size_t)vi; + } else { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, + "maximum (%lld) messages size exceeded: %lld", + (long long)max_message_size, vi); + } } } } @@ -277,7 +286,8 @@ bool lsb_decode_heka_message(lsb_heka_message *m, m->fields_size * sizeof(lsb_heka_field)); if (!tmp) { if (logger && logger->cb) { - logger->cb(logger->context, __func__, 0, "fields reallocation failed"); + logger->cb(logger->context, __func__, 0, + "fields reallocation failed"); } return false; } @@ -370,7 +380,7 @@ bool lsb_find_heka_message(lsb_heka_message *m, if (!ib->msglen) { ib->msglen = decode_header(&ib->buf[ib->scanpos + 2], hlen, - ib->maxsize - LSB_MAX_HDR_SIZE); + ib->maxsize - LSB_MAX_HDR_SIZE, logger); } if (ib->msglen) { From 9dde916d719111991cd66896b70f1d1bd25ce70e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 3 Oct 2016 11:01:09 -0700 Subject: [PATCH 185/235] Add zero copy support to the Heka message and file output (io.write) One less copy is actually more accurate. This was added to reduce the memory churn in output plugins for large messages. For Mozilla telemetry messages the output sandbox memory was reduced from: raw 1858 KiB to 25 (memory use is fixed in the zero copy case) framed 3210 KiB to 87 (memory use is still variable since the framing is created) For Nginx access log messages there was much less difference raw 65 KiB to 25 framed 65 KiB to 34 On my Lenovo P50 laptop I saw a 20 increase in throughput using zero copy. On an AWS c3.2xlarge there was no difference in throughput (I/O appers to be the gating factor in both cases) --- CMakeLists.txt | 2 +- docs/heka/analysis.md | 75 ++++++---- docs/heka/input.md | 3 +- docs/heka/output.md | 119 +++++---------- include/luasandbox/util/heka_message.h | 3 + include/luasandbox_output.h | 25 ++++ src/heka/CMakeLists.txt | 1 + src/heka/message.c | 41 +++-- src/heka/read_message_zc.c | 199 +++++++++++++++++++++++++ src/heka/sandbox.c | 146 +++++++++++++++++- src/heka/test/lua/decode_message.lua | 8 + src/heka/test/lua/read_message.lua | 43 ++++-- src/heka/test/lua/read_message_zc.lua | 83 +++++++++++ src/heka/test/test_heka_sandbox.c | 107 ++++++++++++- src/luasandbox_output.c | 21 +++ 15 files changed, 733 insertions(+), 143 deletions(-) create mode 100644 src/heka/read_message_zc.c create mode 100644 src/heka/test/lua/read_message_zc.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index ca1e755..0ab0a11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,7 +35,7 @@ configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in INSTALL_DESTINATION ${INSTALL_CMAKE_DIR} PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR) write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake - COMPATIBILITY AnyNewerVersion ) # todo change to SameMajorVersion after 1.0 + COMPATIBILITY SameMajorVersion) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake DESTINATION ${INSTALL_CMAKE_DIR}) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index ad1e68d..24b2256 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -16,7 +16,7 @@ Hindsight reference implementation. - the entire module is inaccessible - [os](http://www.lua.org/manual/5.1/manual.html#5.8) - getenv, execute, exit, remove, rename, setlocale, tmpname - + ## Required Lua Functions (called by the host) ### process_message @@ -33,7 +33,8 @@ Recommenation: specify this as a `message_matcher` configuration option. * status_code (number) * success (less than or equal to zero) * fatal error (greater than zero) -* status_message (optional: string) logged when the status code is less than zero +* status_message (optional: string) logged when the status code is less than + zero ### timer_event @@ -42,8 +43,9 @@ Called when the host timer expires or on shutdown. Recommendation: specify this as a `ticker_interval` configuration option. *Arguments* -* ns (number) - nanosecond timestamp of the function call (it is actually `time_t * 1e9` -to keep the timestamp units consistent so it will only have a one second resolution) +* ns (number) - nanosecond timestamp of the function call (it is actually + `time_t * 1e9` to keep the timestamp units consistent so it will only have a + one second resolution) * shutdown (bool) - true if timer_event is being called due to a shutdown *Return* @@ -63,12 +65,14 @@ Provides access to the sandbox configuration variables. ### read_message -Provides access to the Heka message data. Note that both fieldIndex and arrayIndex are zero-based -(i.e. the first element is 0) as opposed to Lua's standard indexing, which is one-based. +Provides access to the Heka message data. Note that both fieldIndex and +arrayIndex are zero-based (i.e. the first element is 0) as opposed to Lua's +standard indexing, which is one-based. *Arguments* * variableName (string) - * framed (returns the Heka message protobuf string including the framing header) + * framed (returns the Heka message protobuf string including the framing + header) * raw (returns the Heka message protobuf string) * size (returns the size of the raw Heka message protobuf string) * Uuid @@ -82,26 +86,35 @@ Provides access to the Heka message data. Note that both fieldIndex and arrayInd * Pid * Fields[*name*] * fieldIndex (unsigned) - only used in combination with the Fields variableName - use to retrieve a specific instance of a repeated field *name*; zero indexed + use to retrieve a specific instance of a repeated field *name*; + zero indexed * arrayIndex (unsigned) - only used in combination with the Fields variableName - use to retrieve a specific element out of a field containing an array; zero indexed + use to retrieve a specific element out of a field containing an array; + zero indexed +* zeroCopy (bool, optional default false) - returns a userdata place holder for + the message variable (only valid for string types). Non string headers + throw an error during construction, non string fields throw an error on + data retrieval. *Return* -* value (number, string, bool, nil depending on the type of variable requested) +* value (number, string, bool, nil, userdata depending on the type of variable + requested) ### decode_message -Converts a Heka protobuf encoded message string into a Lua table or throws an error. +Converts a Heka protobuf encoded message string into a Lua table or throws an +error. *Arguments* -* heka_pb (string) - Heka protobuf binary string +* heka_pb (string, userdata) - Heka protobuf binary string or a zero copy + userdata object containing a Heka protobuf binary string. *Return* -* msg ([Heka message table (array fields)](message.html#array-based-message-fields)) - with the value member always being an array (even if there is only a single item). - This format makes working with the output more consistent. The wide variation - in the inject table formats is to ease the construction of the message especially - when using an LPeg grammar transformation. +* msg ([Heka message table (array fields)](message.html#array-based-message-fields)) + with the value member always being an array (even if there is only a single + item). This format makes working with the output more consistent. The wide + variation in the inject table formats is to ease the construction of the + message especially when using an LPeg grammar transformation. ### inject_message @@ -121,9 +134,10 @@ appropriate configuration value. ### add_to_payload -Appends the arguments to the payload buffer for incremental construction of the final payload output -(`inject_payload` finalizes the buffer and sends the message to the infrastructure). This function -is a rename of the generic sandbox output function to improve the readability of the plugin code. +Appends the arguments to the payload buffer for incremental construction of the +final payload output (`inject_payload` finalizes the buffer and sends the +message to the infrastructure). This function is a rename of the generic sandbox +output function to improve the readability of the plugin code. *Arguments* * arg (number, string, bool, nil, supported userdata) @@ -133,11 +147,13 @@ is a rename of the generic sandbox output function to improve the readability of ### inject_payload -This is a wrapper function for `inject_message` that is included for backwards compatibility. The function -creates a new Heka message using the contents of the payload buffer (pre-populated with `add_to_payload`) and combined -with any additional payload_args passed here. The payload buffer is cleared after the injection. The `payload_type` -and `payload_name` arguments are two pieces of optional metadata stored is message fields. The resulting message is -structured like this: +This is a wrapper function for `inject_message` that is included for backwards +compatibility. The function creates a new Heka message using the contents of the +payload buffer (pre-populated with `add_to_payload`) and combined with any +additional payload_args passed here. The payload buffer is cleared after the +injection. The `payload_type` and `payload_name` arguments are two pieces of +optional metadata stored is message fields. The resulting message is structured +like this: ```lua msg = { Timestamp = @@ -154,8 +170,10 @@ Fields = { *Arguments* -* payload_type (optional: string, default "txt") - describes the content type of the injected payload data -* payload_name (optional: string, default "") - names the content to aid in downstream filtering +* payload_type (optional: string, default "txt") - describes the content type of + the injected payload data +* payload_name (optional: string, default "") - names the content to aid in + downstream filtering * arg3 (optional) -ame type restrictions as `add_to_payload` * ... * argN @@ -166,7 +184,8 @@ Fields = { ### Modes of Operation #### Lock Step -* Receives one call to `process_message`, operates on the message, and returns success (0) or failure (-1) +* Receives one call to `process_message`, operates on the message, and returns + success (0) or failure (-1) #### Example simple counter plugin ```lua diff --git a/docs/heka/input.md b/docs/heka/input.md index ddd0631..21b28d5 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -119,7 +119,8 @@ Note: this operation clears the internal stream reader buffer. ##### read_message -Provides access to the Heka message data within the reader object. +Provides access to the Heka message data within the reader object. The zeroCopy +flag is not accepted here. ```lua local ts = hsr:read_message("Timestamp") diff --git a/docs/heka/output.md b/docs/heka/output.md index a467cb4..167bff4 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -23,11 +23,12 @@ combination with a [message matcher](../util/message_matcher.html) expression. Recommenation: specify this as a `message_matcher` configuration option. *Arguments* -* sequence_id (optional: lightuserdata) - pass in when `async_buffer_size` is configured +* sequence_id (optional: lightuserdata) - pass in when `async_buffer_size` is + configured *Return* -* status_code (number) - see the [Modes of Operation](#modes-of-operation) for a detail explanation -of the return codes +* status_code (number) - see the [Modes of Operation](#modes-of-operation) for + a detail explanation of the return codes * fatal error (greater than zero) * success (0) * non fatal failure (-1) @@ -44,8 +45,9 @@ Called when the host timer expires or on shutdown. Recommendation: specify this as a `ticker_interval` configuration option. *Arguments* -* ns (number) - nanosecond timestamp of the function call (it is actually `time_t * 1e9` -to keep the timestamp units consistent so it will only have a one second resolution) +* ns (number) - nanosecond timestamp of the function call (it is actually + `time_t * 1e9` to keep the timestamp units consistent so it will only have a + one second resolution) * shutdown (bool) - true if timer_event is being called due to a shutdown *Return* @@ -65,7 +67,8 @@ Provides access to the sandbox configuration variables. ### read_message -Provides access to the Heka message data. See [read_message](analysis.html#read_message) for details. +Provides access to the Heka message data. +See [read_message](analysis.html#read_message) for details. ### decode_message @@ -89,19 +92,21 @@ Note: this operation uses the internal output buffer so it is goverened by the * framed (bool default: false) A value of true includes the framing header *Return* -* heka_pb (string) - Heka protobuf binary string, framed as specified or an error is thrown +* heka_pb (string) - Heka protobuf binary string, framed as specified or an + error is thrown ### create_message_matcher -Returns a Heka protocol buffer message matcher; used to dynamic filter messages sent to the output plugin. +Returns a Heka protocol buffer message matcher; used to dynamic filter messages +sent to the output plugin. *Arguments* * message_matcher [message matcher](../util/message_matcher.html) *Return* * message_matcher (userdata) - or an error is thrown - The message matcher object has one method `eval` that returns true if the current message matches, false - if it does not. + The message matcher object has one method `eval` that returns true if the + current message matches, false if it does not. #### Example @@ -118,11 +123,14 @@ timeout/shutdown. * none #### Asynchronous Mode -Advances the output checkpoint and optionally reports the number of failures that occured. +Advances the output checkpoint and optionally reports the number of failures +that occured. *Arguments* -* sequence_id (lightuserdata) - sequence_id for the message that was just successfully delivered/acknowledged -* failures (optional: integer) - number of failures that occured in the asynchronus processing (added to the failure count) +* sequence_id (lightuserdata) - sequence_id for the message that was just + successfully delivered/acknowledged +* failures (optional: integer) - number of failures that occured in the + asynchronus processing (added to the failure count) *Return* * none (throws an error on invalid arg types) @@ -131,16 +139,20 @@ Advances the output checkpoint and optionally reports the number of failures tha #### Lock Step -* `process_message` operates on the message and returns one of the following values: - * success (0) - the message was successfully processed and the output checkpoint is advanced +* `process_message` operates on the message and returns one of the following + values: + * success (0) - the message was successfully processed and the output + checkpoint is advanced * failure (-1) - the message was not successfully processed * the failure count is incremented * any optional error message is written to the log * the message is skipped * the checkpoint is advanced - * skip (-2) - the message was intentionally not processed and the checkpoint is advanced - * retry (-3) - the message was not successfully processed and the host will call `process_message` - again, with the same message, after a one second delay + * skip (-2) - the message was intentionally not processed and the checkpoint + is advanced + * retry (-3) - the message was not successfully processed and the host will + call `process_message` again, with the same message, after a one second + delay #### Example Payload Output @@ -193,16 +205,19 @@ end #### Batching -* `process_message` batches the message/transformation in memory or on disk and returns one of the following values: +* `process_message` batches the message/transformation in memory or on disk and + returns one of the following values: * batching (-4) - the message was successfully added to the batch * failure (-1) - the message cannot be batch * the failure count is incremented * any optional error message is written to the log * the message is skipped * skip (-2) - the message was intentionally not added to the batch - * retry (-3) - the message was not successfully added to the batch and the host will call `process_message` - again, with the same message, after a one second delay - * success (0) - the batch has been successfully committed and the output checkpoint is advanced to the most recent message + * retry (-3) - the message was not successfully added to the batch and the + host will call `process_message` again, with the same message, after a one + second delay + * success (0) - the batch has been successfully committed and the output + checkpoint is advanced to the most recent message #### Example Postgres Output @@ -387,63 +402,5 @@ to the destination and returns one of the following values: **MUST** be called to advance the checkpoint to that specific message #### Example Kafka Output +[kafka.lua](https://github.com/mozilla-services/lua_sandbox_extensions/blob/master/kafka/sandboxes/heka/output/kafka.lua) -```lua --- cfg -message_matcher = "TRUE" -output_limit = 8 * 1024 * 1024 -brokers = "localhost:9092" -ticker_interval = 60 -async_buffer_size = 20000 - -topic_constant = "test" -producer_conf = { - ["queue.buffering.max.messages"] = async_buffer_size, - ["batch.num.messages"] = 200, - ["message.max.bytes"] = output_limit, - ["queue.buffering.max.ms"] = 10, - ["topic.metadata.refresh.interval.ms"] = -1, -} -``` - -```lua --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local brokerlist = read_config("brokerlist") or error("brokerlist must be set") -local topic_constant = read_config("topic_constant") -local topic_variable = read_config("topic_variable") or "Logger" -local producer_conf = read_config("producer_conf") - -local producer = kafka.producer(brokerlist, producer_conf) - -function process_message(sequence_id) - local topic = topic_constant - if not topic then - topic = read_message(topic_variable) or "unknown" - end - producer:create_topic(topic) -- creates the topic if it does not exist - - producer:poll() - local ret = producer:send(topic, -1, sequence_id) -- sends the current message - - if ret ~= 0 then - if ret == 105 then - return -3, "queue full" -- retry - elseif ret == 90 then - return -1, "message too large" -- fail - elseif ret == 2 then - error("unknown topic: " .. topic) - elseif ret == 3 then - error("unknown partition") - end - end - - return -5 -- asynchronous checkpoint management -end - -function timer_event(ns) - producer:poll() -end -``` diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h index 5f71471..b6b4aab 100644 --- a/include/luasandbox/util/heka_message.h +++ b/include/luasandbox/util/heka_message.h @@ -33,6 +33,9 @@ #define LSB_PID "Pid" #define LSB_HOSTNAME "Hostname" #define LSB_FIELDS "Fields" +#define LSB_RAW "raw" +#define LSB_FRAMED "framed" +#define LSB_SIZE "size" typedef enum { LSB_PB_UUID = 1, diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index 859c2b5..6af7a99 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -44,6 +44,31 @@ LSB_EXPORT void lsb_add_output_function(lua_State *lua, lua_CFunction fp); */ LSB_EXPORT lua_CFunction lsb_get_output_function(lua_State *lua, int index); +/** + * Add a zero copy function to the environment table. The environment table must + * be on the top of the stack. This function will receive the userdata as a + * pointer on the Lua stack. + * + * ud_object* ud = (ud_object*)lua_touserdata(lua, -1); + * + * @param lua Pointer the Lua state. + * @param fp Function pointer to the zero copy function. + * + * @return int Number of segments (pointer and length for each) + */ +LSB_EXPORT void lsb_add_zero_copy_function(lua_State *lua, lua_CFunction fp); + + +/** + * Utility function to retrieve a user data zero copy function + * + * @param lua + * @param index + * + * @return lua_CFunction + */ +LSB_EXPORT lua_CFunction lsb_get_zero_copy_function(lua_State *lua, int index); + /** * Write an array of variables on the Lua stack to the output buffer. * diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt index daa8605..74af8b6 100644 --- a/src/heka/CMakeLists.txt +++ b/src/heka/CMakeLists.txt @@ -4,6 +4,7 @@ set(HEKA_SRC message.c +read_message_zc.c sandbox.c stream_reader.c ) diff --git a/src/heka/message.c b/src/heka/message.c index f909fbe..eb4966f 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -690,13 +690,30 @@ encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, int heka_decode_message(lua_State *lua) { int n = lua_gettop(lua); - if (n != 1 || lua_type(lua, 1) != LUA_TSTRING) { - return luaL_argerror(lua, 0, "must have one string argument"); - } + luaL_argcheck(lua, n == 1, n, "incorrect number of arguments"); size_t len; - const char *pbstr = lua_tolstring(lua, 1, &len); - if (len < 20) { + const char *pbstr; + int t = lua_type(lua, 1); + if (t == LUA_TSTRING) { + pbstr = lua_tolstring(lua, 1, &len); + } else if (t == LUA_TUSERDATA) { + lua_CFunction fp = lsb_get_zero_copy_function(lua, 1); + if (!fp) { + return luaL_argerror(lua, 1, "no zero copy support"); + } + int results = fp(lua); + if (results != 2 || lua_type(lua, 2) != LUA_TLIGHTUSERDATA) { + return luaL_error(lua, "invalid zero copy return"); + } + pbstr = lua_touserdata(lua, 2); + len = (size_t)lua_tointeger(lua, 3); + lua_pop(lua, results); + } else { + return luaL_typerror(lua, 1, "string or userdata"); + } + + if (!pbstr || len < 20) { return luaL_error(lua, "invalid message, too short"); } @@ -925,15 +942,20 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) { int n = lua_gettop(lua); if (n < 1 || n > 3) { - return luaL_error(lua, "read_message() incorrect number of arguments"); + return luaL_error(lua, "%s() incorrect number of arguments", __func__); } size_t field_len; const char *field = luaL_checklstring(lua, 1, &field_len); - int fi = (int)luaL_optinteger(lua, 2, 0); + int fi = luaL_optint(lua, 2, 0); luaL_argcheck(lua, fi >= 0, 2, "field index must be >= 0"); - int ai = (int)luaL_optinteger(lua, 3, 0); + int ai = luaL_optint(lua, 3, 0); luaL_argcheck(lua, ai >= 0, 3, "array index must be >= 0"); + if (!m || !m->raw.s) { + lua_pushnil(lua); + return 1; + } + if (strcmp(field, LSB_UUID) == 0) { if (m->uuid.s) { lua_pushlstring(lua, m->uuid.s, m->uuid.len); @@ -1016,7 +1038,8 @@ int heka_read_message(lua_State *lua, lsb_heka_message *m) break; } } else { - lua_pushnil(lua); + luaL_error(lua, "%s() field: '%s' not supported/recognized", __func__, + field); } } return 1; diff --git a/src/heka/read_message_zc.c b/src/heka/read_message_zc.c new file mode 100644 index 0000000..c76d6d0 --- /dev/null +++ b/src/heka/read_message_zc.c @@ -0,0 +1,199 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua Heka read message zero copy references @file */ + +#include +#include + +#include "lauxlib.h" +#include "lua.h" +#include "lualib.h" +#include "luasandbox_output.h" +#include "luasandbox_serialize.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/heka/sandbox.h" + +static const char *metatable_name = "lsb.read_message_zc"; + +typedef struct read_message_zc +{ + lsb_const_string name; + int fi; + int ai; + char field[]; +} read_message_zc; + + +static const lsb_heka_message* get_heka_message(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + luaL_error(lua, "invalid " LSB_HEKA_THIS_PTR); + } + return lsb_heka_get_message(hsb); +} + + +static lsb_const_string read_message(lua_State *lua, read_message_zc *zc) +{ + lsb_const_string ret = { NULL, 0 }; + const lsb_heka_message *m = get_heka_message(lua); + if (!m || !m->raw.s) { + return ret; + } + + if (strcmp(zc->name.s, LSB_RAW) == 0) { + if (m->raw.s) ret = m->raw; + } else if (strcmp(zc->name.s, LSB_PAYLOAD) == 0) { + if (m->payload.s) ret = m->payload; + } else if (strcmp(zc->name.s, LSB_LOGGER) == 0) { + if (m->logger.s) ret = m->logger; + } else if (strcmp(zc->name.s, LSB_TYPE) == 0) { + if (m->type.s) ret = m->type; + } else if (strcmp(zc->name.s, LSB_ENV_VERSION) == 0) { + if (m->env_version.s) ret = m->env_version; + } else if (strcmp(zc->name.s, LSB_HOSTNAME) == 0) { + if (m->hostname.s) ret = m->hostname; + } else if (strcmp(zc->name.s, LSB_UUID) == 0) { + if (m->uuid.s) ret = m->uuid; + } else if (zc->name.len >= 8 + && memcmp(zc->name.s, LSB_FIELDS "[", 7) == 0 + && zc->name.s[zc->name.len - 1] == ']') { + lsb_read_value v; + lsb_const_string f = { zc->name.s + 7, zc->name.len - 8 }; + lsb_read_heka_field(m, &f, zc->fi, zc->ai, &v); + if (v.type == LSB_READ_STRING) { + ret = v.u.s; + } else if (v.type != LSB_READ_NIL) { + luaL_error(lua, "%s() zc->name.s: '%s' contains an unsupported type", + __func__, zc->name.s); + } + } else { + luaL_error(lua, "%s() zc->name.s: '%s' not supported/recognized", __func__, + zc->name.s); + } + return ret; +} + + +static int zc_output(lua_State *lua) +{ + lsb_output_buffer *ob = lua_touserdata(lua, -1); + if (!ob) {return 1;} + read_message_zc *zc = luaL_checkudata(lua, -2, metatable_name); + lsb_const_string f = read_message(lua, zc); + if (!f.s) return 0; + + if (strcmp(zc->field, LSB_FRAMED) == 0) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, f.len); + if (lsb_outputs(ob, header, hlen)) return 1; + } + if (lsb_outputs(ob, f.s, f.len)) return 1; + return 0; +} + + +static int zc_return(lua_State *lua) +{ + read_message_zc *zc = luaL_checkudata(lua, -1, metatable_name); + lua_checkstack(lua, 3); + int cnt = 2; + lsb_const_string f = read_message(lua, zc); + if (strcmp(zc->field, LSB_FRAMED) == 0) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, f.len); + lua_pushlstring(lua, header, hlen); + ++cnt; + } + lua_pushlightuserdata(lua, (void *)f.s); + lua_pushinteger(lua, (int)f.len); + return cnt; +} + + +static int zc_tostring(lua_State *lua) +{ + read_message_zc *zc = luaL_checkudata(lua, -1, metatable_name); + lsb_const_string f = read_message(lua, zc); + if (f.s) { + if (strcmp(zc->field, LSB_FRAMED) == 0) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, f.len); + lua_pushlstring(lua, header, hlen); + lua_pushlstring(lua, f.s, f.len); + lua_concat(lua, 2); + } else { + lua_pushlstring(lua, f.s, f.len); + } + } else { + lua_pushnil(lua); + } + return 1; +} + + +static const struct luaL_reg zclib_m[] = +{ + { "__tostring", zc_tostring }, + { NULL, NULL } +}; + + +int heka_create_read_message_zc(lua_State *lua) +{ + size_t len; + const char *field = luaL_checklstring(lua, 1, &len); + int fi = luaL_optint(lua, 2, 0); + luaL_argcheck(lua, fi >= 0, 2, "field index must be >= 0"); + int ai = luaL_optint(lua, 3, 0); + luaL_argcheck(lua, ai >= 0, 3, "array index must be >= 0"); + + if (!(strcmp(field, LSB_UUID) == 0 + || strcmp(field, LSB_TYPE) == 0 + || strcmp(field, LSB_LOGGER) == 0 + || strcmp(field, LSB_PAYLOAD) == 0 + || strcmp(field, LSB_ENV_VERSION) == 0 + || strcmp(field, LSB_HOSTNAME) == 0 + || strcmp(field, LSB_RAW) == 0 + || strcmp(field, LSB_FRAMED) == 0 + || (len >= 8 + && memcmp(field, LSB_FIELDS "[", 7) == 0 + && field[len - 1] == ']'))) { + luaL_error(lua, "%s() field: '%s' not supported/recognized", __func__, + field); + } + + if (luaL_newmetatable(lua, metatable_name) == 1) { + lua_newtable(lua); + lsb_add_output_function(lua, zc_output); + lsb_add_zero_copy_function(lua, zc_return); + lua_replace(lua, LUA_ENVIRONINDEX); + + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, zclib_m); + } + + read_message_zc *zc = lua_newuserdata(lua, sizeof(read_message_zc) + len + 1); + zc->fi = fi; + zc->ai = ai; + memcpy(zc->field, field, len + 1); + if (strcmp(field, LSB_FRAMED) == 0) { + zc->name.s = LSB_RAW; + zc->name.len = sizeof(LSB_RAW) - 1; + } else { + zc->name.s = zc->field; + zc->name.len = len; + } + + lua_pushvalue(lua, -2); + lua_setmetatable(lua, -2); + return 1; +} diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 8d5973c..17a5b08 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -8,6 +8,7 @@ #include "luasandbox/heka/sandbox.h" +#include #include #include #include @@ -37,6 +38,7 @@ static const char *im_func_name = "inject_message"; static const char *lsb_heka_message_matcher = "lsb.heka_message_matcher"; int heka_create_stream_reader(lua_State *lua); +int heka_create_read_message_zc(lua_State *lua); static int is_running(lua_State *lua) @@ -65,10 +67,12 @@ static int read_message(lua_State *lua) if (!hsb) { return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); } - - if (!hsb->msg || !hsb->msg->raw.s) { - lua_pushnil(lua); - return 1; + if (lua_gettop(lua) == 4) { + luaL_checktype(lua, 4, LUA_TBOOLEAN); + if (lua_toboolean(lua, 4)) { + return heka_create_read_message_zc(lua); + } + lua_pop(lua, 1); // remove the zc flag } return heka_read_message(lua, hsb->msg); } @@ -709,6 +713,131 @@ int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, } +// IO write zero copy replacement +static int pushresult(lua_State *lua, int i, const char *filename) +{ + int en = errno; /* calls to Lua API may change this value */ + if (i) { + lua_pushboolean(lua, 1); + return 1; + } else { + lua_pushnil(lua); + if (filename) lua_pushfstring(lua, "%s: %s", filename, strerror(en)); + else lua_pushfstring(lua, "%s", strerror(en)); + lua_pushinteger(lua, en); + return 3; + } +} + + +static int zc_write(lua_State *lua, FILE *f, int arg) +{ + int n = lua_gettop(lua); + int nargs = n - 1; + int status = 1; + for (; nargs--; arg++) { + switch (lua_type(lua, arg)) { + case LUA_TNUMBER: + /* optimization: could be done exactly as for strings */ + status = status && + fprintf(f, LUA_NUMBER_FMT, lua_tonumber(lua, arg)) > 0; + break; + case LUA_TSTRING: + { + size_t l; + const char *s = luaL_checklstring(lua, arg, &l); + status = status && (fwrite(s, sizeof(char), l, f) == l); + } + break; + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_zero_copy_function(lua, arg); + if (!fp) { + return luaL_argerror(lua, arg, "no zero copy support"); + } + lua_pushvalue(lua, arg); + int results = fp(lua); + int start = n + 2; + int end = start + results; + size_t len; + const char *s; + for (int i = start; i < end; ++i) { + switch (lua_type(lua, i)) { + case LUA_TSTRING: + s = lua_tolstring(lua, i, &len); + if (s && len > 0) { + status = status && (fwrite(s, sizeof(char), len, f) == len); + } + break; + case LUA_TLIGHTUSERDATA: + s = lua_touserdata(lua, i++); + len = (size_t)lua_tointeger(lua, i); + if (s && len > 0) { + status = status && (fwrite(s, sizeof(char), len, f) == len); + } + break; + default: + return luaL_error(lua, "invalid zero copy return"); + } + } + lua_pop(lua, results + 1); // remove the returns values and + // the copy of the userdata + } + break; + default: + return luaL_typerror(lua, arg, "number, string or userdata"); + } + } + return pushresult(lua, status, NULL); +} + +static FILE* getiofile(lua_State *lua, int findex) +{ + static const char *const fnames[] = { "input", "output" }; + FILE *f; + lua_rawgeti(lua, LUA_ENVIRONINDEX, findex); + f = *(FILE **)lua_touserdata(lua, -1); + if (f == NULL) luaL_error(lua, "standard %s file is closed", + fnames[findex - 1]); + return f; +} + + +static int zc_io_write(lua_State *lua) +{ + return zc_write(lua, getiofile(lua, 2), 1); +} + + +static FILE* tofile(lua_State *lua) +{ + FILE **f = (FILE **)luaL_checkudata(lua, 1, LUA_FILEHANDLE); + if (*f == NULL) luaL_error(lua, "attempt to use a closed file"); + return *f; +} + + +static int zc_f_write(lua_State *lua) +{ + return zc_write(lua, tofile(lua), 2); +} + + +static int zc_luaopen_io(lua_State *lua) +{ + luaopen_io(lua); + lua_pushcclosure(lua, zc_io_write, 0); + lua_setfield(lua, -2, "write"); + + luaL_getmetatable(lua, LUA_FILEHANDLE); + lua_pushcclosure(lua, zc_f_write, 0); + lua_setfield(lua, -2, "write"); + lua_pop(lua, 1); // remove the metatable + return 1; +} +// End IO write zero copy replacement + + lsb_heka_sandbox* lsb_heka_create_output(void *parent, const char *lua_file, const char *state_file, @@ -762,6 +891,14 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_add_function(hsb->lsb, update_checkpoint, LSB_HEKA_UPDATE_CHECKPOINT); lsb_add_function(hsb->lsb, mm_create, "create_message_matcher"); + // start io.write override with zero copy functionality + lua_getfield(lua, LUA_REGISTRYINDEX, "_PRELOADED"); + lua_pushstring(lua, LUA_IOLIBNAME); + lua_pushcfunction(lua, zc_luaopen_io); + lua_rawset(lua, -3); + lua_pop(lua, 1); + // end io.write override + if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); @@ -855,7 +992,6 @@ int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) lsb_terminate(hsb->lsb, err); return 1; } - // todo change if we need more than 1 sec resolution lua_pushnumber(lua, t * 1e9); lua_pushboolean(lua, shutdown); diff --git a/src/heka/test/lua/decode_message.lua b/src/heka/test/lua/decode_message.lua index 65bcea2..037a563 100644 --- a/src/heka/test/lua/decode_message.lua +++ b/src/heka/test/lua/decode_message.lua @@ -208,3 +208,11 @@ local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\2 ok, msg = pcall(decode_message, test) assert(not ok) assert(msg == "error in tag: 3 wiretype: 2 offset: 24", msg) + + +function process_message() + msg = decode_message(read_message("raw", nil, nil, true)) + assert(msg.Timestamp == 1e9) + assert(#msg.Fields == 6) + return 0 +end diff --git a/src/heka/test/lua/read_message.lua b/src/heka/test/lua/read_message.lua index 9a6f501..a205593 100644 --- a/src/heka/test/lua/read_message.lua +++ b/src/heka/test/lua/read_message.lua @@ -19,21 +19,30 @@ local tests = { {"framed", 214}, {"Fields[string]", "string"}, {"Fields[notfound]", nil}, - {"Fields[string", nil}, - {"morethan8", nil}, - {"lt8", nil}, } local fields = { - {"number" , 0 , 0 , 1}, - {"numbers" , 0 , 2 , 3}, - {"bool" , 0 , 0 , true}, - {"bools" , 0 , 1 , false}, - {"strings" , 0 , 0 , "s1"}, - {"strings" , 0 , 2 , "s3"}, - {"strings" , 10, 0 , nil}, - {"strings" , 0 , 10, nil}, - {"notfound" , 0 , 0 , nil}, + {{"Fields[number]" , 0 , 0 }, 1}, + {{"Fields[number]" , 0 , 0, false }, 1}, + {{"Fields[numbers]" , 0 , 2 }, 3}, + {{"Fields[bool]" , 0 , 0 }, true}, + {{"Fields[bools]" , 0 , 1 }, false}, + {{"Fields[strings]" , 0 , 0 }, "s1"}, + {{"Fields[strings]" , 0 , 2 }, "s3"}, + {{"Fields[strings]" , 10, 0 }, nil}, + {{"Fields[strings]" , 0 , 10}, nil}, + {{"Fields[notfound]" , 0 , 0 }, nil}, +} + +local errors = { + {"Fields[string"}, + {"morethan8"}, + {"lt8"}, + {"Fields[string]", -1, 0}, + {"Fields[string]", 0, -1}, + {"Fields[string]", "a", 0}, + {"Fields[string]", 0, "a"}, + {"Fields[string]", 0, 0, "str"}, } @@ -50,8 +59,14 @@ function process_message() end for i, v in ipairs(fields) do - local r = read_message(string.format("Fields[%s]", v[1]), v[2], v[3]) - assert(v[4] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[4]), tostring(r))) + local r = read_message(unpack(v[1])) + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) end + + for i, v in ipairs(errors) do + local ok, r = pcall(read_message, unpack(v)) + assert(not ok, string.format("test: %d should have errored", i)) + end + return 0 end diff --git a/src/heka/test/lua/read_message_zc.lua b/src/heka/test/lua/read_message_zc.lua new file mode 100644 index 0000000..2b5b14a --- /dev/null +++ b/src/heka/test/lua/read_message_zc.lua @@ -0,0 +1,83 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +local tests = { + {read_message("Uuid", 0, 0, true), "abcdefghijklmnop"}, + {read_message("Type", nil, nil, true), "type"}, + {read_message("Logger", 0, 0, true), "logger"}, + {read_message("Payload", 0, 0, true), "payload"}, + {read_message("EnvVersion", 0, 0, true), "env_version"}, + {read_message("Hostname", 0, 0, true), "hostname"}, + {read_message("Fields[string]" , 0 , 0 , true), "string"}, + {read_message("Fields[notfound]", 0 , 0 , true), nil}, + {read_message("Fields[strings]" , 0 , 0 , true), "s1"}, + {read_message("Fields[strings]" , 0 , 2 , true), "s3"}, + {read_message("Fields[strings]" , 10, 0 , true), nil}, + {read_message("Fields[strings]" , 0 , 10, true), nil}, + {read_message("Fields[notfound]", 0 , 0 , true), nil}, +} + +local tests_size = { + {read_message("raw", 0 , 0 , true), 208}, + {read_message("framed", 0 , 0 , true), 214}, +} + +local creation_errors = { + {"Timestamp", 0, 0, true}, + {"Severity", 0, 0, true}, + {"Pid", 0, 0, true}, + {"size", 0, 0, true}, + {"Fields[string", 0, 0, true}, + {"morethan8", 0, 0, true}, + {"lt8", 0, 0, true}, + {"Fields[string]", -1, 0, true}, + {"Fields[string]", 0, -1, true}, + {"Fields[string]", "a", 0, true}, + {"Fields[string]", 0, "a", true}, + {"Fields[string]", 0, 0, true, 0}, +} + +local fetch_errors = { + read_message("Fields[bool]", 0, 0, true), + read_message("Fields[number]", 0, 0, true), +} + +function process_message() + for i, v in ipairs(tests) do + local r = read_message_zc(v[1]) + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + + for i, v in ipairs(tests) do + local r = tostring(v[1]) + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + + add_to_payload(read_message("Uuid", 0, 0, true)) + add_to_payload(read_message("framed", 0, 0, true)) + inject_payload() + + for i, v in ipairs(tests_size) do + local r = read_message_zc(v[1]) + assert(v[2] == #r, string.format("test: %d expected: %d received: %d", i, tostring(v[2]), tostring(#r))) + end + + for i, v in ipairs(tests_size) do + local r = tostring(v[1]) + assert(v[2] == #r, string.format("test: %d expected: %d received: %d", i, tostring(v[2]), tostring(#r))) + end + + for i, v in ipairs(creation_errors) do + local ok, r = pcall(read_message, unpack(v)) + assert(not ok, string.format("test: %d creation should have errored", i)) + end + + for i, v in ipairs(fetch_errors) do + local ok, r = pcall(read_message_zc, v) + assert(not ok, string.format("test: %d fetch should have errored", i)) + end + return 0 +end diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 74b6f9b..eed250f 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -8,13 +8,16 @@ #include "test.h" +#include #include #include #include #include #include +#include "../sandbox_impl.h" #include "luasandbox/heka/sandbox.h" +#include "luasandbox_output.h" // {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; @@ -176,6 +179,34 @@ static int aim(void *parent, const char *pb, size_t pb_len) return 0; } +static int aim1(void *parent, const char *pb, size_t pb_len) +{ + if (parent) { + fprintf(stderr, "parent set\n"); + return 1; + } + if (!pb) { + fprintf(stderr, "pb null set\n"); + return 1; + } + size_t expected = 306; + if (pb_len != expected) { + fprintf(stderr, "pb_len expected: %" PRIuSIZE " received: %" PRIuSIZE "\n", + expected, pb_len); + + for (size_t i = 0; i < pb_len; ++i) { + if (isprint(pb[i])) { + fprintf(stderr, "%c", pb[i]); + } else { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + } + fprintf(stderr, "\n"); + return 1; + } + return 0; +} + static int ucp(void *parent, void *sequence_id) { @@ -417,7 +448,7 @@ static char* test_pm_input() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); - for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ + for (unsigned i = 0; i < sizeof results / sizeof results[0];++i){ int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); const char *err = lsb_heka_get_error(hsb); mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", @@ -457,7 +488,7 @@ static char* test_pm_error() }; lsb_heka_sandbox *hsb; - for (unsigned i = 0; i < sizeof results / sizeof results[0]; ++i){ + for (unsigned i = 0; i < sizeof results / sizeof results[0];++i){ hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); mu_assert(hsb, "lsb_heka_create_input failed"); int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); @@ -604,9 +635,14 @@ static char* test_decode_message() { lsb_heka_message m; mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + lsb_heka_sandbox *hsb; hsb = lsb_heka_create_output(NULL, "lua/decode_message.lua", NULL, NULL, &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); + int rv = lsb_heka_pm_output(hsb, &m, NULL, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); e = lsb_heka_destroy_sandbox(hsb); lsb_free_heka_message(&m); return NULL; @@ -621,7 +657,69 @@ static char* test_read_message() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); - mu_assert(hsb, "lsb_heka_create_analysist failed"); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + int rv = lsb_heka_pm_analysis(hsb, &m, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static int rm_zc(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n == 1, n, "incorrect number of arguments"); + luaL_checktype(lua, 1, LUA_TUSERDATA); + lua_CFunction fp = lsb_get_zero_copy_function(lua, 1); + if (!fp) { + return luaL_argerror(lua, 1, "no zero copy support"); + } + int results = fp(lua); + int start = n + 1; + int end = start + results; + int cnt = 0; + for (int i = start; i < end; ++i) { + switch (lua_type(lua, i)) { + case LUA_TSTRING: + lua_pushvalue(lua, i); + ++cnt; + break; + case LUA_TLIGHTUSERDATA: + { + const char *s = lua_touserdata(lua, i++); + size_t len = (size_t)lua_tointeger(lua, i); + if (s && len > 0) { + lua_pushlstring(lua, s, len); + ++cnt; + } + } + break; + default: + return luaL_error(lua, "invalid zero copy return"); + } + } + if (cnt) { + lua_concat(lua, cnt); + } else { + lua_pushnil(lua); + } + return 1; +} + + +static char* test_read_message_zc() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message_zc.lua", NULL, NULL, &logger, aim1); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + + lsb_add_function(hsb->lsb, rm_zc, "read_message_zc"); int rv = lsb_heka_pm_analysis(hsb, &m, false); mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, lsb_heka_get_error(hsb)); @@ -647,7 +745,7 @@ static char* test_get_type() lsb_heka_sandbox *hsb; hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); char t = lsb_heka_get_type(NULL); - mu_assert( t == '\0', "received %c", t); + mu_assert(t == '\0', "received %c", t); t = lsb_heka_get_type(hsb); mu_assert(t == 'a', "received %c", t); e = lsb_heka_destroy_sandbox(hsb); @@ -699,6 +797,7 @@ static char* all_tests() mu_run_test(test_encode_message); mu_run_test(test_decode_message); mu_run_test(test_read_message); + mu_run_test(test_read_message_zc); mu_run_test(test_get_message); mu_run_test(test_get_type); diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c index 93c18ec..a477694 100644 --- a/src/luasandbox_output.c +++ b/src/luasandbox_output.c @@ -19,6 +19,7 @@ #include "luasandbox_serialize.h" static const char *output_function = "lsb_output"; +static const char *zero_copy_function = "lsb_zero_copy"; void lsb_add_output_function(lua_State *lua, lua_CFunction fp) { @@ -40,6 +41,26 @@ lua_CFunction lsb_get_output_function(lua_State *lua, int index) } +void lsb_add_zero_copy_function(lua_State *lua, lua_CFunction fp) +{ + lua_pushstring(lua, zero_copy_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +lua_CFunction lsb_get_zero_copy_function(lua_State *lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, zero_copy_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + + void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) { if (!append) { From 160890391ff0f62d9312787580be5aa39897c1c4 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 6 Oct 2016 06:39:10 -0700 Subject: [PATCH 186/235] Add access to the logger struct to aid in module debugging The specific use case for this addition is to connect the librdkafka log callback to the sandbox logger. --- include/luasandbox.h | 11 +++++++++++ src/luasandbox.c | 6 ++++++ src/test/test_generic_sandbox.c | 3 +++ 3 files changed, 20 insertions(+) diff --git a/include/luasandbox.h b/include/luasandbox.h index 70b49de..7718a63 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -223,6 +223,17 @@ LSB_EXPORT const char* lsb_get_lua_file(lsb_lua_sandbox *lsb); */ LSB_EXPORT void* lsb_get_parent(lsb_lua_sandbox *lsb); +/** + * Access the logger struct stored in the sandbox. The logger callback is only + * available to modules in debug mode (same as print). + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_logger Pointer to the logger struct + * + */ +LSB_EXPORT const lsb_logger* lsb_get_logger(lsb_lua_sandbox *lsb); + /** * Create a CFunction for use by the Sandbox. The Lua sandbox pointer is pushed * to upvalue index 1. diff --git a/src/luasandbox.c b/src/luasandbox.c index a4ce909..1c05c74 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -710,6 +710,12 @@ void* lsb_get_parent(lsb_lua_sandbox *lsb) } +const lsb_logger* lsb_get_logger(lsb_lua_sandbox *lsb) +{ + return lsb ? &lsb->logger : NULL; +} + + lsb_state lsb_get_state(lsb_lua_sandbox *lsb) { return lsb ? lsb->state : LSB_UNKNOWN; diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c index 773f3ce..373978b 100644 --- a/src/test/test_generic_sandbox.c +++ b/src/test/test_generic_sandbox.c @@ -161,6 +161,7 @@ static char* test_api_assertion() mu_assert(lsb_get_lua(NULL) == NULL, "not null"); mu_assert(lsb_get_lua_file(NULL) == NULL, "not null"); mu_assert(lsb_get_parent(NULL) == NULL, "not null"); + mu_assert(lsb_get_logger(NULL) == NULL, "not null"); mu_assert(lsb_get_state(NULL) == LSB_UNKNOWN, "not unknown"); lsb_add_function(NULL, lsb_test_write_output, "foo"); lsb_add_function(sb, NULL, "foo"); @@ -694,6 +695,8 @@ static char* test_print() }; lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;", &printer); mu_assert(sb, "lsb_create() received: NULL"); + mu_assert(lsb_get_logger(sb) != NULL, "no logger"); + mu_assert(lsb_get_logger(sb)->cb == printer.cb, "incorrect logger callback"); lsb_err_value ret = lsb_init(sb, NULL); mu_assert(!ret, "lsb_init() received: %s", ret); From 59542bfcd1904031431dc03a3a888636e413d235 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 6 Oct 2016 08:08:03 -0700 Subject: [PATCH 187/235] Pin the hostname in the unit test output comparison --- src/heka/test/test_heka_sandbox.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index eed250f..69a2005 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -716,7 +716,7 @@ static char* test_read_message_zc() mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); lsb_heka_sandbox *hsb; - hsb = lsb_heka_create_analysis(NULL, "lua/read_message_zc.lua", NULL, NULL, &logger, aim1); + hsb = lsb_heka_create_analysis(NULL, "lua/read_message_zc.lua", NULL, "Hostname = 'ubuntu'", &logger, aim1); mu_assert(hsb, "lsb_heka_create_analysis failed"); lsb_add_function(hsb->lsb, rm_zc, "read_message_zc"); From fcc21e7c7b1f8b7dd8dc3f581d1f541276d3b896 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 20 Oct 2016 12:48:32 -0700 Subject: [PATCH 188/235] Allow inject_message to update the checkpoint without a message --- CMakeLists.txt | 2 +- docs/heka/input.md | 28 ++++++++++++++++++---------- include/luasandbox/heka/sandbox.h | 7 ++++--- src/heka/sandbox.c | 8 ++++++++ src/heka/test/lua/iim.lua | 4 +++- src/heka/test/test_heka_sandbox.c | 2 +- 6 files changed, 35 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ab0a11..df89aeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.0 LANGUAGES C) +project(luasandbox VERSION 1.2.1 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/heka/input.md b/docs/heka/input.md index 21b28d5..7c9f421 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -68,15 +68,19 @@ to the values provide in the message table, if no value is provided it defaults to the appropriate configuration value. *Arguments* -* msg ([Heka message table](message.html), [Heka stream reader](#heka-stream-reader-methods) or Heka protobuf string) -* checkpoint (optional: number, string) - checkpoint to be returned in the `process_message` call +* msg ([Heka message table](message.html), + [Heka stream reader](#heka-stream-reader-methods), + Heka protobuf string, + or nil (if only updating the checkpoint)) +* checkpoint (optional: number, string) - checkpoint to be returned in the + `process_message` call *Return* * none - throws an error on invalid input ### create_stream_reader Creates a Heka stream reader to enable parsing of a framed Heka protobuf stream -in a Lua sandbox. See: +in a Lua sandbox. See: [Example of a Heka protobuf reader](#example-of-a-heka-protobuf-stdin-reader) *Arguments* @@ -97,19 +101,23 @@ local found, consumed, need = hsr:find_message(buf) ``` *Arguments* -* buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data or a userdate file object -* decode (bool default: true) - true if the framed message should be protobuf decoded +* buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data + or a userdate file object +* decode (bool default: true) - true if the framed message should be protobuf + decoded *Return* * found (bool) - true if a message was found -* consumed (number) - number of bytes consumed so the offset can be tracked for checkpointing purposes -* need/read (number) - number of bytes needed to complete the message or fill the underlying buffer - or in the case of a file object the number of bytes added to the buffer +* consumed (number) - number of bytes consumed so the offset can be tracked for + checkpointing purposes +* need/read (number) - number of bytes needed to complete the message or fill + the underlying buffer or in the case of a file object the number of bytes + added to the buffer ##### decode_message -Converts a Heka protobuf encoded message string into a stream reader representation. -Note: this operation clears the internal stream reader buffer. +Converts a Heka protobuf encoded message string into a stream reader +representation. Note: this operation clears the internal stream reader buffer. *Arguments* * heka_pb (string) - Heka protobuf binary string diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 8df9fd3..dc08b39 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -74,9 +74,10 @@ LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; * of the checkpoint values will be set in a call. Numeric checkpoints can have * a measurable performance benefit but are not always applicable so both * options are provided to support various types of input plugins. This function - * can be called with a NULL pb pointer by the 'is_running' API; it should be - * treated as a no-op and used as a synchronization point to collect statistics - * and communicate shutdown status. + * can be called with a NULL pb pointer by the 'is_running' API or if only a + * checkpoint update is being performed; it should be treated as a no-op and + * used as a synchronization point to collect statistics and communicate + * shutdown status. * * @param parent Opaque pointer the host object owning this sandbox * @param pb Pointer to a Heka protobuf encoded message being injected. diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 17a5b08..f6ad758 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -198,6 +198,14 @@ static int inject_message_input(lua_State *lua) output.len = 0; output.s = lsb_get_output(lsb, &output.len); break; + case LUA_TNIL: + if (lua_isnoneornil(lua, 2)) { + return luaL_error(lua, "%s() message cannot be nil without a checkpoint" + " update", im_func_name); + } + output.len = 0; + output.s = NULL; + break; default: return luaL_error(lua, "%s() unsupported message type: %s", im_func_name, lua_typename(lua, t)); diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index 77c2d94..b279d52 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -11,12 +11,14 @@ local msgs = { {{Timestamp = 1, Uuid = "00000000-0000-0000-0000-000000000000"}, nil}, {{Timestamp = 2, Uuid = "00000000-0000-0000-0000-000000000000", Logger = "logger", Hostname = "hostname", Type = "type", Payload = "payload", EnvVersion = "envversion", Pid = 99, Severity = 5}, 99}, {{Timestamp = 3, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}}, "foo.log:123"}, + {nil, "foo.log:123"}, + {nil, 123}, } local err_msgs = { {msg = {Timestamp = 1e9, Fields = {counts={2,"ten",4}}}, err = "inject_message() failed: array has mixed types"}, {msg = {Timestamp = 1e9, Fields = {counts={}}}, err = "inject_message() failed: unsupported type: nil"}, - {err = "inject_message() unsupported message type: nil"}, + {err = "inject_message() message cannot be nil without a checkpoint update"}, {msg = {Timestamp = 1e9, Fields = {counts={{1},{2}}}}, err = "inject_message() failed: unsupported array type: table"}, {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=2}}}, err = "inject_message() failed: invalid string value_type: 2"}, {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=3}}}, err = "inject_message() failed: invalid string value_type: 3"}, diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 69a2005..a55a791 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -591,7 +591,7 @@ static char* test_im_input() "Logger = 'iim'\n", &logger, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(8 == stats.im_cnt, "received %llu", stats.im_cnt); mu_assert(332 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); From 3c003dd6c1991bab23f8d07f57229b601c1f8ec6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 20 Oct 2016 13:04:00 -0700 Subject: [PATCH 189/235] Add coroutine state support to output() and inject_message() --- include/luasandbox_output.h | 19 +++- src/heka/message.c | 212 ++++++++++++++++++------------------ src/heka/message_impl.h | 4 +- src/heka/sandbox.c | 8 +- src/luasandbox.c | 2 +- src/luasandbox_output.c | 37 ++++--- 6 files changed, 157 insertions(+), 125 deletions(-) diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h index 6af7a99..27851b7 100644 --- a/include/luasandbox_output.h +++ b/include/luasandbox_output.h @@ -14,7 +14,8 @@ #include "luasandbox.h" #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif #include "luasandbox/lua.h" @@ -81,6 +82,22 @@ LSB_EXPORT lua_CFunction lsb_get_zero_copy_function(lua_State *lua, int index); LSB_EXPORT void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append); +/** + * Write an array of variables on the Lua stack to the output buffer. After + * adding support for coroutines we need an extra variable to specify the + * correct Lua state. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer the Lua state + * @param start Lua stack index of first variable. + * @param end Lua stack index of the last variable. + * @param append 0 to overwrite the output buffer, 1 to append the output to it + * + */ +LSB_EXPORT void +lsb_output_coroutine(lsb_lua_sandbox *lsb, lua_State *lua, int start, + int end, int append); + /** * Retrieve the data in the output buffer and reset the buffer. The returned * output string will remain valid until additional sandbox output is performed. diff --git a/src/heka/message.c b/src/heka/message.c index eb4966f..ef8b96b 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -243,7 +243,7 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) * Retrieve the string value for a Lua table entry (the table should be on top * of the stack). If the entry is not found or not a string nothing is encoded. * - * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. * @param ob Pointer to the output data buffer. * @param tag Field identifier. * @param name Key used for the Lua table entry lookup. @@ -252,17 +252,17 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) * @return lsb_err_value NULL on success error message on failure */ static lsb_err_value -encode_string(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, - const char *name, int index) +encode_string(lua_State *lua, lsb_output_buffer *ob, char tag, const char *name, + int index) { lsb_err_value ret = NULL; - lua_getfield(lsb->lua, index, name); - if (lua_isstring(lsb->lua, -1)) { + lua_getfield(lua, index, name); + if (lua_isstring(lua, -1)) { size_t len; - const char *s = lua_tolstring(lsb->lua, -1, &len); + const char *s = lua_tolstring(lua, -1, &len); ret = lsb_pb_write_string(ob, tag, s, len); } - lua_pop(lsb->lua, 1); + lua_pop(lua, 1); return ret; } @@ -272,7 +272,7 @@ encode_string(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, * of the stack). If the entry is not found or not a number nothing is encoded, * otherwise the number is varint encoded. * - * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. * @param ob Pointer to the output data buffer. * @param tag Field identifier. * @param name Key used for the Lua table entry lookup. @@ -281,17 +281,17 @@ encode_string(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, * @return lsb_err_value NULL on success error message on failure */ static lsb_err_value -encode_int(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, - const char *name, int index) +encode_int(lua_State *lua, lsb_output_buffer *ob, char tag, const char *name, + int index) { lsb_err_value ret = NULL; - lua_getfield(lsb->lua, index, name); - if (lua_isnumber(lsb->lua, -1)) { - unsigned long long i = (unsigned long long)lua_tonumber(lsb->lua, -1); + lua_getfield(lua, index, name); + if (lua_isnumber(lua, -1)) { + unsigned long long i = (unsigned long long)lua_tonumber(lua, -1); ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_VARINT); if (!ret) ret = lsb_pb_write_varint(ob, i); } - lua_pop(lsb->lua, 1); + lua_pop(lua, 1); return ret; } @@ -300,6 +300,7 @@ encode_int(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, * Encodes the field value. * * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. * @param ob Pointer to the output data buffer. * @param first Boolean indicator used to add addition protobuf data in the * correct order. @@ -309,14 +310,15 @@ encode_int(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, * @return lsb_err_value NULL on success error message on failure */ static lsb_err_value -encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, - const char *representation, int value_type); +encode_field_value(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + int first, const char *representation, int value_type); /** * Encodes a field that has an array of values. * * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. * @param ob Pointer to the output data buffer. * @param t Lua type of the array values. * @param representation String representation of the field i.e., "ms" @@ -324,27 +326,27 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, * @return lsb_err_value NULL on success error message on failure */ static lsb_err_value -encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, - const char *representation, int value_type) +encode_field_array(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + int t, const char *representation, int value_type) { lsb_err_value ret = NULL; - int first = (int)lua_objlen(lsb->lua, -1); + int first = (int)lua_objlen(lua, -1); int multiple = first > 1 ? first : 0; size_t len_pos = 0; - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (!ret && lua_next(lsb->lua, -2) != 0) { - if (lua_type(lsb->lua, -1) != t) { + lua_checkstack(lua, 2); + lua_pushnil(lua); + while (!ret && lua_next(lua, -2) != 0) { + if (lua_type(lua, -1) != t) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); return LSB_ERR_HEKA_INPUT; } - ret = encode_field_value(lsb, ob, first, representation, value_type); + ret = encode_field_value(lsb, lua, ob, first, representation, value_type); if (first) { len_pos = ob->pos; first = 0; } - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. + lua_pop(lua, 1); // Remove the value leaving the key on top for + // the next interation. } if (!ret && multiple && value_type == LSB_PB_INTEGER) { // fix up the varint packed length @@ -370,41 +372,43 @@ encode_field_array(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int t, * Encodes a field that contains metadata in addition to its value. * * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. * @param ob Pointer to the output data buffer. * * @return lsb_err_value NULL on success error message on failure */ static lsb_err_value -encode_field_object(lsb_lua_sandbox *lsb, lsb_output_buffer *ob) +encode_field_object(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob) { const char *representation = NULL; - lua_getfield(lsb->lua, -1, "representation"); - if (lua_isstring(lsb->lua, -1)) { - representation = lua_tostring(lsb->lua, -1); + lua_getfield(lua, -1, "representation"); + if (lua_isstring(lua, -1)) { + representation = lua_tostring(lua, -1); } int value_type = -1; - lua_getfield(lsb->lua, -2, "value_type"); - if (lua_isnumber(lsb->lua, -1)) { - value_type = (int)lua_tointeger(lsb->lua, -1); + lua_getfield(lua, -2, "value_type"); + if (lua_isnumber(lua, -1)) { + value_type = (int)lua_tointeger(lua, -1); } - lua_getfield(lsb->lua, -3, "value"); - lsb_err_value ret = encode_field_value(lsb, ob, 1, representation, value_type); - lua_pop(lsb->lua, 3); // remove representation, value_type and value + lua_getfield(lua, -3, "value"); + lsb_err_value ret = encode_field_value(lsb, lua, ob, 1, representation, + value_type); + lua_pop(lua, 3); // remove representation, value_type and value return ret; } static lsb_err_value -encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, - const char *representation, int value_type) +encode_field_value(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + int first, const char *representation, int value_type) { lsb_err_value ret = NULL; size_t len; const char *s; - int t = lua_type(lsb->lua, -1); + int t = lua_type(lua, -1); switch (t) { case LUA_TSTRING: switch (value_type) { @@ -431,7 +435,7 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, if (ret) return ret; } } - s = lua_tolstring(lsb->lua, -1, &len); + s = lua_tolstring(lua, -1, &len); if (value_type == LSB_PB_BYTES) { ret = lsb_pb_write_string(ob, LSB_PB_VALUE_BYTES, s, len); if (ret) return ret; @@ -481,9 +485,9 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, } } if (value_type == LSB_PB_INTEGER) { - ret = lsb_pb_write_varint(ob, lua_tointeger(lsb->lua, -1)); + ret = lsb_pb_write_varint(ob, lua_tointeger(lua, -1)); } else { - ret = lsb_pb_write_double(ob, lua_tonumber(lsb->lua, -1)); + ret = lsb_pb_write_double(ob, lua_tonumber(lua, -1)); } if (ret) return ret; break; @@ -512,34 +516,34 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, } if (ret) return ret; } - ret = lsb_pb_write_bool(ob, lua_toboolean(lsb->lua, -1)); + ret = lsb_pb_write_bool(ob, lua_toboolean(lua, -1)); break; case LUA_TTABLE: { - lua_rawgeti(lsb->lua, -1, 1); - int t = lua_type(lsb->lua, -1); - lua_pop(lsb->lua, 1); // remove the array test value + lua_rawgeti(lua, -1, 1); + int t = lua_type(lua, -1); + lua_pop(lua, 1); // remove the array test value switch (t) { case LUA_TNIL: - ret = encode_field_object(lsb, ob); + ret = encode_field_object(lsb, lua, ob); break; case LUA_TNUMBER: case LUA_TSTRING: case LUA_TBOOLEAN: - ret = encode_field_array(lsb, ob, t, representation, value_type); + ret = encode_field_array(lsb, lua, ob, t, representation, value_type); break; default: snprintf(lsb->error_message, LSB_ERROR_SIZE, - "unsupported array type: %s", lua_typename(lsb->lua, t)); + "unsupported array type: %s", lua_typename(lua, t)); return LSB_ERR_LUA; } } break; case LUA_TLIGHTUSERDATA: - lua_getfield(lsb->lua, -4, "userdata"); - if (lua_type(lsb->lua, -1) != LUA_TUSERDATA) { + lua_getfield(lua, -4, "userdata"); + if (lua_type(lua, -1) != LUA_TUSERDATA) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "a lightuserdata output must also specify a userdata value"); return LSB_ERR_LUA; @@ -548,7 +552,7 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, case LUA_TUSERDATA: { - lua_CFunction fp = lsb_get_output_function(lsb->lua, -1); + lua_CFunction fp = lsb_get_output_function(lua, -1); size_t len_pos = 0; if (!fp) { snprintf(lsb->error_message, LSB_ERROR_SIZE, @@ -577,9 +581,9 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, ret = lsb_pb_write_varint(ob, 0); // length tbd later if (ret) return ret; - lua_pushlightuserdata(lsb->lua, ob); - int result = fp(lsb->lua); - lua_pop(lsb->lua, 1); // remove output function + lua_pushlightuserdata(lua, ob); + int result = fp(lua); + lua_pop(lua, 1); // remove output function if (result) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "userdata output callback failed: %d", result); @@ -587,12 +591,12 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, } ret = lsb_pb_update_field_length(ob, len_pos); } - if (t == LUA_TLIGHTUSERDATA) lua_pop(lsb->lua, 1); // remove the userdata + if (t == LUA_TLIGHTUSERDATA) lua_pop(lua, 1); // remove the userdata break; default: snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", - lua_typename(lsb->lua, t)); + lua_typename(lua, t)); return LSB_ERR_LUA; } return ret; @@ -604,6 +608,7 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, * message fields. * * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State * @param ob Pointer to the output data buffer. * @param tag Field identifier. * @param name Key used for the Lua table entry lookup. @@ -612,18 +617,18 @@ encode_field_value(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, int first, * @return lsb_err_value NULL on success error message on failure */ static lsb_err_value -encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, - const char *name, int index) +encode_fields(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + char tag, const char *name, int index) { lsb_err_value ret = NULL; - lua_getfield(lsb->lua, index, name); - if (!lua_istable(lsb->lua, -1)) { + lua_getfield(lua, index, name); + if (!lua_istable(lua, -1)) { return ret; } - lua_rawgeti(lsb->lua, -1, 1); // test for the array notation + lua_rawgeti(lua, -1, 1); // test for the array notation size_t len_pos, len; - if (lua_istable(lsb->lua, -1)) { + if (lua_istable(lua, -1)) { int i = 1; do { ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); @@ -633,30 +638,30 @@ encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, ret = lsb_pb_write_varint(ob, 0); // length tbd later if (ret) return ret; - lua_getfield(lsb->lua, -1, "name"); - if (lua_isstring(lsb->lua, -1)) { - const char *s = lua_tolstring(lsb->lua, -1, &len); + lua_getfield(lua, -1, "name"); + if (lua_isstring(lua, -1)) { + const char *s = lua_tolstring(lua, -1, &len); ret = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, "field name must be a string"); ret = LSB_ERR_HEKA_INPUT; } - lua_pop(lsb->lua, 1); // remove the name + lua_pop(lua, 1); // remove the name if (ret) return ret; - ret = encode_field_object(lsb, ob); + ret = encode_field_object(lsb, lua, ob); if (!ret) ret = lsb_pb_update_field_length(ob, len_pos); if (ret) return ret; - lua_pop(lsb->lua, 1); // remove the current field object - lua_rawgeti(lsb->lua, -1, ++i); // grab the next field object - } while (!ret && !lua_isnil(lsb->lua, -1)); + lua_pop(lua, 1); // remove the current field object + lua_rawgeti(lua, -1, ++i); // grab the next field object + } while (!ret && !lua_isnil(lua, -1)); } else { - lua_pop(lsb->lua, 1); // remove the array test value - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (lua_next(lsb->lua, -2) != 0) { + lua_pop(lua, 1); // remove the array test value + lua_checkstack(lua, 2); + lua_pushnil(lua); + while (lua_next(lua, -2) != 0) { ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); if (ret) return ret; @@ -664,8 +669,8 @@ encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, ret = lsb_pb_write_varint(ob, 0); // length tbd later if (ret) return ret; - if (lua_isstring(lsb->lua, -2)) { - const char *s = lua_tolstring(lsb->lua, -2, &len); + if (lua_isstring(lua, -2)) { + const char *s = lua_tolstring(lua, -2, &len); ret = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, @@ -674,15 +679,15 @@ encode_fields(lsb_lua_sandbox *lsb, lsb_output_buffer *ob, char tag, } if (ret) return ret; - ret = encode_field_value(lsb, ob, 1, NULL, -1); + ret = encode_field_value(lsb, lua, ob, 1, NULL, -1); if (!ret) ret = lsb_pb_update_field_length(ob, len_pos); if (ret) return ret; - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. + lua_pop(lua, 1); // Remove the value leaving the key on top for + // the next interation. } } - lua_pop(lsb->lua, 1); // remove the fields table + lua_pop(lua, 1); // remove the fields table return ret; } @@ -854,7 +859,7 @@ int heka_encode_message(lua_State *lua) if (!lsb) return luaL_error(lua, "encode_message() invalid " LSB_THIS_PTR); lsb->output.pos = 0; - lsb_err_value ret = heka_encode_message_table(lsb, 1); + lsb_err_value ret = heka_encode_message_table(lsb, lua, 1); if (ret) { const char *err = lsb_get_error(lsb); if (strlen(err) == 0) err = ret; @@ -887,51 +892,52 @@ int heka_encode_message(lua_State *lua) } -lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, int idx) +lsb_err_value +heka_encode_message_table(lsb_lua_sandbox *lsb, lua_State *lua, int idx) { lsb_err_value ret = NULL; lsb_output_buffer *ob = &lsb->output; ob->pos = 0; // use existing or create a type 4 uuid - lua_getfield(lsb->lua, idx, LSB_UUID); + lua_getfield(lua, idx, LSB_UUID); size_t len; - const char *uuid = lua_tolstring(lsb->lua, -1, &len); + const char *uuid = lua_tolstring(lua, -1, &len); ret = lsb_write_heka_uuid(ob, uuid, len); - lua_pop(lsb->lua, 1); // remove uuid + lua_pop(lua, 1); // remove uuid if (ret) return ret; // use existing or create a timestamp - lua_getfield(lsb->lua, idx, LSB_TIMESTAMP); + lua_getfield(lua, idx, LSB_TIMESTAMP); long long ts; - if (lua_isnumber(lsb->lua, -1)) { - ts = (long long)lua_tonumber(lsb->lua, -1); + if (lua_isnumber(lua, -1)) { + ts = (long long)lua_tonumber(lua, -1); } else { ts = lsb_get_timestamp(); } - lua_pop(lsb->lua, 1); // remove timestamp + lua_pop(lua, 1); // remove timestamp lsb_heka_sandbox *hsb = lsb_get_parent(lsb); if (hsb->restricted_headers) { - lua_pushstring(lsb->lua, hsb->name); - lua_setfield(lsb->lua, idx, LSB_LOGGER); - lua_pushstring(lsb->lua, hsb->hostname); - lua_setfield(lsb->lua, idx, LSB_HOSTNAME); + lua_pushstring(lua, hsb->name); + lua_setfield(lua, idx, LSB_LOGGER); + lua_pushstring(lua, hsb->hostname); + lua_setfield(lua, idx, LSB_HOSTNAME); } else { - set_missing_headers(lsb->lua, idx, hsb); + set_missing_headers(lua, idx, hsb); } ret = lsb_pb_write_key(ob, LSB_PB_TIMESTAMP, LSB_PB_WT_VARINT); if (!ret) ret = lsb_pb_write_varint(ob, ts); - if (!ret) ret = encode_string(lsb, ob, LSB_PB_TYPE, LSB_TYPE, idx); - if (!ret) ret = encode_string(lsb, ob, LSB_PB_LOGGER, LSB_LOGGER, idx); - if (!ret) ret = encode_int(lsb, ob, LSB_PB_SEVERITY, LSB_SEVERITY, idx); - if (!ret) ret = encode_string(lsb, ob, LSB_PB_PAYLOAD, LSB_PAYLOAD, idx); - if (!ret) ret = encode_string(lsb, ob, LSB_PB_ENV_VERSION, LSB_ENV_VERSION, + if (!ret) ret = encode_string(lua, ob, LSB_PB_TYPE, LSB_TYPE, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_LOGGER, LSB_LOGGER, idx); + if (!ret) ret = encode_int(lua, ob, LSB_PB_SEVERITY, LSB_SEVERITY, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_PAYLOAD, LSB_PAYLOAD, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_ENV_VERSION, LSB_ENV_VERSION, idx); - if (!ret) ret = encode_int(lsb, ob, LSB_PB_PID, LSB_PID, idx); - if (!ret) ret = encode_string(lsb, ob, LSB_PB_HOSTNAME, LSB_HOSTNAME, idx); - if (!ret) ret = encode_fields(lsb, ob, LSB_PB_FIELDS, LSB_FIELDS, idx); + if (!ret) ret = encode_int(lua, ob, LSB_PB_PID, LSB_PID, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_HOSTNAME, LSB_HOSTNAME, idx); + if (!ret) ret = encode_fields(lsb, lua, ob, LSB_PB_FIELDS, LSB_FIELDS, idx); if (!ret) ret = lsb_expand_output_buffer(ob, 1); ob->buf[ob->pos] = 0; // prevent possible overrun if treated as a string return ret; diff --git a/src/heka/message_impl.h b/src/heka/message_impl.h index ee4b286..7c27434 100644 --- a/src/heka/message_impl.h +++ b/src/heka/message_impl.h @@ -43,11 +43,13 @@ int heka_encode_message(lua_State *lua); * sandbox with heka_encode_message. * * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. * @param idx Lua stack index of the message table. * * @return lsb_err_value NULL on success error message on failure */ -lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, int idx); +lsb_err_value +heka_encode_message_table(lsb_lua_sandbox *lsb, lua_State *lua, int idx); /** diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index f6ad758..94e1122 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -190,7 +190,7 @@ static int inject_message_input(lua_State *lua) } break; case LUA_TTABLE: - if (heka_encode_message_table(lsb, 1)) { + if (heka_encode_message_table(lsb, lua, 1)) { const char *err = lsb_get_error(lsb); if (strlen(err) == 0) err = "exceeded output_limit"; return luaL_error(lua, "%s() failed: %s", im_func_name, err); @@ -201,7 +201,7 @@ static int inject_message_input(lua_State *lua) case LUA_TNIL: if (lua_isnoneornil(lua, 2)) { return luaL_error(lua, "%s() message cannot be nil without a checkpoint" - " update", im_func_name); + " update", im_func_name); } output.len = 0; output.s = NULL; @@ -230,7 +230,7 @@ static int inject_message_analysis(lua_State *lua) lua_pop(lua, 1); // remove this ptr if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); - if (heka_encode_message_table(lsb, 1)) { + if (heka_encode_message_table(lsb, lua, 1)) { return luaL_error(lua, "%s() failed: %s", im_func_name, lsb_get_error(lsb)); } @@ -273,7 +273,7 @@ static int inject_payload(lua_State *lua) } if (n > 2) { - lsb_output(lsb, 3, n, 1); + lsb_output_coroutine(lsb, lua, 3, n, 1); lua_pop(lua, n - 2); } size_t len = 0; diff --git a/src/luasandbox.c b/src/luasandbox.c index 1c05c74..e9615df 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -130,7 +130,7 @@ static int output(lua_State *lua) if (n == 0) { return luaL_argerror(lsb->lua, 0, "must have at least one argument"); } - lsb_output(lsb, 1, n, 1); + lsb_output_coroutine(lsb, lua, 1, n, 1); return 0; } diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c index a477694..a1bad4c 100644 --- a/src/luasandbox_output.c +++ b/src/luasandbox_output.c @@ -62,6 +62,13 @@ lua_CFunction lsb_get_zero_copy_function(lua_State *lua, int index) void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) +{ + lsb_output_coroutine(lsb, lsb->lua, start, end, append); +} + + +void lsb_output_coroutine(lsb_lua_sandbox *lsb, lua_State *lua, int start, + int end, int append) { if (!append) { lsb->output.pos = 0; @@ -69,16 +76,16 @@ void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) int result = 0; for (int i = start; result == 0 && i <= end; ++i) { - switch (lua_type(lsb->lua, i)) { + switch (lua_type(lua, i)) { case LUA_TNUMBER: - if (lsb_outputd(&lsb->output, lua_tonumber(lsb->lua, i))) { + if (lsb_outputd(&lsb->output, lua_tonumber(lua, i))) { result = 1; } break; case LUA_TSTRING: { size_t len; - const char *s = lua_tolstring(lsb->lua, i, &len); + const char *s = lua_tolstring(lua, i, &len); if (lsb_outputs(&lsb->output, s, len)) { result = 1; } @@ -90,26 +97,26 @@ void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) } break; case LUA_TBOOLEAN: - if (lsb_outputf(&lsb->output, "%s", lua_toboolean(lsb->lua, i) - ? "true" : "false")) { + if (lsb_outputf(&lsb->output, "%s", lua_toboolean(lua, i) + ? "true" : "false")) { result = 1; } break; case LUA_TUSERDATA: { - lua_CFunction fp = lsb_get_output_function(lsb->lua, i); + lua_CFunction fp = lsb_get_output_function(lua, i); if (!fp) { - luaL_argerror(lsb->lua, i, "unknown userdata type"); + luaL_argerror(lua, i, "unknown userdata type"); return; // never reaches here but the compiler doesn't know it } - lua_pushvalue(lsb->lua, i); - lua_pushlightuserdata(lsb->lua, &lsb->output); - result = fp(lsb->lua); - lua_pop(lsb->lua, 2); // remove the copy of the value and the output + lua_pushvalue(lua, i); + lua_pushlightuserdata(lua, &lsb->output); + result = fp(lua); + lua_pop(lua, 2); // remove the copy of the value and the output } break; default: - luaL_argerror(lsb->lua, i, "unsupported type"); + luaL_argerror(lua, i, "unsupported type"); break; } } @@ -117,14 +124,14 @@ void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; } if (result != 0) { if (lsb->error_message[0] == 0) { - luaL_error(lsb->lua, "output_limit exceeded"); + luaL_error(lua, "output_limit exceeded"); } - luaL_error(lsb->lua, lsb->error_message); + luaL_error(lua, lsb->error_message); } } From 5337b3f5616205d272d305d976d31b2edcc62eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Hertzog?= Date: Tue, 27 Sep 2016 12:00:43 +0200 Subject: [PATCH 190/235] Inject CFLAGS/CXXFLAGS/CPPFLAGS from environment Bug: https://github.com/mozilla-services/lua_sandbox/issues/157 --- cmake/mozsvc.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index cf9c0ea..a7d3dad 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -24,8 +24,8 @@ if(MSVC) else() # Predefined Macros: clang|gcc -dM -E -x c /dev/null # Compiler options: http://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html#Invoking-GCC - set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra -fPIC -fvisibility=hidden") - set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") + set(CMAKE_C_FLAGS "-std=gnu99 -pedantic $ENV{CFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -fvisibility=hidden") + set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic $ENV{CXXFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") set(CMAKE_C_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) From 6d1259196fb27c710c44b34f02c4c8ca03947f02 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 12 Dec 2016 13:28:00 -0800 Subject: [PATCH 191/235] Fix the unit tests on Windows and 32 bit systems --- src/heka/read_message_zc.c | 5 +++-- src/util/test/test_heka_message.c | 34 ++++++++++++++++++++++++++++--- src/util/test/test_string.c | 11 ++++++---- 3 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/heka/read_message_zc.c b/src/heka/read_message_zc.c index c76d6d0..18f5017 100644 --- a/src/heka/read_message_zc.c +++ b/src/heka/read_message_zc.c @@ -9,13 +9,14 @@ #include #include +#include "../luasandbox_defines.h" #include "lauxlib.h" #include "lua.h" #include "lualib.h" +#include "luasandbox/heka/sandbox.h" +#include "luasandbox/util/heka_message.h" #include "luasandbox_output.h" #include "luasandbox_serialize.h" -#include "luasandbox/util/heka_message.h" -#include "luasandbox/heka/sandbox.h" static const char *metatable_name = "lsb.read_message_zc"; diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c index 2a5aa3e..a509f28 100644 --- a/src/util/test/test_heka_message.c +++ b/src/util/test/test_heka_message.c @@ -333,23 +333,51 @@ static char* test_write_heka_uuid() return NULL; } +#if SIZE_MAX < 65536 +static char* test_write_heka_header() +{ + char header[LSB_MIN_HDR_SIZE]; + size_t hlen; + for (unsigned i = 0; i < sizeof(size_t); ++i) { + hlen = lsb_write_heka_header(header, (size_t)1 << (i * 8)); + mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); + } + hlen = lsb_write_heka_header(header, (size_t)1 << 15); + mu_assert(hlen == 7, "received %" PRIuSIZE, hlen); + return NULL; +} +#elif SIZE_MAX < 4294967296 static char* test_write_heka_header() { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen; + for (unsigned i = 0; i < sizeof(size_t); ++i) { + hlen = lsb_write_heka_header(header, (size_t)1 << (i * 8)); + mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); + } + hlen = lsb_write_heka_header(header, (size_t)1 << 31); + mu_assert(hlen == 9, "received %" PRIuSIZE, hlen); + return NULL; +} +#else +static char* test_write_heka_header() // limit to 8 byte test +{ char header[LSB_MIN_HDR_SIZE]; size_t hlen; for (unsigned i = 0; i < 7; ++i) { - hlen = lsb_write_heka_header(header, 1LL << (i * 8)); + hlen = lsb_write_heka_header(header, (size_t)1 << (i * 8)); mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); } - hlen = lsb_write_heka_header(header, 1LL << 56); + hlen = lsb_write_heka_header(header, (size_t)1 << 56); mu_assert(hlen == 13, "received %" PRIuSIZE, hlen); - hlen = lsb_write_heka_header(header, 1LL << 63); + hlen = lsb_write_heka_header(header, (size_t)1 << 63); mu_assert(hlen == 14, "received %" PRIuSIZE, hlen); return NULL; } +#endif static char* all_tests() diff --git a/src/util/test/test_string.c b/src/util/test/test_string.c index 3459cfd..179534f 100644 --- a/src/util/test/test_string.c +++ b/src/util/test/test_string.c @@ -6,6 +6,7 @@ /** @brief lsb_input_buffer unit tests @file */ +#include #include #include "luasandbox/test/mu_test.h" @@ -47,14 +48,15 @@ static char* test_success() for (unsigned i = 0; i < sizeof tests / sizeof(testcase); ++i){ testcase *tc = &tests[i]; - char d[strlen(tc->s) + 1]; - size_t len = sizeof(d); + size_t len = strlen(tc->s) + 1; + char *d = malloc(len); char *us = lsb_lua_string_unescape(d, tc->s, &len); mu_assert(us, "test: %d valid string: %s", i, tc->s); mu_assert(len == tc->len, "test: %d string: %s expected len: %" PRIuSIZE " received: %" PRIuSIZE, i, tc->s, tc->len, len); mu_assert(memcmp(us, tc->r, len) == 0, "test: %d memcmp string: %s", i, tc->s); + free(d); } return NULL; } @@ -71,10 +73,11 @@ static char* test_failure() for (unsigned i = 0; i < sizeof tests / sizeof(testcase); ++i){ testcase *tc = &tests[i]; - char d[tc->len + 1]; - size_t len = sizeof(d); + size_t len = tc->len + 1; + char *d = malloc(len); mu_assert(!lsb_lua_string_unescape(d, tc->s, &len), "test: %d invalid string: %s", i, tc->s); + free(d); } mu_assert(!lsb_lua_string_unescape(NULL, tests[0].s, &tests[0].len), "NULL destination"); From 43b789e927ec949265f4d1797c2bdfb403804ecb Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 20 Dec 2016 10:15:41 -0800 Subject: [PATCH 192/235] Remove CPACK_DEBIAN_PACKAGE_SHLIBDEPS (defaulting it to off) --- cmake/mozsvc.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index a7d3dad..379ce7c 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -49,7 +49,6 @@ if(CMAKE_BUILD_TYPE MATCHES "^[Dd][Ee][Bb][Uu][Gg]$") else() set(CPACK_STRIP_FILES TRUE) endif() -set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) include(CPack) include(CTest) From 40a17e32f6cee5fa800ef41d9024d05ca66b4032 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Fri, 23 Dec 2016 15:42:58 +0100 Subject: [PATCH 193/235] Build and test using travis --- .travis.yml | 20 +++++ build/functions.sh | 197 +++++++++++++++++++++++++++++++++++++++++++++ build/run.sh | 19 +++++ 3 files changed, 236 insertions(+) create mode 100644 .travis.yml create mode 100755 build/functions.sh create mode 100755 build/run.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..14eaedc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,20 @@ +language: generic +sudo: required +services: + - docker + +env: + matrix: + - DOCKER_IMAGE=centos:6 CMAKE_URL=auto + - DOCKER_IMAGE=centos:7 CMAKE_URL=auto + - DOCKER_IMAGE=debian:7 CMAKE_URL=auto + - DOCKER_IMAGE=debian:8 + - DOCKER_IMAGE=debian:8 CC=clang + - DOCKER_IMAGE=debian:stretch + - DOCKER_IMAGE=fedora:latest + - DOCKER_IMAGE=ubuntu:12.04 CMAKE_URL=auto + - DOCKER_IMAGE=ubuntu:latest # LTS + - DOCKER_IMAGE=ubuntu:devel + +script: + - ./build/run.sh build diff --git a/build/functions.sh b/build/functions.sh new file mode 100755 index 0000000..9289ac7 --- /dev/null +++ b/build/functions.sh @@ -0,0 +1,197 @@ +#!/bin/sh + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Author: Mathieu Parent + +# ================================================================= +# Show usage +usage() { + echo "Build, test and make packages (optionally using Docker)" >&2 + echo " $0 build" >&2 + echo "" >&2 + echo "Options are passed using environment variables:" >&2 + echo " - DOCKER_IMAGE: Run the build in a container (default: )" >&2 + echo " - CPACK_GENERATOR: DEB, RPM, TGZ, ... (default guessed)" >&2 + echo " - CMAKE_URL: If set, download cmake from it (default: , auto also supported)" >&2 + echo " - CMAKE_SHA256: Used to checksum CMAKE_URL (default: )" >&2 + echo " - CC: C compiler (default: gcc)" >&2 + echo " - CXX: C++ compiler (default: g++)" >&2 +} + +# ================================================================= +# Run command within container +docker_run() { + DOCKER_IMAGE="${DOCKER_IMAGE:-debian:jessie}" + DOCKER_VOLUME="$(dirname "$PWD")" + ( set -x + docker pull "$DOCKER_IMAGE" + ret=0 + docker run \ + --volume "${DOCKER_VOLUME}:${DOCKER_VOLUME}" \ + --workdir "$PWD" \ + --rm=true --tty=true \ + --env "CPACK_GENERATOR=${CPACK_GENERATOR}" \ + --env "CMAKE_URL=${CMAKE_URL}" \ + --env "CMAKE_SHA256=${CMAKE_SHA256}" \ + --env "CC=${CC}" \ + --env "CXX=${CXX}" \ + "$DOCKER_IMAGE" \ + $@ + ) +} + +# ================================================================= +# Guess environment variables +setup_env() { + if [ "$(id -u)" = 0 ]; then + as_root="" + else + as_root="sudo" + fi + if [ -z "${CPACK_GENERATOR}" ]; then + if [ -x "`command -v apt-get 2>/dev/null`" ]; then + CPACK_GENERATOR=DEB + elif [ -x "`command -v rpm 2>/dev/null`" ]; then + CPACK_GENERATOR=RPM + else + CPACK_GENERATOR=TGZ + fi + fi + CC="${CC:-gcc}" + CXX="${CXX:-g++}" +} + +# ================================================================= +# Install cmake from URL +install_cmake() { + if echo "$PATH" | grep -qF "/cmake-dist/bin:" ; then + # Already done + return + fi + if [ "$CMAKE_URL" = "auto" ]; then + CMAKE_URL="https://cmake.org/files/v3.7/cmake-3.7.1-Linux-x86_64.tar.gz" + CMAKE_SHA256=7b4b7a1d9f314f45722899c0521c261e4bfab4a6b532609e37fef391da6bade2 + elif [ -z "$CMAKE_URL" ]; then + echo "Missing env CMAKE_URL" + exit 1 + elif [ -z "$CMAKE_SHA256" ]; then + echo "Missing env CMAKE_SHA256" + exit 1 + fi + ( set -x + wget -O cmake.tar.gz "${CMAKE_URL}" + echo "${CMAKE_SHA256} cmake.tar.gz" > cmake-SHA256.txt + sha256sum --check cmake-SHA256.txt || { + echo 'Checksum error' + exit 1 + } + mkdir cmake-dist + cd cmake-dist + tar xzf "../cmake.tar.gz" --strip-components=1 + ) + PATH="$PWD/cmake-dist/bin:${PATH}" + echo "+PATH=${PATH}" +} + +# ================================================================= +# Install packages from distribution +# The following substitutions are done: +# - cmake is replaced by downloaded, if CMAKE_URL is set +# - rpm-build is ignored on DEB distributions +# - ca-certificates is ignored on RPM distributions +install_packages() { + local packages + packages="$@" + do_install_cmake=no + if [ -n "$CMAKE_URL" ] && echo " $packages " | grep -qF " cmake "; then + packages="$(echo " $packages " | sed "s/ cmake / wget ca-certificates /")" + do_install_cmake=yes + fi + packages="$(echo " $packages " | sed -e "s/ c-compiler / $CC /" -e "s/ c++-compiler / $CXX /")" + if [ "$CPACK_GENERATOR" = "DEB" ]; then + packages="$(echo "$packages" | sed -e "s/ rpm-build / /" -e "s/ clang++ / clang /")" + ( set -x; + $as_root apt-get update + $as_root apt-get install -y $packages + ) + elif [ "$CPACK_GENERATOR" = "RPM" ]; then + packages="$(echo "$packages" | sed -e "s/ ca-certificates / /" -e "s/ g++ / gcc-c++ /" -e "s/ clang++ / clang /")" + ( set -x; + $as_root yum install -y $packages + ) + else + echo "Unhandled CPACK_GENERATOR: $CPACK_GENERATOR" >&2 + exit 1 + fi + if [ "$do_install_cmake" = "yes" ]; then + install_cmake + fi +} + +# ================================================================= +# Install all packages from directory +install_packages_from_dir() { + local dir + dir="$1" + if [ ! -d "$dir" ]; then + echo "Not a directory: $dir" + exit 1 + fi + if [ "$CPACK_GENERATOR" = "DEB" ]; then + (set -x; $as_root dpkg -i "$dir"/*.deb) + elif [ "$CPACK_GENERATOR" = "RPM" ]; then + (set -x; $as_root rpm -i "$dir"/*.rpm) + else + echo "Unhandled CPACK_GENERATOR: $CPACK_GENERATOR" >&2 + exit 1 + fi +} + +# ================================================================= +# Build +build() { + ( set -x + + rm -rf ./release + # From README.md: + mkdir release + cd release + + cmake -DCMAKE_BUILD_TYPE=release .. + make + + ctest -V + cpack -G "${CPACK_GENERATOR}" + ) +} + +# ================================================================= +# Main +main() { + if [ -n "$DOCKER_IMAGE" ]; then + docker_run "$0" build + else + setup_env + install_packages c-compiler cmake make rpm-build + if [ "$(basename "$PWD")" != "lua_sandbox" ]; then + local old_dir + local lsb_dir + old_dir="$PWD" + lsb_dir="$(dirname "$old_dir")/lua_sandbox" + echo "+cd $lsb_dir" + cd "$lsb_dir" + build + install_packages_from_dir ./release + echo "+cd $old_dir" + cd "$old_dir" + fi + if [ -n "$build_function" ]; then + "$build_function" + else + build + fi + fi +} diff --git a/build/run.sh b/build/run.sh new file mode 100755 index 0000000..71b6d04 --- /dev/null +++ b/build/run.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Author: Mathieu Parent + +set -e + +. "$(dirname $0)/functions.sh" + +if [ "$1" != "build" -o $# -ge 2 ]; then + usage + exit 1 +fi + +build_function= +main From c1920ea30e91812a07296cce8f4b3ce0c3ce2401 Mon Sep 17 00:00:00 2001 From: Mathieu Parent Date: Sat, 31 Dec 2016 14:56:57 +0100 Subject: [PATCH 194/235] Fix CVE-2014-5461 in embedded lua See http://www.lua.org/bugs.html#5.2.2-1 --- src/lua/ldo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lua/ldo.c b/src/lua/ldo.c index d1bf786..30333bf 100644 --- a/src/lua/ldo.c +++ b/src/lua/ldo.c @@ -274,7 +274,7 @@ int luaD_precall (lua_State *L, StkId func, int nresults) { CallInfo *ci; StkId st, base; Proto *p = cl->p; - luaD_checkstack(L, p->maxstacksize); + luaD_checkstack(L, p->maxstacksize + p->numparams); func = restorestack(L, funcr); if (!p->is_vararg) { /* no varargs? */ base = func + 1; From bd444be3d60803770a75c46ef1b034f175c1f2ea Mon Sep 17 00:00:00 2001 From: runner Date: Mon, 9 Jan 2017 14:27:44 +0800 Subject: [PATCH 195/235] fix compile error on msvc2015 fix compile error on msvc2015 --- include/luasandbox/test/mu_test.h | 2 ++ src/luasandbox_defines.h | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/luasandbox/test/mu_test.h b/include/luasandbox/test/mu_test.h index e8d6bc0..313617e 100644 --- a/include/luasandbox/test/mu_test.h +++ b/include/luasandbox/test/mu_test.h @@ -12,8 +12,10 @@ #include #ifdef _WIN32 +#if _MSC_VER < 1900 #define snprintf _snprintf #endif +#endif #if defined(_MSC_VER) #define PRIuSIZE "Iu" diff --git a/src/luasandbox_defines.h b/src/luasandbox_defines.h index 578081d..aab10fa 100644 --- a/src/luasandbox_defines.h +++ b/src/luasandbox_defines.h @@ -10,8 +10,10 @@ #define luasandbox_defines_h_ #ifdef _WIN32 +#if _MSC_VER < 1900 #define snprintf _snprintf #endif +#endif #if __linux #define CLOSE_ON_EXEC "e" From 6ee682bb5e7c62bb68617d2a29c2da869dc74cb7 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 23 Jan 2017 07:14:07 -0800 Subject: [PATCH 196/235] Increment version for Sprint Jan09-Jan22 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 920cc24..d3e9f8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.1 LANGUAGES C) +project(luasandbox VERSION 1.2.2 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) From c0781ca5a7e5d28a6c389e559308167eb3120928 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 1 Feb 2017 09:43:30 -0800 Subject: [PATCH 197/235] Issue 184 Better error messages for inject_message callback returns --- CMakeLists.txt | 2 +- include/luasandbox/heka/sandbox.h | 7 +++++++ src/heka/sandbox.c | 26 ++++++++++++++++++++++---- src/heka/test/lua/aim.lua | 6 +++++- src/heka/test/lua/iim.lua | 10 ++++++++-- src/heka/test/test_heka_sandbox.c | 21 ++++++++++++--------- 6 files changed, 55 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d3e9f8b..70e7100 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.2 LANGUAGES C) +project(luasandbox VERSION 1.2.3 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index dc08b39..1c48bea 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -42,6 +42,13 @@ enum lsb_heka_pm_rv { LSB_HEKA_PM_ASYNC = -5 }; +enum lsb_heka_im_rv { + LSB_HEKA_IM_SUCCESS = 0, + LSB_HEKA_IM_ERROR = 1, // generic error for backward compatibility + LSB_HEKA_IM_CHECKPOINT = 2, + LSB_HEKA_IM_LIMIT = 3, +}; + typedef struct lsb_heka_sandbox lsb_heka_sandbox; typedef struct lsb_heka_stats { diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 94e1122..61401b8 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -212,9 +212,18 @@ static int inject_message_input(lua_State *lua) } lsb_heka_sandbox *hsb = lsb_get_parent(lsb); - if (hsb->cb.iim(hsb->parent, output.s, output.len, ncp, scp) != 0) { - return luaL_error(lua, "%s() failed: rejected by the callback", + int rv = hsb->cb.iim(hsb->parent, output.s, output.len, ncp, scp); + switch (rv) { + case LSB_HEKA_IM_SUCCESS: + break; + case LSB_HEKA_IM_CHECKPOINT: + return luaL_error(lua, "%s() failed: checkpoint update", im_func_name); + case LSB_HEKA_IM_ERROR: + // fall through + default: + return luaL_error(lua, "%s() failed: rejected by the callback rv: %d", + im_func_name, rv); } ++hsb->stats.im_cnt; hsb->stats.im_bytes += output.len; @@ -237,9 +246,18 @@ static int inject_message_analysis(lua_State *lua) size_t output_len = 0; const char *output = lsb_get_output(lsb, &output_len); lsb_heka_sandbox *hsb = lsb_get_parent(lsb); - if (hsb->cb.aim(hsb->parent, output, output_len) != 0) { - return luaL_error(lua, "%s() failed: rejected by the callback", + int rv = hsb->cb.aim(hsb->parent, output, output_len); + switch (rv) { + case LSB_HEKA_IM_SUCCESS: + break; + case LSB_HEKA_IM_LIMIT: + return luaL_error(lua, "%s() failed: injection limit exceeded", im_func_name); + case LSB_HEKA_IM_ERROR: + // fall through + default: + return luaL_error(lua, "%s() failed: rejected by the callback rv: %d", + im_func_name, rv); } ++hsb->stats.im_cnt; hsb->stats.im_bytes += output_len; diff --git a/src/heka/test/lua/aim.lua b/src/heka/test/lua/aim.lua index 762df3f..0433676 100644 --- a/src/heka/test/lua/aim.lua +++ b/src/heka/test/lua/aim.lua @@ -62,4 +62,8 @@ assert("inject_message() failed: array has mixed types" == err, string.format("r ok, err = pcall(inject_message, {}) assert(not ok) -assert("inject_message() failed: rejected by the callback" == err, string.format("received: %s", err)) +assert("inject_message() failed: rejected by the callback rv: 99" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_message, {}) +assert(not ok) +assert("inject_message() failed: injection limit exceeded" == err, string.format("received: %s", err)) diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua index b279d52..5264787 100644 --- a/src/heka/test/lua/iim.lua +++ b/src/heka/test/lua/iim.lua @@ -119,7 +119,13 @@ assert("bad argument #1 to '?' (lsb.heka_stream_reader expected, got number)" == -- String tests -inject_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\005") +local pbstr = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\005" +inject_message(pbstr) + +ok, err = pcall(inject_message, pbstr, 2) +if ok then error(string.format("test should have failed")) end +eerr = "inject_message() failed: checkpoint update" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) ok, err = pcall(inject_message, "\000") if ok then error(string.format("string test should have failed")) end @@ -128,5 +134,5 @@ assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) ok, err = pcall(inject_message, {}) if ok then error(string.format("test should have failed")) end -eerr = "inject_message() failed: rejected by the callback" +eerr = "inject_message() failed: rejected by the callback rv: 1" assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index a55a791..e3ff784 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -42,7 +42,7 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, const char *cp_string) { if (!pb) { - return 0; + return LSB_HEKA_IM_SUCCESS; } static int cnt = 0; @@ -60,11 +60,12 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x03iim\x4a\x02hn\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 165, .cp_numeric = NAN, .cp_string = "foo.log:123" }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, }; if (cnt >= (int)(sizeof results / sizeof results[0])) { fprintf(stderr, "tests and results are mis-matched\n"); - return 1; + return LSB_HEKA_IM_ERROR; } if (parent) { @@ -101,7 +102,7 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, if (ncp_failed) { fprintf(stderr, "test: %d cp_numeric expected: %g received: %g\n", cnt, results[cnt].cp_numeric, cp_numeric); - return 1; + return LSB_HEKA_IM_CHECKPOINT; } bool ncs_failed = false; @@ -116,10 +117,10 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, fprintf(stderr, "test: %d cp_string expected: %s received: %s\n", cnt, results[cnt].cp_string ? results[cnt].cp_string : "NULL", cp_string ? cp_string : "NULL"); - return 1; + return LSB_HEKA_IM_CHECKPOINT; } cnt++; - return 0; + return LSB_HEKA_IM_SUCCESS; } @@ -138,11 +139,12 @@ static int aim(void *parent, const char *pb, size_t pb_len) { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x1f\x7d\x09\x9e\xf5\x9d\x40\x1d\xa8\xaf\x6a\xff\xc3\x21\xeb\x42\x10\x80\x88\xe4\xaa\xa0\xa9\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 88, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x5b\x7d\xee\xa0\x02\xbc\x45\xbb\xaf\xa9\xcc\x2c\xdd\x65\xde\x45\x10\x80\x88\xdc\xad\xcd\xbf\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 110, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, }; if (cnt >= (int)(sizeof results / sizeof results[0])) { fprintf(stderr, "tests and results are mis-matched\n"); - return 1; + return LSB_HEKA_IM_LIMIT; } if (parent) { @@ -152,7 +154,8 @@ static int aim(void *parent, const char *pb, size_t pb_len) if (pb_len != results[cnt].pb_len) { fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); - return 1; + cnt++; + return 99; } if (cnt == 0) { @@ -176,7 +179,7 @@ static int aim(void *parent, const char *pb, size_t pb_len) if (!rv) return 1; } cnt++; - return 0; + return LSB_HEKA_IM_SUCCESS; } static int aim1(void *parent, const char *pb, size_t pb_len) @@ -204,7 +207,7 @@ static int aim1(void *parent, const char *pb, size_t pb_len) fprintf(stderr, "\n"); return 1; } - return 0; + return LSB_HEKA_IM_SUCCESS; } From 55b152e2939a07afd924511957559cbfc936e2c5 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 1 Feb 2017 10:06:09 -0800 Subject: [PATCH 198/235] Issue 185 sporadic unit test timing failure --- src/heka/test/lua/input.lua | 12 ++++++++++++ src/heka/test/test_heka_sandbox.c | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index 3e7156c..e7fe0cc 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -16,12 +16,22 @@ assert(not inject_payload) require "string" msg = {Timestamp = 8} +local function add_work() + local cnt = 0 + for i = 1, 1000 do + cnt = cnt + 1 + end +end + function process_message(cp) if cp == 0 then + add_work() return -2, "host specific failure" elseif cp == 1 then + add_work() return -1, "failed" elseif cp == 2 then + add_work() return 0, "ok" elseif cp == 3 then error("boom") @@ -32,6 +42,7 @@ function process_message(cp) elseif cp == 6 then error(string.rep("a", 255)) elseif cp == "string" then + add_work() return 0, "string" elseif cp == 7 then error(nil) @@ -40,5 +51,6 @@ function process_message(cp) elseif cp == 9 then assert(not is_running(), "not running") end + add_work() return 0, "no cp" end diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index e3ff784..f11e770 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -480,10 +480,10 @@ static char* test_pm_error() }; struct pm_result results[] = { - { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:27: boom" }, + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:37: boom" }, { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, - { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:33: aaaaaaaaaaaaaaaaaaaaaaaaa" + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:43: aaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, // >max error message From 78a2c40aad5da33663a33181934c4ca2b3adf732 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 6 Feb 2017 07:20:05 -0800 Subject: [PATCH 199/235] Increase the test runtime --- src/heka/test/lua/input.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index e7fe0cc..ab9526b 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -18,7 +18,7 @@ msg = {Timestamp = 8} local function add_work() local cnt = 0 - for i = 1, 1000 do + for i = 1, 10000 do cnt = cnt + 1 end end From 04c5b02aee46475795538378adf548f3e516ac5e Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sat, 4 Mar 2017 15:55:22 -0800 Subject: [PATCH 200/235] Issue 189 Checkpoint only updates should not increment the inject message count --- CMakeLists.txt | 2 +- src/heka/sandbox.c | 6 ++++-- src/heka/test/test_heka_sandbox.c | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 70e7100..0b9c7a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.3 LANGUAGES C) +project(luasandbox VERSION 1.2.4 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 61401b8..d8ad626 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -225,8 +225,10 @@ static int inject_message_input(lua_State *lua) return luaL_error(lua, "%s() failed: rejected by the callback rv: %d", im_func_name, rv); } - ++hsb->stats.im_cnt; - hsb->stats.im_bytes += output.len; + if (output.s) { + ++hsb->stats.im_cnt; + hsb->stats.im_bytes += output.len; + } return 0; } diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index f11e770..ee30a8a 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -594,7 +594,7 @@ static char* test_im_input() "Logger = 'iim'\n", &logger, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(8 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); mu_assert(332 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); From d55ee530ed7cda2d85a2682f82616ba0cffc9340 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 21 Apr 2017 12:52:32 -0700 Subject: [PATCH 201/235] Issue 194 - Change the *_limit cfg validation to unsigned --- CMakeLists.txt | 2 +- docs/sandbox.md | 65 +++++++++++++++++++++------------ include/luasandbox.h | 1 + src/luasandbox.c | 43 ++++++++++------------ src/test/test_generic_sandbox.c | 9 +++++ 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0b9c7a6..ac827a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.4 LANGUAGES C) +project(luasandbox VERSION 1.2.5 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/sandbox.md b/docs/sandbox.md index 54bacc7..ce0d841 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -2,26 +2,39 @@ ## Configuration -* **output_limit** - the largest output string an input or analysis plugin can inject into the host (bytes, default 64KiB) -* **memory_limit** - the maximum amount of memory a plugin can use before being terminated (bytes, default 8MiB) -* **instruction_limit** - the maximum number of Lua instructions a plugin can execute in a single API function call (count, default 1MM) -* **path** - The path used by require to search for a Lua loader. See [package loaders](http://www.lua.org/manual/5.1/manual.html#pdf-package.loaders) - for the path syntax. By default no paths are set in the sandbox and everything has been moved to a sandbox configuration table. -* **cpath** - The path used by require to search for a C loader (same notes as above) -* **disabled_modules** - Hash specifying which modules should be completely inaccessible. The existence of the key in the table will - disable the module. +* **output_limit** - the largest output string an input or analysis plugin can + inject into the host (bytes (unsigned), default 65536, 0 for unlimited*) +* **memory_limit** - the maximum amount of memory a plugin can use before being + terminated (bytes (unsigned), default 8388608, 0 for unlimited*) +* **instruction_limit** - the maximum number of Lua instructions a plugin can + execute in a single API function call (count (unsigned), default 1000000, 0 + for unlimited) +* **path** - The path used by require to search for a Lua loader. See + [package loaders](http://www.lua.org/manual/5.1/manual.html#pdf-package.loaders) + for the path syntax. By default no paths are set in the sandbox and + everything has been moved to a sandbox configuration table. +* **cpath** - The path used by require to search for a C loader (same notes as + above) +* **disabled_modules** - Hash specifying which modules should be completely + inaccessible. The existence of the key in the table will disable the module. ```lua disabled_modules = {io = 1} ``` -* **remove_entries** - Hash specifying which functions within a module should be inaccessible. +* **remove_entries** - Hash specifying which functions within a module should be + inaccessible ```lua remove_entries = { os = {"getenv", "execute"}, string = {"dump"} } ``` -* **log_level** - Integer specifying the syslog severity level, when set to debug (7) the print function will be wired to the specified logger -* *user defined* any other variable (string, bool, number, table) is passed through as-is and available via [read_config](#read_config) +* **log_level** - syslog severity level, when set to debug (7) the print + function will be wired to the specified logger (default error (3)) +* *user defined* any other variable (string, bool, number, table) is passed + through as-is and available via [read_config](#read_config) + +*_0 == SIZE_MAX which in not necessarily the upper limit of the +configuration range UINT_MAX_ ## Lua functions exposed to C by the core sandbox @@ -32,23 +45,26 @@ lsb_pcall_teardown() when defining an API. ### require -By default only the base library is loaded additional libraries must be loaded with require(). +By default only the base library is loaded, additional libraries must be loaded +with require(). *Arguments* - libraryName (string) *Return* -- a table - For non user provided libraries the table is also globally registered - with the library name. User provided libraries may implement there own semantics - i.e. the grammar libraries return a table but do not globally register the table name - (see the individual module documentation for the correct usage). +- a table - For non user provided libraries the table is also globally + registered with the library name. User provided libraries may implement their + own semantics i.e. the grammar libraries return a table but do not globally + register the table name (see the individual module documentation for the + correct usage). *Notes* The following modules have been modified, as described, for use in the sandbox. - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) - - The require() function has been modified to not expose any of the package table to the sandbox. + - The require() function has been modified to not expose any of the package + table to the sandbox. - [math](http://www.lua.org/manual/5.1/manual.html#5.6) - Added Functions - erf(x) - Returns the error function value for x. @@ -72,7 +88,7 @@ cannot exceed the output_limit configuration parameter. See lsb_get_output() to connect the output to the host application. *Arguments* -- arg (number, string, bool, nil, userdata implementing output support) - Lua +- arg (number, string, bool, nil, userdata implementing output support) - Lua variable or literal to be appended the output buffer *Return* @@ -105,17 +121,20 @@ The best place to start is with some examples: #### Lua Functions Exposed to C - int **process** (double) - - exposes a process function that takes a test case number as its argument and returns and integer result code. + - exposes a process function that takes a test case number as its argument + and returns and integer result code. - void **report** (double) - - exposes a report function that takes a test case number as its argument and returns nothing. + - exposes a report function that takes a test case number as its argument + and returns nothing. #### C Functions Exposed to Lua - void **write_output** (optionalArg1, optionalArg2, optionalArgN) - - captures whatever is in the output buffer for use by the host application, appending any optional arguments - (optional arguments have the same restriction as output). + - captures whatever is in the output buffer for use by the host application, + appending any optional arguments (optional arguments have the same + restriction as output). ### Heka Sandbox API [Heka Sandbox API](/lua_sandbox/doxygen/heka_2sandbox_8h.html) - +q diff --git a/include/luasandbox.h b/include/luasandbox.h index 7718a63..cd89842 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -33,6 +33,7 @@ #define LSB_MEMORY_LIMIT "memory_limit" #define LSB_INSTRUCTION_LIMIT "instruction_limit" #define LSB_OUTPUT_LIMIT "output_limit" +#define LSB_LOG_LEVEL "log_level" #define LSB_LUA_PATH "path" #define LSB_LUA_CPATH "cpath" #define LSB_NIL_ERROR "" diff --git a/src/luasandbox.c b/src/luasandbox.c index e9615df..456ab6f 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -215,12 +215,12 @@ static int unprotected_panic(lua_State *lua) } -static int get_int(lua_State *lua, int idx, const char *item) +static int get_number(lua_State *lua, int idx, const char *item) { lua_getfield(lua, idx, item); - int i = (int)lua_tointeger(lua, -1); + double d = lua_tonumber(lua, -1); lua_pop(lua, 1); - return i; + return d; } @@ -246,31 +246,25 @@ static int check_string(lua_State *L, int idx, const char *name, } -static int check_int(lua_State *L, int idx, const char *name, int val) +static int check_unsigned(lua_State *L, int idx, const char *name, unsigned val) { lua_getfield(L, idx, name); - int t = lua_type(L, -1); - lua_Integer i; - switch (t) { + double d; + switch (lua_type(L, -1)) { case LUA_TNUMBER: - i = lua_tointeger(L, -1); - if (i < 0) { - lua_pushfstring(L, "%s must be set to a positive integer", name); - return 1; - } - if (i > INT_MAX) { - lua_pushfstring(L, "%s cannot exceed INT_MAX %d", name, INT_MAX); + d = lua_tonumber(L, -1); + if (d < 0 || d > UINT_MAX) { + lua_pushfstring(L, "%s must be an unsigned int", name); return 1; } break; case LUA_TNIL: // add the default to the config - lua_pushinteger(L, val); + lua_pushnumber(L, (lua_Number)val); lua_setglobal(L, name); break; // use the default default: lua_pushfstring(L, "%s must be set to a number", name); return 1; - break; } lua_pop(L, 1); return 0; @@ -291,13 +285,16 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger *logger) int ret = luaL_dostring(L, cfg); if (ret) goto cleanup; - ret = check_int(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); + ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); + if (ret) goto cleanup; + + ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_MEMORY_LIMIT, 8 * 1024 * 1024); if (ret) goto cleanup; - ret = check_int(L, LUA_GLOBALSINDEX, LSB_MEMORY_LIMIT, 8 * 1024 * 1024); + ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_INSTRUCTION_LIMIT, 1000000); if (ret) goto cleanup; - ret = check_int(L, LUA_GLOBALSINDEX, LSB_INSTRUCTION_LIMIT, 1000000); + ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_LOG_LEVEL, 3); if (ret) goto cleanup; ret = check_string(L, LUA_GLOBALSINDEX, LSB_LUA_PATH, NULL); @@ -476,10 +473,10 @@ lsb_lua_sandbox* lsb_create(void *parent, copy_table(lsb->lua, lua_cfg, &lsb->logger); lua_pop(lua_cfg, 2); lua_close(lua_cfg); - size_t ml = get_int(lsb->lua, -1, "memory_limit"); - size_t il = get_int(lsb->lua, -1, "instruction_limit"); - size_t ol = get_int(lsb->lua, -1, "output_limit"); - int log_level = get_int(lsb->lua, -1, "log_level"); + size_t ml = (size_t)get_number(lsb->lua, -1, LSB_MEMORY_LIMIT); + size_t il = (size_t)get_number(lsb->lua, -1, LSB_INSTRUCTION_LIMIT); + size_t ol = (size_t)get_number(lsb->lua, -1, LSB_OUTPUT_LIMIT); + double log_level = get_number(lsb->lua, -1, LSB_LOG_LEVEL); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); lua_pushlightuserdata(lsb->lua, lsb); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c index 373978b..a366854 100644 --- a/src/test/test_generic_sandbox.c +++ b/src/test/test_generic_sandbox.c @@ -189,6 +189,9 @@ static char* test_create() sb = lsb_create(NULL, "lua/counter.lua", cfg, NULL); mu_assert(sb, "lsb_create() failed"); lsb_destroy(sb); + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 3e9", NULL); + mu_assert(sb, "lsb_create() failed"); + lsb_destroy(sb); return NULL; } @@ -204,6 +207,12 @@ static char* test_create_error() sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 'aaa'", NULL); mu_assert(!sb, "lsb_create() invalid config"); + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = -1", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 5e9", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + sb = lsb_create(NULL, "lua/counter.lua", "instruction_limit = 'aaa'", NULL); mu_assert(!sb, "lsb_create() invalid config"); From 0c94b5fdd93b0979301e84a83901fbddc0903214 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 1 May 2017 06:49:21 -0700 Subject: [PATCH 202/235] Add the contribution guidelines to the README --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index bee7ec8..d4770ed 100644 --- a/README.md +++ b/README.md @@ -60,3 +60,24 @@ infrastructure and make it embeddable into any tool or language. ctest cpack -G TGZ # (DEB|RPM|ZIP) + +## Releases + +* The master branch is the current release and is considered stable at all + times. +* New versions can be released as frequently as every two weeks (our sprint + cycle). The only exception would be for a high priority patch. +* All active work is flagged with the sprint milestone and tracked in the + project dashboard. +* New releases occur the day after the sprint finishes. + * The version in the dev branch is updated + * The changes are merged into master + * A new tag is created + +## Contributions + +* All pull requests must be made against the dev branch, direct commits to + master are not permitted. +* All non trivial contributions should start with an issue being filed (if it is + a new feature please propose your design/approach before doing any work as not + all feature requests are accepted). From 6a9845abfa266f2c1a2dda79c22483b0da3555d6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 3 May 2017 10:19:38 -0700 Subject: [PATCH 203/235] Switch to gitbook for documentation generation --- README.md | 11 +-- cmake/doxygen.cmake | 4 +- docs/cli/index.md | 3 +- docs/heka/analysis.md | 26 +++--- docs/heka/index.md | 13 +-- docs/heka/input.md | 9 +- docs/heka/output.md | 22 ++--- docs/sandbox.md | 7 +- docs/util/message_matcher.md | 9 +- gen_gh_pages.lua | 167 +++++------------------------------ 10 files changed, 75 insertions(+), 196 deletions(-) diff --git a/README.md b/README.md index d4770ed..729e91d 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,10 @@ environment including functionality like global data preservation/restoration on shutdown/startup, output collection in textual or binary formats and an array of parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) -These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/mozilla-services/hindsight). -The goal was to decouple the Heka/Hindsight functionality from any particular -infrastructure and make it embeddable into any tool or language. +These libraries and utilities have been mostly extracted from +[Hindsight](https://github.com/mozilla-services/hindsight). The goal was to +decouple the Heka/Hindsight functionality from any particular infrastructure and +make it embeddable into any tool or language. ### Features @@ -39,8 +40,8 @@ infrastructure and make it embeddable into any tool or language. #### Optional (used for documentation) * Graphviz (2.28.0) - http://graphviz.org/Download..php -* Doxygen (1.8.11+)- http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc -* pandoc (1.17) - http://pandoc.org/ +* Doxygen (1.8.11+) - http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc +* gitbook (2.3) - https://www.gitbook.com/ * lua (5.1) - https://www.lua.org/download.html ### CMake Build Instructions diff --git a/cmake/doxygen.cmake b/cmake/doxygen.cmake index 063d70e..90749a2 100644 --- a/cmake/doxygen.cmake +++ b/cmake/doxygen.cmake @@ -4,7 +4,7 @@ find_package(Doxygen QUIET) find_program(LUA_EXE lua QUIET) -find_program(PANDOC_EXE pandoc QUIET) +find_program(gitbook gitbook QUIET) if(DOXYGEN_FOUND AND LUA_EXE AND PANDOC_EXE) set(DOXYCONF_IN ${CMAKE_SOURCE_DIR}/doxygen.in.conf) set(DOXYCONF_OUT ${CMAKE_BINARY_DIR}/doxygen.conf) @@ -37,7 +37,7 @@ PROJECT_NUMBER = ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION add_custom_target(docs ${DOXYGEN_EXECUTABLE} ${DOXYCONF_OUT} COMMAND lua gen_gh_pages.lua "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}" - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) + "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) else() message("The optional documentation tools were not found; the doc target has not been created") endif() diff --git a/docs/cli/index.md b/docs/cli/index.md index 80ea510..f0594f9 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -19,5 +19,6 @@ notes: All output is written to stdout and all log/error messages are written to stderr. ``` -See the [message matcher](../util/message_matcher.html) documentation for more details about the message_matcher expression. +See the [message matcher](/util/message_matcher.md) documentation for more +details about the message_matcher expression. diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index 24b2256..2ed7587 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -22,7 +22,7 @@ Hindsight reference implementation. ### process_message Called when the host has a message available for analysis. Usually used in -combination with a [message matcher](../util/message_matcher.html) expression. +combination with a [message matcher](/util/message_matcher.md) expression. Recommenation: specify this as a `message_matcher` configuration option. @@ -86,15 +86,13 @@ standard indexing, which is one-based. * Pid * Fields[*name*] * fieldIndex (unsigned) - only used in combination with the Fields variableName - use to retrieve a specific instance of a repeated field *name*; - zero indexed +use to retrieve a specific instance of a repeated field *name*; zero indexed * arrayIndex (unsigned) - only used in combination with the Fields variableName - use to retrieve a specific element out of a field containing an array; - zero indexed +use to retrieve a specific element out of a field containing an array; zero +indexed * zeroCopy (bool, optional default false) - returns a userdata place holder for - the message variable (only valid for string types). Non string headers - throw an error during construction, non string fields throw an error on - data retrieval. +the message variable (only valid for string types). Non string headers throw an +error during construction, non string fields throw an error on data retrieval. *Return* * value (number, string, bool, nil, userdata depending on the type of variable @@ -110,11 +108,11 @@ error. userdata object containing a Heka protobuf binary string. *Return* -* msg ([Heka message table (array fields)](message.html#array-based-message-fields)) - with the value member always being an array (even if there is only a single - item). This format makes working with the output more consistent. The wide - variation in the inject table formats is to ease the construction of the - message especially when using an LPeg grammar transformation. +* msg ([Heka message table (array fields)](message.md#array-based-message-fields)) +with the value member always being an array (even if there is only a +single item). This format makes working with the output more consistent. The +wide variation in the inject table formats is to ease the construction of the +message especially when using an LPeg grammar transformation. ### inject_message @@ -127,7 +125,7 @@ provide in the message table, if no value is provided it defaults to the appropriate configuration value. *Arguments* -* msg ([Heka message table](message.html)) +* msg ([Heka message table](message.md)) *Return* * none (throws an error if the table does not match the Heka message schema) diff --git a/docs/heka/index.md b/docs/heka/index.md index 2a898b7..d17f813 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -3,8 +3,8 @@ ## Overview This document describes the 1.0 release of the Heka sandbox API built on the -[Generic Sandbox Interface](../sandbox.html). The 1.0 release is a refined -implementation of its predecessor which was developed in +[Generic Sandbox Interface](/sandbox.md). The 1.0 release is a refined +implementation of its predecessor which was developed in [Heka](https://github.com/mozilla-services/heka). The goal is to decople it from Go and make it easily embeddable in any language. The Go version of Heka has been deprecated and replaced by [Hindsight](https://github.com/mozilla-services/hindsight). @@ -37,13 +37,14 @@ There are a few intentional changes between tho original Heka sandbox and this v #### Input Sandbox -1. [create_stream_reader](input.html#create_stream_reader) function was added. -1. [is_running](input.html#is_running) function was added. +1. [create_stream_reader](input.md#createstreamreader) function was added. 1. +[is_running](input.md#is_running) function was added. #### Output Sandbox -1. [update_checkpoint](output.html#update_checkpoint) was added for batch and asynchronous processing. -1. [create_message_matcher](output.html#create_message_matcher) function was added. +1. [update_checkpoint](output.md#update_checkpoint) was added for batch and +asynchronous processing. 1. +[create_message_matcher](output.md#createmessagematcher) function was added. ### Removals diff --git a/docs/heka/input.md b/docs/heka/input.md index 7c9f421..46fabaf 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -55,8 +55,8 @@ shutdown status. ### decode_message -Converts a Heka protobuf encoded message string into a Lua table. -See [decode_message](analysis.html#decode_message) for details. +Converts a Heka protobuf encoded message string into a Lua table. See +[decode_message](analysis.md#decodemessage) for details. ### inject_message @@ -68,7 +68,7 @@ to the values provide in the message table, if no value is provided it defaults to the appropriate configuration value. *Arguments* -* msg ([Heka message table](message.html), +* msg ([Heka message table](message.md), [Heka stream reader](#heka-stream-reader-methods), Heka protobuf string, or nil (if only updating the checkpoint)) @@ -133,8 +133,7 @@ flag is not accepted here. ```lua local ts = hsr:read_message("Timestamp") -``` -See [read_message](analysis.html#read_message) for details. +``` See [read_message](analysis.md#readmessage) for details. ## Modes of Operation diff --git a/docs/heka/output.md b/docs/heka/output.md index 167bff4..5836170 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -18,7 +18,7 @@ Hindsight reference implementation. ### process_message Called when the host has a message available for analysis. Usually used in -combination with a [message matcher](../util/message_matcher.html) expression. +combination with a [message matcher](/util/message_matcher.md) expression. Recommenation: specify this as a `message_matcher` configuration option. @@ -67,13 +67,13 @@ Provides access to the sandbox configuration variables. ### read_message -Provides access to the Heka message data. -See [read_message](analysis.html#read_message) for details. +Provides access to the Heka message data. See +[read_message](analysis.md#readmessage) for details. ### decode_message -Converts a Heka protobuf encoded message string into a Lua table. -See [decode_message](analysis.html#decode_message) for details. +Converts a Heka protobuf encoded message string into a Lua table. See +[decode_message](analysis.md#decodemessage) for details. ### encode_message @@ -88,12 +88,12 @@ Note: this operation uses the internal output buffer so it is goverened by the `output_limit` configuration setting. *Arguments* -* msg ([Heka message table](message.html)) +* msg ([Heka message table](message.md)) * framed (bool default: false) A value of true includes the framing header *Return* -* heka_pb (string) - Heka protobuf binary string, framed as specified or an - error is thrown +* heka_pb (string) - Heka protobuf binary string, framed as specified + or an error is thrown ### create_message_matcher @@ -101,7 +101,7 @@ Returns a Heka protocol buffer message matcher; used to dynamic filter messages sent to the output plugin. *Arguments* -* message_matcher [message matcher](../util/message_matcher.html) +* message_matcher [message matcher](/util/message_matcher.md) *Return* * message_matcher (userdata) - or an error is thrown @@ -110,7 +110,7 @@ sent to the output plugin. #### Example -See: [heka_tcp_matcher.lua](https://github.com/mozilla-services/lua_sandbox_extensions/socket/sandboxes/heka/output/heka_tcp_matcher.html) +See: [heka_tcp_matcher.lua](https://mozilla-services.github.io/lua_sandbox_extensions/socket/sandboxes/heka/output/heka_tcp_matcher.html) ### update_checkpoint @@ -398,7 +398,7 @@ to the destination and returns one of the following values: * skip (-2) - the message was intentionally not queued * retry (-3) - the message was not successfully queued and the host will call `process_message` again, with the same message, after a one second delay -* When an asynchronously sent message is acknowledged [update_checkpoint](#update_checkpoint) +* When an asynchronously sent message is acknowledged [update_checkpoint](#updatecheckpoint) **MUST** be called to advance the checkpoint to that specific message #### Example Kafka Output diff --git a/docs/sandbox.md b/docs/sandbox.md index ce0d841..813c61f 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -31,7 +31,7 @@ remove_entries = { * **log_level** - syslog severity level, when set to debug (7) the print function will be wired to the specified logger (default error (3)) * *user defined* any other variable (string, bool, number, table) is passed - through as-is and available via [read_config](#read_config) + through as-is and available via [read_config](#readconfig) *_0 == SIZE_MAX which in not necessarily the upper limit of the configuration range UINT_MAX_ @@ -116,7 +116,7 @@ The best place to start is with some examples: ### Unit Test API -[Unit Test API](/lua_sandbox/doxygen/test_2sandbox_8h.html) +[Unit Test API](https://mozilla-services.github.io/lua_sandbox/doxygen/test_2sandbox_8h.html) #### Lua Functions Exposed to C @@ -136,5 +136,4 @@ The best place to start is with some examples: ### Heka Sandbox API -[Heka Sandbox API](/lua_sandbox/doxygen/heka_2sandbox_8h.html) -q +[Heka Sandbox API](https://mozilla-services.github.io/lua_sandbox/doxygen/heka_2sandbox_8h.html) diff --git a/docs/util/message_matcher.md b/docs/util/message_matcher.md index 5f49d12..9021435 100644 --- a/docs/util/message_matcher.md +++ b/docs/util/message_matcher.md @@ -1,7 +1,7 @@ # Message Matcher Syntax -The message matcher allows sandboxes to select which messages they want to consume -(see [Heka Message Structure](../heka/message.html)) +The message matcher allows sandboxes to select which messages they want to +consume (see [Heka Message Structure](/heka/message.md)) ## Examples @@ -44,8 +44,9 @@ The message matcher allows sandboxes to select which messages they want to consu ## Message Variables -All message variables must be on the left hand side of the relational comparison - +* All message variables must be on the left hand side of the relational +comparison + ### String * Uuid - 16 byte raw binary type 4 UUID (useful for partitioning data) diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index 331fe84..57b2825 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -6,158 +6,37 @@ require "os" require "io" require "string" -local function output_css() - local fh = assert(io.open("gh-pages/docs.css", "w")) - fh:write([[ - html { - height: 100%; - } - - body { - font-family:verdana, arial, sans-serif; - font-size:small; - width: 90%; - background:white; - margin-left: auto; - margin-right: auto; - height: 100%; - } - - h1 { - border-bottom:1px black solid; - } - - h2 { - border-bottom:1px gray solid; - } - - h3 { - border-bottom:1px lightgray solid; - } - - h4 { - border-bottom:1px black dotted; - } - - h5 { - border-bottom:1px gray dotted; - } - h6 { - border-bottom:1px lightgray dotted; - } - - #title { - width:100%; - font-size:large; - font-weight: bold; - font-style: normal; - font-variant: normal; - text-transform: uppercase; - letter-spacing: .1em; - } - - .menu { - display:table-cell; - font-size: small; - font-weight: normal; - font-style: normal; - font-variant: small-caps; - color: #000000; - height: 100%; - padding-right: 10px; - white-space: nowrap; - } - - .menu ul{ - list-style-type: none; - margin-left: 5px; - margin-right: 0px; - padding-left: 10px; - padding-right: 0px; - } - - .main-content { - border-left:1px lightgray dotted; - padding-left:10px; - display:table-cell; - width:100%; - } - - code, pre.code, pre.sourceCode - { - background-color: whitesmoke; - } - ]]) - fh:close() -end - - -local function output_menu(before, after, paths, version) - local fh = assert(io.open(before, "w")) - fh:write(string.format('
              Lua Sandbox Library (%s)
              \n', version)) - fh:write([[
              - - - - - - -
              -
              +local function output_menu(output_dir, version) + local fh = assert(io.open(string.format("%s/SUMMARY.md", output_dir), "w")) + fh:write(string.format("* [Lua Sandbox Library (%s)](README.md)\n\n", version)) + fh:write([[ +* [Generic Sandbox](sandbox.md) +* [Heka Sandbox](heka/index.md) + * [Input Interface](heka/input.md) + * [Analysis Interface](heka/analysis.md) + * [Output Interface](heka/output.md) + * [Message Schema](heka/message.md) + * [Message Matcher](util/message_matcher.md) +* [Command Line Tools](cli/index.md) +* [Source Documentation](https://mozilla-services.github.io/lua_sandbox/doxygen/index.html) +* [Sandbox Extensions](https://mozilla-services.github.io/lua_sandbox_extensions) ]]) fh:close() - - fh = assert(io.open(after, "w")) - fh:write("
              \n") - fh:close() -end - - -local function md_to_html(paths, version) - local before = "/tmp/before.html" - local after = "/tmp/after.html" - output_menu(before, after, paths, version) - - local fh = assert(io.popen("find gh-pages -name \\*.md")) - for line in fh:lines() do - local css_path = "/lua_sandbox/docs.css" - local cmd = string.format("pandoc --from markdown_github-hard_line_breaks --to html --standalone -B %s -A %s -c %s -o %s.html %s", before, after, css_path, line:sub(1, #line -3), line) - local rv = os.execute(cmd) - if rv ~= 0 then error(cmd) end - os.remove(line) - end - fh:close() - - os.remove(before) - os.remove(after) end local args = {...} local function main() - local rv = os.execute("rsync -rav docs/ gh-pages/") - if rv ~= 0 then error"rsync" end - output_css() - local paths = {entries = {}} - md_to_html(paths, args[1]) + local output_dir = string.format("%s/gb-source", arg[3]) + local rv = os.execute(string.format("rsync -rav docs/ %s/", output_dir)) + if rv ~= 0 then error"rsync setup" end + + os.execute(string.format("mv %s/index.md %s/README.md", output_dir, output_dir)) + output_menu(output_dir, args[1]) + os.execute(string.format("gitbook build %s", output_dir)) + local rv = os.execute(string.format("rsync -rav %s/_book/ %s/", output_dir, "gh-pages/")) + if rv ~= 0 then error"rsync publish" end end main() From 94d54341c6ac4e608baf8702e8f37b16b5ce5ce9 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 3 May 2017 10:45:49 -0700 Subject: [PATCH 204/235] Don't test the timer results on systems with low resolution clocks --- src/heka/test/lua/analysis.lua | 6 ++++-- src/heka/test/lua/input.lua | 4 +++- src/heka/test/test_heka_sandbox.c | 24 ++++++++++++++++++------ 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/heka/test/lua/analysis.lua b/src/heka/test/lua/analysis.lua index fac796d..f862d90 100644 --- a/src/heka/test/lua/analysis.lua +++ b/src/heka/test/lua/analysis.lua @@ -15,11 +15,13 @@ function process_message() return 0 end +local work_cnt = 1 function timer_event(ns, shutdown) local cnt = 0 - for i=1, 10000 do - cnt = cnt + 1 -- make sure we have something to measure even with a low res clock + for i=1, work_cnt * 1000 do + cnt = cnt + 1 end + work_cnt = work_cnt + 1 if ns == 1e9 then assert(shutdown, "not shutting down") diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua index ab9526b..8a8aa7e 100644 --- a/src/heka/test/lua/input.lua +++ b/src/heka/test/lua/input.lua @@ -16,11 +16,13 @@ assert(not inject_payload) require "string" msg = {Timestamp = 8} +local work_cnt = 1 local function add_work() local cnt = 0 - for i = 1, 10000 do + for i = 1, work_cnt * 1000 do cnt = cnt + 1 end + work_cnt = work_cnt + 1 end function process_message(cp) diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index ee30a8a..2d07362 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -19,6 +19,8 @@ #include "luasandbox/heka/sandbox.h" #include "luasandbox_output.h" +static unsigned long long clockres = 1; + // {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; @@ -375,8 +377,10 @@ static char* test_timer_event() mu_assert(0 == stats.pm_failures, "received %llu", stats.pm_failures); mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); - mu_assert(0 < stats.te_avg, "received %g", stats.te_avg); - mu_assert(0 < stats.te_sd, "received %g", stats.te_sd); + if (clockres <= 100) { + mu_assert(0 < stats.te_avg, "received %g res %llu", stats.te_avg, clockres); + mu_assert(0 < stats.te_sd, "received %g", stats.te_sd); + } e = lsb_heka_destroy_sandbox(hsb); @@ -464,8 +468,10 @@ static char* test_pm_input() mu_assert(5 == stats.pm_cnt, "expected %llu", stats.pm_cnt); mu_assert(1 == stats.pm_failures, "expected %llu", stats.pm_failures); mu_assert(0 < stats.mem_cur, "expected %llu", stats.mem_cur); - mu_assert(0 < stats.pm_avg, "received %g", stats.pm_avg); - mu_assert(0 < stats.pm_sd, "received %g", stats.pm_sd); + if (clockres <= 100) { + mu_assert(0 < stats.pm_avg, "received %g res %llu", stats.pm_avg, clockres); + mu_assert(0 < stats.pm_sd, "received %g", stats.pm_sd); + } e = lsb_heka_destroy_sandbox(hsb); return NULL; } @@ -480,10 +486,10 @@ static char* test_pm_error() }; struct pm_result results[] = { - { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:37: boom" }, + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:39: boom" }, { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, - { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:43: aaaaaaaaaaaaaaaaaaaaaaaaa" + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:45: aaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, // >max error message @@ -783,6 +789,12 @@ static char* benchmark_decode_message() static char* all_tests() { +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + clockres = ts.tv_sec * 1000000000ULL + ts.tv_nsec; +#endif + mu_run_test(test_api_assertion); mu_run_test(test_create_input_sandbox); mu_run_test(test_create_analysis_sandbox); From 71b87efb85be982e7720f34543be7060ab7218df Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 12 May 2017 08:13:15 -0700 Subject: [PATCH 205/235] Correct documentation formatting issues --- docs/heka/analysis.md | 2 +- docs/heka/index.md | 40 +++++++++++++++++++++++++--------------- docs/heka/input.md | 15 ++++++++------- docs/heka/output.md | 23 +++++++++++++---------- 4 files changed, 47 insertions(+), 33 deletions(-) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index 2ed7587..d38481b 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -2,7 +2,7 @@ ## Recommendations Since the sandbox does not run in isolation there are some expectations of how -the host infrastructure behaves. The current recommendation are based on the +the host infrastructure behaves. The current recommendations are based on the Hindsight reference implementation. ## Disabled Functionality diff --git a/docs/heka/index.md b/docs/heka/index.md index d17f813..188d272 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -7,49 +7,59 @@ This document describes the 1.0 release of the Heka sandbox API built on the implementation of its predecessor which was developed in [Heka](https://github.com/mozilla-services/heka). The goal is to decople it from Go and make it easily embeddable in any language. The Go version of Heka has -been deprecated and replaced by [Hindsight](https://github.com/mozilla-services/hindsight). +been deprecated and replaced by +[Hindsight](https://github.com/mozilla-services/hindsight). ## Sandbox API Changes from the Go Heka Sandbox -There are a few intentional changes between tho original Heka sandbox and this version. +There are a few intentional changes between tho original Heka sandbox and this +version. ### Changes #### Input Sandbox -1. `inject_message` accepts a numeric or string checkpoint as the second argument -1. `process_message` receives the checkpoint value as the first argument (if it was provided by `inject_message`). +1. `inject_message` accepts a numeric or string checkpoint as the second +argument +1. `process_message` receives the checkpoint value as the first argument (if it +was provided by `inject_message`). #### Output Sandbox -1. `process_message` receives a sequence_id as the first argument (if it was provided by `update_checkpoint`). - Extended return codes have been added to support skipping, retrying, batching, and asynchronous output. +1. `process_message` receives a sequence_id as the first argument (if it was +provided by `update_checkpoint`). Extended return codes have been added to +support skipping, retrying, batching, and asynchronous output. #### Analysis/Output Sandbox 1. `read_message` - * returns `nil` for optional header fields if they don't exist instead of an empty string or zero + * returns `nil` for optional header fields if they don't exist instead of an + empty string or zero * added a `framed` parameter to retrive the raw message with stream framing - * added a `size` parameter to retrive size of the raw message without having to copy it down -1. `timer_event` has a second parameter `shutdown` that is set to true when the sandbox is exiting. + * added a `size` parameter to retrive size of the raw message without having + to copy it down +1. `timer_event` has a second parameter `shutdown` that is set to true when the +sandbox is exiting. ### Additions #### Input Sandbox -1. [create_stream_reader](input.md#createstreamreader) function was added. 1. -[is_running](input.md#is_running) function was added. +1. [create_stream_reader](input.md#createstreamreader) function was added. +1. [is_running](input.md#is_running) function was added. #### Output Sandbox 1. [update_checkpoint](output.md#update_checkpoint) was added for batch and -asynchronous processing. 1. -[create_message_matcher](output.md#createmessagematcher) function was added. +asynchronous processing. +1. [create_message_matcher](output.md#createmessagematcher) function was added. ### Removals -1. The `write_message` API was removed; messages are immutable and this API broke that rule. -1. The `read_next_field` API was removed; instead the raw message should be decoded and the Lua table iterated. +1. The `write_message` API was removed; messages are immutable and this API +broke that rule. +1. The `read_next_field` API was removed; instead the raw message should be +decoded and the Lua table iterated. ### Notes diff --git a/docs/heka/input.md b/docs/heka/input.md index 46fabaf..b626638 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -2,7 +2,7 @@ ## Recommendations Since the sandbox does not run in isolation there are some expectations of how -the host infrastructure behaves. The current recommendation are based on the +the host infrastructure behaves. The current recommendations are based on the Hindsight reference implementation. ## Disabled Functionality @@ -21,13 +21,14 @@ Entry point for message creation. *Arguments* * checkpoint (nil number, string) - value of the last checkpoint value passed - into `inject_message` +into `inject_message` *Return* * status_code (number) - success (less than or equal to zero) - fatal error (greater than zero) -* status_message (optional: string) logged when the status code is less than zero +* status_message (optional: string) logged when the status code is less than +zero ## Available C Functions (called from the plugin) @@ -102,7 +103,7 @@ local found, consumed, need = hsr:find_message(buf) *Arguments* * buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data - or a userdate file object + or a userdata file object * decode (bool default: true) - true if the framed message should be protobuf decoded @@ -133,13 +134,13 @@ flag is not accepted here. ```lua local ts = hsr:read_message("Timestamp") -``` See [read_message](analysis.md#readmessage) for details. +``` +See [read_message](analysis.md#readmessage) for details. ## Modes of Operation ### Run Once -* Set the `ticker_interval` to zero and return from `process_message` when you - are done. +* Set the `ticker_interval` to zero and return from `process_message` done. * The `instruction_limit` configuration can be set if desired. #### Example startup ping diff --git a/docs/heka/output.md b/docs/heka/output.md index 5836170..dfb5b9c 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -2,7 +2,7 @@ ## Recommendations Since the sandbox does not run in isolation there are some expectations of how -the host infrastructure behaves. The current recommendation are based on the +the host infrastructure behaves. The current recommendations are based on the Hindsight reference implementation. ## Disabled Functionality @@ -97,8 +97,8 @@ Note: this operation uses the internal output buffer so it is goverened by the ### create_message_matcher -Returns a Heka protocol buffer message matcher; used to dynamic filter messages -sent to the output plugin. +Returns a Heka protocol buffer message matcher; used to dynamically filter +messages sent to the output plugin. *Arguments* * message_matcher [message matcher](/util/message_matcher.md) @@ -387,19 +387,22 @@ end #### Asynchronous -* `async_buffer_size` **RECOMMENDED** that this configuration variable be set and consumed by the host -* `process_message` is called with a sequence_id parameter and asynchronously sends the message/transformation -to the destination and returns one of the following values: +* `async_buffer_size` Recommendation: this configuration variable should be set +and consumed by the host +* `process_message` is called with a sequence_id parameter and asynchronously +sends the message/transformation to the destination and returns one of the +following values: * asynchronous (-5) - the message was successfully queued * failure (-1) - the message cannot be queue * the failure count is incremented * any optional error message is written to the log * the message is skipped * skip (-2) - the message was intentionally not queued - * retry (-3) - the message was not successfully queued and the host will call `process_message` - again, with the same message, after a one second delay -* When an asynchronously sent message is acknowledged [update_checkpoint](#updatecheckpoint) -**MUST** be called to advance the checkpoint to that specific message + * retry (-3) - the message was not successfully queued and the host will call + `process_message` again, with the same message, after a one second delay +* When an asynchronously sent message is acknowledged +[update_checkpoint](#updatecheckpoint) **MUST** be called to advance the +checkpoint to that specific message #### Example Kafka Output [kafka.lua](https://github.com/mozilla-services/lua_sandbox_extensions/blob/master/kafka/sandboxes/heka/output/kafka.lua) From ea7f1722bf6cdc7039c43e80ed33ca7f300ad618 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 7 Jul 2017 13:19:22 -0700 Subject: [PATCH 206/235] Fix the Windows compiler warnings --- CMakeLists.txt | 2 +- src/luasandbox.c | 45 +++++++++++++++++++++++++-------- src/test/test_generic_sandbox.c | 2 +- 3 files changed, 37 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ac827a9..1afd1d5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.5 LANGUAGES C) +project(luasandbox VERSION 1.2.6 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/luasandbox.c b/src/luasandbox.c index 456ab6f..85ebe4f 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -215,12 +215,12 @@ static int unprotected_panic(lua_State *lua) } -static int get_number(lua_State *lua, int idx, const char *item) +static size_t get_size(lua_State *lua, int idx, const char *item) { lua_getfield(lua, idx, item); - double d = lua_tonumber(lua, -1); + size_t size = (size_t)lua_tonumber(lua, -1); lua_pop(lua, 1); - return d; + return size; } @@ -271,6 +271,31 @@ static int check_unsigned(lua_State *L, int idx, const char *name, unsigned val) } +static int check_size(lua_State *L, int idx, const char *name, size_t val) +{ + lua_getfield(L, idx, name); + double d; + switch (lua_type(L, -1)) { + case LUA_TNUMBER: + d = lua_tonumber(L, -1); + if (d < 0 || d > SIZE_MAX) { + lua_pushfstring(L, "%s must be a size_t", name); + return 1; + } + break; + case LUA_TNIL: // add the default to the config + lua_pushnumber(L, (lua_Number)val); + lua_setglobal(L, name); + break; // use the default + default: + lua_pushfstring(L, "%s must be set to a number", name); + return 1; + } + lua_pop(L, 1); + return 0; +} + + static lua_State* load_sandbox_config(const char *cfg, lsb_logger *logger) { lua_State *L = luaL_newstate(); @@ -285,13 +310,13 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger *logger) int ret = luaL_dostring(L, cfg); if (ret) goto cleanup; - ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); + ret = check_size(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); if (ret) goto cleanup; - ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_MEMORY_LIMIT, 8 * 1024 * 1024); + ret = check_size(L, LUA_GLOBALSINDEX, LSB_MEMORY_LIMIT, 8 * 1024 * 1024); if (ret) goto cleanup; - ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_INSTRUCTION_LIMIT, 1000000); + ret = check_size(L, LUA_GLOBALSINDEX, LSB_INSTRUCTION_LIMIT, 1000000); if (ret) goto cleanup; ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_LOG_LEVEL, 3); @@ -473,10 +498,10 @@ lsb_lua_sandbox* lsb_create(void *parent, copy_table(lsb->lua, lua_cfg, &lsb->logger); lua_pop(lua_cfg, 2); lua_close(lua_cfg); - size_t ml = (size_t)get_number(lsb->lua, -1, LSB_MEMORY_LIMIT); - size_t il = (size_t)get_number(lsb->lua, -1, LSB_INSTRUCTION_LIMIT); - size_t ol = (size_t)get_number(lsb->lua, -1, LSB_OUTPUT_LIMIT); - double log_level = get_number(lsb->lua, -1, LSB_LOG_LEVEL); + size_t ml = get_size(lsb->lua, -1, LSB_MEMORY_LIMIT); + size_t il = get_size(lsb->lua, -1, LSB_INSTRUCTION_LIMIT); + size_t ol = get_size(lsb->lua, -1, LSB_OUTPUT_LIMIT); + size_t log_level = get_size(lsb->lua, -1, LSB_LOG_LEVEL); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); lua_pushlightuserdata(lsb->lua, lsb); lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c index a366854..30e7a15 100644 --- a/src/test/test_generic_sandbox.c +++ b/src/test/test_generic_sandbox.c @@ -210,7 +210,7 @@ static char* test_create_error() sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = -1", NULL); mu_assert(!sb, "lsb_create() invalid config"); - sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 5e9", NULL); + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 1.85e19", NULL); mu_assert(!sb, "lsb_create() invalid config"); sb = lsb_create(NULL, "lua/counter.lua", "instruction_limit = 'aaa'", NULL); From 2ab331562f83ebbf7e5b0da620661dfd27315c8f Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 6 Sep 2017 09:53:23 -0700 Subject: [PATCH 207/235] Issue 204 Fix the implicit fallthrough warnings --- src/lua/ldebug.c | 2 +- src/lua/llex.c | 1 + src/lua/ltable.c | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c index da76361..d9e8fcf 100644 --- a/src/lua/ldebug.c +++ b/src/lua/ldebug.c @@ -416,7 +416,7 @@ static Instruction symbexec (const Proto *pt, int lastpc, int reg) { case OP_FORLOOP: case OP_FORPREP: checkreg(pt, a+3); - /* go through */ + /* FALLTHRU */ case OP_JMP: { int dest = pc+1+b; /* not full check and jump is forward and do not skip `lastpc'? */ diff --git a/src/lua/llex.c b/src/lua/llex.c index 88c6790..2d2b019 100644 --- a/src/lua/llex.c +++ b/src/lua/llex.c @@ -367,6 +367,7 @@ static int llex (LexState *ls, SemInfo *seminfo) { } else if (sep == -1) return '['; else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING); + return 0; /* never reached but will silence the FALLTHRU compiler warning */ } case '=': { next(ls); diff --git a/src/lua/ltable.c b/src/lua/ltable.c index 18e7294..8859231 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -478,6 +478,7 @@ const TValue *luaH_get (Table *t, const TValue *key) { return luaH_getnum(t, k); /* use specialized version */ /* else go through */ } + /* FALLTHRU */ default: { Node *n = mainposition(t, key); do { /* check whether `key' is somewhere in the chain */ From 97ec49d4b9b97934c66da64b0731c32f7d638598 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 19 Sep 2017 12:33:23 -0700 Subject: [PATCH 208/235] Use the default package naming --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1afd1d5..cb8aba6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,9 @@ set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data anal set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) -set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") +set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") +set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") +set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") include(GNUInstallDirs) @@ -36,7 +38,7 @@ configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR) write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake COMPATIBILITY SameMajorVersion) -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake DESTINATION ${INSTALL_CMAKE_DIR}) From d4546d4713df415b3ad6fc4008c49d44bbb5d27c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 30 Oct 2017 14:48:24 -0700 Subject: [PATCH 209/235] Issue 208 - Add a string match modifier for literals --- CMakeLists.txt | 2 +- docs/util/message_matcher.md | 5 + include/luasandbox/util/string_matcher.h | 15 +- src/util/heka_message_matcher.c | 36 +- src/util/heka_message_matcher_impl.h | 3 +- src/util/heka_message_matcher_parser.c | 520 +++++++++++++--------- src/util/heka_message_matcher_parser.leg | 19 +- src/util/string_matcher.c | 38 +- src/util/test/test_heka_message_matcher.c | 19 +- src/util/test/test_string_matcher.c | 37 ++ 10 files changed, 456 insertions(+), 238 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cb8aba6..65fb963 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.6 LANGUAGES C) +project(luasandbox VERSION 1.2.7 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/util/message_matcher.md b/docs/util/message_matcher.md index 9021435..d5bab31 100644 --- a/docs/util/message_matcher.md +++ b/docs/util/message_matcher.md @@ -12,6 +12,7 @@ consume (see [Heka Message Structure](/heka/message.md)) * Fields[MyBool] == TRUE * TRUE * Fields[created] =~ "^2015" +* Fields[string] =~ "foo.example.com"% -- literal pattern vs "foo%.example%.com" * Fields[widget] != NIL * Timestamp >= "2016-05-24T00:00:00Z" * Timestamp >= 1464048000000000000 @@ -77,6 +78,10 @@ comparison ## Lua Pattern Matching Expression * Patterns are quoted string values + * An 'escape' pattern modifier of `%` is allowed e.g. `"foo.bar"%` is + treated as a literal instead of a pattern and behaves like the 'plain' + option on string.find(). If there are no pattern match characters in the + string this modifier is set automatically. * See [Lua Patterns](http://www.lua.org/manual/5.1/manual.html#5.4.1) * Capture groups are ignored diff --git a/include/luasandbox/util/string_matcher.h b/include/luasandbox/util/string_matcher.h index 3f7dddd..22d9642 100644 --- a/include/luasandbox/util/string_matcher.h +++ b/include/luasandbox/util/string_matcher.h @@ -26,10 +26,23 @@ extern "C" { * @param p Lua match pattern * http://www.lua.org/manual/5.1/manual.html#pdf-string.match * - * @return bool True if the sting matches the pattern + * @return bool True if the string matches the pattern */ LSB_UTIL_EXPORT bool lsb_string_match(const char *s, size_t len, const char *p); + +/** + * Searches for a string literal within a string + * + * @param s String to search + * @param ls Length of the string + * @param p Literal match string + * @param lp Length of the match string + * + * @return bool True if the string contains the literal + */ +LSB_UTIL_EXPORT bool lsb_string_find(const char *s, size_t ls, const char *p, size_t lp); + #ifdef __cplusplus } #endif diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c index a28b266..3057c51 100644 --- a/src/util/heka_message_matcher.c +++ b/src/util/heka_message_matcher.c @@ -54,9 +54,17 @@ static bool string_test(match_node *mn, lsb_const_string *val) return cmp > 0; } case OP_RE: - return lsb_string_match(val->s, val->len, mn->value.s); + if (mn->value_mod == '%') { + return lsb_string_find(val->s, val->len, mn->value.s, mn->value_len); + } else { + return lsb_string_match(val->s, val->len, mn->value.s); + } case OP_NRE: - return !lsb_string_match(val->s, val->len, mn->value.s); + if (mn->value_mod == '%') { + return !lsb_string_find(val->s, val->len, mn->value.s, mn->value_len); + } else { + return !lsb_string_match(val->s, val->len, mn->value.s); + } default: break; } @@ -119,25 +127,31 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) lsb_const_string variable = { .s = mn->variable, .len = mn->variable_len }; + if (!lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val)) { + if (mn->value_type == TYPE_NIL) { + return mn->op == OP_EQ; + } + return false; + } + switch (mn->value_type) { case TYPE_STRING: - if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) - && val.type == LSB_READ_STRING) { + if (val.type == LSB_READ_STRING) { return string_test(mn, &val.u.s); } break; case TYPE_NUMERIC: + if (val.type == LSB_READ_NUMERIC) { + return numeric_test(mn, val.u.d); + } + break; case TYPE_BOOLEAN: - if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val) - && (val.type == LSB_READ_NUMERIC || val.type == LSB_READ_BOOL)) { + if (val.type == LSB_READ_BOOL || val.type == LSB_READ_NUMERIC) { return numeric_test(mn, val.u.d); } break; case TYPE_NIL: - if (lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val)) { - return mn->op == OP_NE; - } - return mn->op == OP_EQ; + return mn->op == OP_NE; } } break; @@ -176,7 +190,7 @@ void lsb_destroy_message_matcher(lsb_message_matcher *mm) { if (!mm) return; - for (int i = 0; i < mm->nodes[0].size; ++i) { + for (int i = 0; i < mm->size; ++i) { free(mm->nodes[i].variable); switch (mm->nodes[i].value_type) { case TYPE_STRING: diff --git a/src/util/heka_message_matcher_impl.h b/src/util/heka_message_matcher_impl.h index 57d9c9b..21eb36d 100644 --- a/src/util/heka_message_matcher_impl.h +++ b/src/util/heka_message_matcher_impl.h @@ -39,7 +39,7 @@ typedef enum { typedef struct match_node { unsigned char id; unsigned char op; - unsigned char size; + unsigned char value_mod; unsigned char value_type; unsigned char value_len; unsigned char variable_len; @@ -55,6 +55,7 @@ typedef struct match_node { struct lsb_message_matcher { + unsigned char size; match_node *nodes; }; diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index 0a2105e..ff5ba97 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -1,9 +1,10 @@ -/* A recursive-descent parser generated by peg 0.1.15 */ +/* A recursive-descent parser generated by peg 0.1.18 */ #include #include #include -#define YYRULECOUNT 54 +#define YYRULECOUNT 55 +#line 1 "heka_message_matcher_parser.leg" /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=2 et sw=2 tw=80: */ @@ -229,6 +230,15 @@ static void set_string_value(context *ctx, char *s) } +static void set_match_mod(context *ctx) +{ + if (ctx->mn.value_mod == '\0' + && strpbrk(ctx->mn.value.s, "^$*+?.[%-") == NULL) { // literal + ctx->mn.value_mod = '%'; + } +} + + static bool check_string_len(char *s) { int i, j; @@ -255,48 +265,48 @@ static int cond_cnt(context *ctx) #ifndef YY_MALLOC -#define YY_MALLOC(C, N) malloc(N) +#define YY_MALLOC(C, N) malloc(N) #endif #ifndef YY_REALLOC -#define YY_REALLOC(C, P, N) realloc(P, N) +#define YY_REALLOC(C, P, N) realloc(P, N) #endif #ifndef YY_FREE -#define YY_FREE(C, P) free(P) +#define YY_FREE(C, P) free(P) #endif #ifndef YY_LOCAL -#define YY_LOCAL(T) static T +#define YY_LOCAL(T) static T #endif #ifndef YY_ACTION -#define YY_ACTION(T) static T +#define YY_ACTION(T) static T #endif #ifndef YY_RULE -#define YY_RULE(T) static T +#define YY_RULE(T) static T #endif #ifndef YY_PARSE -#define YY_PARSE(T) T +#define YY_PARSE(T) T #endif #ifndef YYPARSE -#define YYPARSE yyparse +#define YYPARSE yyparse #endif #ifndef YYPARSEFROM -#define YYPARSEFROM yyparsefrom +#define YYPARSEFROM yyparsefrom #endif #ifndef YYRELEASE -#define YYRELEASE yyrelease +#define YYRELEASE yyrelease #endif #ifndef YY_BEGIN -#define YY_BEGIN ( yy->__begin= yy->__pos, 1) +#define YY_BEGIN ( yy->__begin= yy->__pos, 1) #endif #ifndef YY_END -#define YY_END ( yy->__end= yy->__pos, 1) +#define YY_END ( yy->__end= yy->__pos, 1) #endif #ifdef YY_DEBUG -# define yyprintf(args) fprintf args +# define yyprintf(args) fprintf args #else # define yyprintf(args) #endif #ifndef YYSTYPE -#define YYSTYPE int +#define YYSTYPE int #endif #ifndef YY_STACK_SIZE #define YY_STACK_SIZE 128 @@ -335,16 +345,16 @@ struct _yycontext { }; #ifdef YY_CTX_LOCAL -#define YY_CTX_PARAM_ yycontext *yyctx, -#define YY_CTX_PARAM yycontext *yyctx -#define YY_CTX_ARG_ yyctx, -#define YY_CTX_ARG yyctx +#define YY_CTX_PARAM_ yycontext *yyctx, +#define YY_CTX_PARAM yycontext *yyctx +#define YY_CTX_ARG_ yyctx, +#define YY_CTX_ARG yyctx #ifndef YY_INPUT -#define YY_INPUT(yy, buf, result, max_size) \ - { \ - int yyc= getchar(); \ - result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ - yyprintf((stderr, "<%c>", yyc)); \ +#define YY_INPUT(yy, buf, result, max_size) \ + { \ + int yyc= getchar(); \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ + yyprintf((stderr, "<%c>", yyc)); \ } #endif #else @@ -355,11 +365,11 @@ struct _yycontext { yycontext _yyctx= { 0, 0 }; yycontext *yyctx= &_yyctx; #ifndef YY_INPUT -#define YY_INPUT(buf, result, max_size) \ - { \ - int yyc= getchar(); \ - result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ - yyprintf((stderr, "<%c>", yyc)); \ +#define YY_INPUT(buf, result, max_size) \ + { \ + int yyc= getchar(); \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ + yyprintf((stderr, "<%c>", yyc)); \ } #endif #endif @@ -455,10 +465,10 @@ YY_LOCAL(int) yyText(yycontext *yy, int begin, int end) else { while (yy->__textlen < (yyleng + 1)) - { - yy->__textlen *= 2; - yy->__text= (char *)YY_REALLOC(yy, yy->__text, yy->__textlen); - } + { + yy->__textlen *= 2; + yy->__text= (char *)YY_REALLOC(yy, yy->__text, yy->__textlen); + } memcpy(yy->__text, yy->__buf + begin, yyleng); } yy->__text[yyleng]= '\0'; @@ -522,31 +532,32 @@ YY_LOCAL(void) yySet(yycontext *yy, char *text, int count) { yy->__val[count]= #endif /* YY_PART */ -#define YYACCEPT yyAccept(yy, yythunkpos0) - -YY_RULE(int) yy_second_frac(yycontext *yy); /* 54 */ -YY_RULE(int) yy_second(yycontext *yy); /* 53 */ -YY_RULE(int) yy_minute(yycontext *yy); /* 52 */ -YY_RULE(int) yy_hour(yycontext *yy); /* 51 */ -YY_RULE(int) yy_timeoffset(yycontext *yy); /* 50 */ -YY_RULE(int) yy_partialtime(yycontext *yy); /* 49 */ -YY_RULE(int) yy_fulltime(yycontext *yy); /* 48 */ -YY_RULE(int) yy_day(yycontext *yy); /* 47 */ -YY_RULE(int) yy_month(yycontext *yy); /* 46 */ -YY_RULE(int) yy_year(yycontext *yy); /* 45 */ -YY_RULE(int) yy_fulldate(yycontext *yy); /* 44 */ -YY_RULE(int) yy_rfc3339(yycontext *yy); /* 43 */ -YY_RULE(int) yy_ts_quoted(yycontext *yy); /* 42 */ -YY_RULE(int) yy_zero_to_255(yycontext *yy); /* 41 */ -YY_RULE(int) yy_index(yycontext *yy); /* 40 */ -YY_RULE(int) yy_nil(yycontext *yy); /* 39 */ -YY_RULE(int) yy_fields(yycontext *yy); /* 38 */ -YY_RULE(int) yy_exponent(yycontext *yy); /* 37 */ -YY_RULE(int) yy_decimal(yycontext *yy); /* 36 */ -YY_RULE(int) yy_number(yycontext *yy); /* 35 */ -YY_RULE(int) yy_sign(yycontext *yy); /* 34 */ -YY_RULE(int) yy_numeric_value(yycontext *yy); /* 33 */ -YY_RULE(int) yy_numeric_headers(yycontext *yy); /* 32 */ +#define YYACCEPT yyAccept(yy, yythunkpos0) + +YY_RULE(int) yy_second_frac(yycontext *yy); /* 55 */ +YY_RULE(int) yy_second(yycontext *yy); /* 54 */ +YY_RULE(int) yy_minute(yycontext *yy); /* 53 */ +YY_RULE(int) yy_hour(yycontext *yy); /* 52 */ +YY_RULE(int) yy_timeoffset(yycontext *yy); /* 51 */ +YY_RULE(int) yy_partialtime(yycontext *yy); /* 50 */ +YY_RULE(int) yy_fulltime(yycontext *yy); /* 49 */ +YY_RULE(int) yy_day(yycontext *yy); /* 48 */ +YY_RULE(int) yy_month(yycontext *yy); /* 47 */ +YY_RULE(int) yy_year(yycontext *yy); /* 46 */ +YY_RULE(int) yy_fulldate(yycontext *yy); /* 45 */ +YY_RULE(int) yy_rfc3339(yycontext *yy); /* 44 */ +YY_RULE(int) yy_ts_quoted(yycontext *yy); /* 43 */ +YY_RULE(int) yy_zero_to_255(yycontext *yy); /* 42 */ +YY_RULE(int) yy_index(yycontext *yy); /* 41 */ +YY_RULE(int) yy_nil(yycontext *yy); /* 40 */ +YY_RULE(int) yy_fields(yycontext *yy); /* 39 */ +YY_RULE(int) yy_exponent(yycontext *yy); /* 38 */ +YY_RULE(int) yy_decimal(yycontext *yy); /* 37 */ +YY_RULE(int) yy_number(yycontext *yy); /* 36 */ +YY_RULE(int) yy_sign(yycontext *yy); /* 35 */ +YY_RULE(int) yy_numeric_value(yycontext *yy); /* 34 */ +YY_RULE(int) yy_numeric_headers(yycontext *yy); /* 33 */ +YY_RULE(int) yy_string_match_mod(yycontext *yy); /* 32 */ YY_RULE(int) yy_string_match(yycontext *yy); /* 31 */ YY_RULE(int) yy_string_value(yycontext *yy); /* 30 */ YY_RULE(int) yy_string_headers(yycontext *yy); /* 29 */ @@ -586,6 +597,7 @@ YY_ACTION(void) yy_1_nil(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_nil\n")); { +#line 365 yy->ctx.mn.value_type = TYPE_NIL;; } #undef yythunkpos @@ -599,6 +611,7 @@ YY_ACTION(void) yy_1_second_frac(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second_frac\n")); { +#line 363 yy->ctx.mn.value.d += strtod(yytext, NULL); } #undef yythunkpos @@ -612,6 +625,7 @@ YY_ACTION(void) yy_1_second(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second\n")); { +#line 362 __ = atoi(yytext); } #undef yythunkpos @@ -625,6 +639,7 @@ YY_ACTION(void) yy_1_minute(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_minute\n")); { +#line 358 __ = atoi(yytext); } #undef yythunkpos @@ -638,6 +653,7 @@ YY_ACTION(void) yy_1_hour(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_hour\n")); { +#line 357 __ = atoi(yytext); } #undef yythunkpos @@ -653,6 +669,7 @@ YY_ACTION(void) yy_2_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_timeoffset\n")); { +#line 353 update_offset(&yy->ctx, yytext[0], h, m); } #undef yythunkpos @@ -670,6 +687,7 @@ YY_ACTION(void) yy_1_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_timeoffset\n")); { +#line 352 update_offset(&yy->ctx, '+', 0, 0); } #undef yythunkpos @@ -688,6 +706,7 @@ YY_ACTION(void) yy_1_partialtime(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_partialtime\n")); { +#line 351 update_time(&yy->ctx, h, m, s); } #undef yythunkpos @@ -704,6 +723,7 @@ YY_ACTION(void) yy_1_day(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_day\n")); { +#line 347 __ = atoi(yytext); } #undef yythunkpos @@ -717,6 +737,7 @@ YY_ACTION(void) yy_1_month(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_month\n")); { +#line 342 __ = atoi(yytext); } #undef yythunkpos @@ -730,6 +751,7 @@ YY_ACTION(void) yy_1_year(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_year\n")); { +#line 339 __ = atoi(yytext); } #undef yythunkpos @@ -746,6 +768,7 @@ YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fulldate\n")); { +#line 338 update_date(&yy->ctx, y, m, d); } #undef yythunkpos @@ -762,6 +785,7 @@ YY_ACTION(void) yy_1_ts_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_ts_test\n")); { +#line 336 set_timestamp(&yy->ctx); } #undef yythunkpos @@ -775,6 +799,7 @@ YY_ACTION(void) yy_1_index(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_index\n")); { +#line 330 __ = atoi(yytext); } #undef yythunkpos @@ -790,6 +815,7 @@ YY_ACTION(void) yy_3_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_fields\n")); { +#line 329 yy->ctx.mn.ai = a; } #undef yythunkpos @@ -807,6 +833,7 @@ YY_ACTION(void) yy_2_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_fields\n")); { +#line 329 yy->ctx.mn.fi = f; } #undef yythunkpos @@ -824,6 +851,7 @@ YY_ACTION(void) yy_1_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fields\n")); { +#line 329 set_field(&yy->ctx, yytext); } #undef yythunkpos @@ -839,6 +867,7 @@ YY_ACTION(void) yy_1_numeric_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_numeric_value\n")); { +#line 319 set_numeric_value(&yy->ctx, yytext); } #undef yythunkpos @@ -852,6 +881,7 @@ YY_ACTION(void) yy_2_numeric_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_numeric_headers\n")); { +#line 316 yy->ctx.mn.id = LSB_PB_PID; } #undef yythunkpos @@ -865,12 +895,27 @@ YY_ACTION(void) yy_1_numeric_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_numeric_headers\n")); { +#line 315 yy->ctx.mn.id = LSB_PB_SEVERITY; } #undef yythunkpos #undef yypos #undef yy } +YY_ACTION(void) yy_1_string_match_mod(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_match_mod\n")); + { +#line 313 + yy->ctx.mn.value_mod = '%'; + } +#undef yythunkpos +#undef yypos +#undef yy +} YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ @@ -878,6 +923,7 @@ YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_value\n")); { +#line 309 set_string_value(&yy->ctx, yytext); } #undef yythunkpos @@ -891,6 +937,7 @@ YY_ACTION(void) yy_6_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_6_string_headers\n")); { +#line 305 yy->ctx.mn.id = LSB_PB_UUID; } #undef yythunkpos @@ -904,6 +951,7 @@ YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_5_string_headers\n")); { +#line 304 yy->ctx.mn.id = LSB_PB_PAYLOAD; } #undef yythunkpos @@ -917,6 +965,7 @@ YY_ACTION(void) yy_4_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_4_string_headers\n")); { +#line 303 yy->ctx.mn.id = LSB_PB_ENV_VERSION; } #undef yythunkpos @@ -930,6 +979,7 @@ YY_ACTION(void) yy_3_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_string_headers\n")); { +#line 302 yy->ctx.mn.id = LSB_PB_HOSTNAME; } #undef yythunkpos @@ -943,6 +993,7 @@ YY_ACTION(void) yy_2_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_string_headers\n")); { +#line 301 yy->ctx.mn.id = LSB_PB_LOGGER; } #undef yythunkpos @@ -956,12 +1007,27 @@ YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_headers\n")); { +#line 300 yy->ctx.mn.id = LSB_PB_TYPE; } #undef yythunkpos #undef yypos #undef yy } +YY_ACTION(void) yy_1_string_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_test\n")); + { +#line 298 + set_match_mod(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ @@ -969,6 +1035,7 @@ YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_close\n")); { +#line 296 pop_to_paren(&yy->ctx); } #undef yythunkpos @@ -982,6 +1049,7 @@ YY_ACTION(void) yy_1_open(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_open\n")); { +#line 295 push_op(&yy->ctx, OP_OPEN); } #undef yythunkpos @@ -995,6 +1063,7 @@ YY_ACTION(void) yy_1_or(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_or\n")); { +#line 294 push_op(&yy->ctx, OP_OR); } #undef yythunkpos @@ -1008,6 +1077,7 @@ YY_ACTION(void) yy_1_and(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_and\n")); { +#line 293 push_op(&yy->ctx, OP_AND); } #undef yythunkpos @@ -1021,6 +1091,7 @@ YY_ACTION(void) yy_2_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean\n")); { +#line 291 set_boolean_value(&yy->ctx, 0); } #undef yythunkpos @@ -1034,6 +1105,7 @@ YY_ACTION(void) yy_1_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean\n")); { +#line 290 set_boolean_value(&yy->ctx, 1); } #undef yythunkpos @@ -1047,6 +1119,7 @@ YY_ACTION(void) yy_2_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean_test\n")); { +#line 289 yy->ctx.mn.op = OP_FALSE; } #undef yythunkpos @@ -1060,6 +1133,7 @@ YY_ACTION(void) yy_1_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean_test\n")); { +#line 288 yy->ctx.mn.op = OP_TRUE; } #undef yythunkpos @@ -1073,6 +1147,7 @@ YY_ACTION(void) yy_1_op_lt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lt\n")); { +#line 279 yy->ctx.mn.op = OP_LT; } #undef yythunkpos @@ -1086,6 +1161,7 @@ YY_ACTION(void) yy_1_op_lte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lte\n")); { +#line 278 yy->ctx.mn.op = OP_LTE; } #undef yythunkpos @@ -1099,6 +1175,7 @@ YY_ACTION(void) yy_1_op_gt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gt\n")); { +#line 277 yy->ctx.mn.op = OP_GT; } #undef yythunkpos @@ -1112,6 +1189,7 @@ YY_ACTION(void) yy_1_op_gte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gte\n")); { +#line 276 yy->ctx.mn.op = OP_GTE; } #undef yythunkpos @@ -1125,6 +1203,7 @@ YY_ACTION(void) yy_1_op_sne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_sne\n")); { +#line 275 yy->ctx.mn.op = OP_NRE; } #undef yythunkpos @@ -1138,6 +1217,7 @@ YY_ACTION(void) yy_1_op_seq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_seq\n")); { +#line 274 yy->ctx.mn.op = OP_RE; } #undef yythunkpos @@ -1151,6 +1231,7 @@ YY_ACTION(void) yy_1_op_ne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_ne\n")); { +#line 273 yy->ctx.mn.op = OP_NE; } #undef yythunkpos @@ -1164,6 +1245,7 @@ YY_ACTION(void) yy_1_op_eq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_eq\n")); { +#line 272 yy->ctx.mn.op = OP_EQ; } #undef yythunkpos @@ -1177,6 +1259,7 @@ YY_ACTION(void) yy_1_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_test\n")); { +#line 270 push_output(&yy->ctx, &yy->ctx.mn); } #undef yythunkpos @@ -1190,6 +1273,7 @@ YY_ACTION(void) yy_1_match(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_match\n")); { +#line 261 pop_all_ops(&yy->ctx); } #undef yythunkpos @@ -1679,426 +1763,439 @@ YY_RULE(int) yy_numeric_headers(yycontext *yy) yyprintf((stderr, " fail %s @ %s\n", "numeric_headers", yy->__buf+yy->__pos)); return 0; } +YY_RULE(int) yy_string_match_mod(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_match_mod")); if (!yymatchChar(yy, '%')) goto l67; yyDo(yy, yy_1_string_match_mod, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); + return 1; + l67:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); + return 0; +} YY_RULE(int) yy_string_match(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_match")); - { int yypos68= yy->__pos, yythunkpos68= yy->__thunkpos; if (!yy_op_seq(yy)) goto l69; goto l68; - l69:; yy->__pos= yypos68; yy->__thunkpos= yythunkpos68; if (!yy_op_sne(yy)) goto l67; + { int yypos69= yy->__pos, yythunkpos69= yy->__thunkpos; if (!yy_op_seq(yy)) goto l70; goto l69; + l70:; yy->__pos= yypos69; yy->__thunkpos= yythunkpos69; if (!yy_op_sne(yy)) goto l68; } - l68:; + l69:; yyprintf((stderr, " ok %s @ %s\n", "string_match", yy->__buf+yy->__pos)); return 1; - l67:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l68:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_match", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_value(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_value")); - { int yypos71= yy->__pos, yythunkpos71= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l72; yyText(yy, yy->__begin, yy->__end); { + { int yypos72= yy->__pos, yythunkpos72= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l73; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l72; +if (!(YY_BEGIN)) goto l73; #undef yytext #undef yyleng } - l73:; - { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; - { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l76; goto l75; - l76:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; - { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l77; goto l74; - l77:; yy->__pos= yypos77; yy->__thunkpos= yythunkpos77; - } if (!yymatchDot(yy)) goto l74; + l74:; + { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; + { int yypos76= yy->__pos, yythunkpos76= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l77; goto l76; + l77:; yy->__pos= yypos76; yy->__thunkpos= yythunkpos76; + { int yypos78= yy->__pos, yythunkpos78= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l78; goto l75; + l78:; yy->__pos= yypos78; yy->__thunkpos= yythunkpos78; + } if (!yymatchDot(yy)) goto l75; } - l75:; goto l73; - l74:; yy->__pos= yypos74; yy->__thunkpos= yythunkpos74; + l76:; goto l74; + l75:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l72; +if (!(YY_END)) goto l73; #undef yytext #undef yyleng - } if (!yymatchChar(yy, '"')) goto l72; goto l71; - l72:; yy->__pos= yypos71; yy->__thunkpos= yythunkpos71; if (!yymatchChar(yy, '\'')) goto l70; yyText(yy, yy->__begin, yy->__end); { + } if (!yymatchChar(yy, '"')) goto l73; goto l72; + l73:; yy->__pos= yypos72; yy->__thunkpos= yythunkpos72; if (!yymatchChar(yy, '\'')) goto l71; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l70; +if (!(YY_BEGIN)) goto l71; #undef yytext #undef yyleng } - l78:; - { int yypos79= yy->__pos, yythunkpos79= yy->__thunkpos; - { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l81; goto l80; - l81:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; - { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l82; goto l79; - l82:; yy->__pos= yypos82; yy->__thunkpos= yythunkpos82; - } if (!yymatchDot(yy)) goto l79; + l79:; + { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; + { int yypos81= yy->__pos, yythunkpos81= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l82; goto l81; + l82:; yy->__pos= yypos81; yy->__thunkpos= yythunkpos81; + { int yypos83= yy->__pos, yythunkpos83= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l83; goto l80; + l83:; yy->__pos= yypos83; yy->__thunkpos= yythunkpos83; + } if (!yymatchDot(yy)) goto l80; } - l80:; goto l78; - l79:; yy->__pos= yypos79; yy->__thunkpos= yythunkpos79; + l81:; goto l79; + l80:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l70; +if (!(YY_END)) goto l71; #undef yytext #undef yyleng - } if (!yymatchChar(yy, '\'')) goto l70; + } if (!yymatchChar(yy, '\'')) goto l71; } - l71:; yyText(yy, yy->__begin, yy->__end); { + l72:; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(check_string_len(yytext))) goto l70; +if (!(check_string_len(yytext))) goto l71; #undef yytext #undef yyleng } yyDo(yy, yy_1_string_value, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "string_value", yy->__buf+yy->__pos)); return 1; - l70:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l71:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_value", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_headers(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_headers")); - { int yypos84= yy->__pos, yythunkpos84= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l85; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l85; goto l84; - l85:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Logger")) goto l86; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l86; goto l84; - l86:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Hostname")) goto l87; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l87; goto l84; - l87:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "EnvVersion")) goto l88; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l88; goto l84; - l88:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Payload")) goto l89; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l89; goto l84; - l89:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Uuid")) goto l83; yyDo(yy, yy_6_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l83; - } - l84:; + { int yypos85= yy->__pos, yythunkpos85= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l86; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l86; goto l85; + l86:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Logger")) goto l87; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l87; goto l85; + l87:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Hostname")) goto l88; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l88; goto l85; + l88:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "EnvVersion")) goto l89; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l89; goto l85; + l89:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Payload")) goto l90; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l90; goto l85; + l90:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Uuid")) goto l84; yyDo(yy, yy_6_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l84; + } + l85:; yyprintf((stderr, " ok %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); return 1; - l83:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l84:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_boolean(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "boolean")); - { int yypos91= yy->__pos, yythunkpos91= yy->__thunkpos; if (!yy_true(yy)) goto l92; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l91; - l92:; yy->__pos= yypos91; yy->__thunkpos= yythunkpos91; if (!yy_false(yy)) goto l90; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); + { int yypos92= yy->__pos, yythunkpos92= yy->__thunkpos; if (!yy_true(yy)) goto l93; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l92; + l93:; yy->__pos= yypos92; yy->__thunkpos= yythunkpos92; if (!yy_false(yy)) goto l91; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); } - l91:; + l92:; yyprintf((stderr, " ok %s @ %s\n", "boolean", yy->__buf+yy->__pos)); return 1; - l90:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l91:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "boolean", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_false(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l93; if (!yy_sp(yy)) goto l93; + yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l94; if (!yy_sp(yy)) goto l94; yyprintf((stderr, " ok %s @ %s\n", "false", yy->__buf+yy->__pos)); return 1; - l93:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l94:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "false", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_true(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l94; if (!yy_sp(yy)) goto l94; + yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l95; if (!yy_sp(yy)) goto l95; yyprintf((stderr, " ok %s @ %s\n", "true", yy->__buf+yy->__pos)); return 1; - l94:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l95:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "true", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_relational(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "relational")); - { int yypos96= yy->__pos, yythunkpos96= yy->__thunkpos; if (!yy_op_eq(yy)) goto l97; goto l96; - l97:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_ne(yy)) goto l98; goto l96; - l98:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_gte(yy)) goto l99; goto l96; - l99:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_gt(yy)) goto l100; goto l96; - l100:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_lte(yy)) goto l101; goto l96; - l101:; yy->__pos= yypos96; yy->__thunkpos= yythunkpos96; if (!yy_op_lt(yy)) goto l95; - } - l96:; + { int yypos97= yy->__pos, yythunkpos97= yy->__thunkpos; if (!yy_op_eq(yy)) goto l98; goto l97; + l98:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_ne(yy)) goto l99; goto l97; + l99:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_gte(yy)) goto l100; goto l97; + l100:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_gt(yy)) goto l101; goto l97; + l101:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_lte(yy)) goto l102; goto l97; + l102:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_lt(yy)) goto l96; + } + l97:; yyprintf((stderr, " ok %s @ %s\n", "relational", yy->__buf+yy->__pos)); return 1; - l95:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l96:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "relational", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_lt(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l102; if (!yy_sp(yy)) goto l102; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l103; if (!yy_sp(yy)) goto l103; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); return 1; - l102:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l103:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_lte(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l103; if (!yy_sp(yy)) goto l103; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l104; if (!yy_sp(yy)) goto l104; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); return 1; - l103:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l104:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_gt(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l104; if (!yy_sp(yy)) goto l104; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); return 1; - l104:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_gte(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); return 1; - l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_sne(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); return 1; - l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_seq(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); return 1; - l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_ne(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l109; if (!yy_sp(yy)) goto l109; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); return 1; - l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_eq(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l109; if (!yy_sp(yy)) goto l109; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l110; if (!yy_sp(yy)) goto l110; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); return 1; - l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l110:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_boolean_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "boolean_test")); - { int yypos111= yy->__pos, yythunkpos111= yy->__thunkpos; if (!yy_true(yy)) goto l112; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l111; - l112:; yy->__pos= yypos111; yy->__thunkpos= yythunkpos111; if (!yy_false(yy)) goto l110; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); + { int yypos112= yy->__pos, yythunkpos112= yy->__thunkpos; if (!yy_true(yy)) goto l113; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l112; + l113:; yy->__pos= yypos112; yy->__thunkpos= yythunkpos112; if (!yy_false(yy)) goto l111; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); } - l111:; + l112:; yyprintf((stderr, " ok %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); return 1; - l110:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l111:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_ts_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "ts_test")); if (!yymatchString(yy, "Timestamp")) goto l113; if (!yy_sp(yy)) goto l113; if (!yy_relational(yy)) goto l113; if (!yy_sp(yy)) goto l113; - { int yypos114= yy->__pos, yythunkpos114= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l115; goto l114; - l115:; yy->__pos= yypos114; yy->__thunkpos= yythunkpos114; if (!yy_ts_quoted(yy)) goto l113; + yyprintf((stderr, "%s\n", "ts_test")); if (!yymatchString(yy, "Timestamp")) goto l114; if (!yy_sp(yy)) goto l114; if (!yy_relational(yy)) goto l114; if (!yy_sp(yy)) goto l114; + { int yypos115= yy->__pos, yythunkpos115= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l116; goto l115; + l116:; yy->__pos= yypos115; yy->__thunkpos= yythunkpos115; if (!yy_ts_quoted(yy)) goto l114; } - l114:; yyDo(yy, yy_1_ts_test, yy->__begin, yy->__end); + l115:; yyDo(yy, yy_1_ts_test, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); return 1; - l113:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l114:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_field_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l116; if (!yy_sp(yy)) goto l116; - { int yypos117= yy->__pos, yythunkpos117= yy->__thunkpos; if (!yy_relational(yy)) goto l118; if (!yy_sp(yy)) goto l118; - { int yypos119= yy->__pos, yythunkpos119= yy->__thunkpos; if (!yy_string_value(yy)) goto l120; goto l119; - l120:; yy->__pos= yypos119; yy->__thunkpos= yythunkpos119; if (!yy_numeric_value(yy)) goto l118; + yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l117; if (!yy_sp(yy)) goto l117; + { int yypos118= yy->__pos, yythunkpos118= yy->__thunkpos; if (!yy_relational(yy)) goto l119; if (!yy_sp(yy)) goto l119; + { int yypos120= yy->__pos, yythunkpos120= yy->__thunkpos; if (!yy_string_value(yy)) goto l121; goto l120; + l121:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; if (!yy_numeric_value(yy)) goto l119; } - l119:; goto l117; - l118:; yy->__pos= yypos117; yy->__thunkpos= yythunkpos117; if (!yy_string_match(yy)) goto l121; if (!yy_sp(yy)) goto l121; if (!yy_string_value(yy)) goto l121; goto l117; - l121:; yy->__pos= yypos117; yy->__thunkpos= yythunkpos117; - { int yypos122= yy->__pos, yythunkpos122= yy->__thunkpos; if (!yy_op_eq(yy)) goto l123; goto l122; - l123:; yy->__pos= yypos122; yy->__thunkpos= yythunkpos122; if (!yy_op_ne(yy)) goto l116; + l120:; goto l118; + l119:; yy->__pos= yypos118; yy->__thunkpos= yythunkpos118; if (!yy_string_match(yy)) goto l122; if (!yy_sp(yy)) goto l122; if (!yy_string_value(yy)) goto l122; goto l118; + l122:; yy->__pos= yypos118; yy->__thunkpos= yythunkpos118; + { int yypos123= yy->__pos, yythunkpos123= yy->__thunkpos; if (!yy_op_eq(yy)) goto l124; goto l123; + l124:; yy->__pos= yypos123; yy->__thunkpos= yythunkpos123; if (!yy_op_ne(yy)) goto l117; } - l122:; if (!yy_sp(yy)) goto l116; - { int yypos124= yy->__pos, yythunkpos124= yy->__thunkpos; if (!yy_boolean(yy)) goto l125; goto l124; - l125:; yy->__pos= yypos124; yy->__thunkpos= yythunkpos124; if (!yy_nil(yy)) goto l116; + l123:; if (!yy_sp(yy)) goto l117; + { int yypos125= yy->__pos, yythunkpos125= yy->__thunkpos; if (!yy_boolean(yy)) goto l126; goto l125; + l126:; yy->__pos= yypos125; yy->__thunkpos= yythunkpos125; if (!yy_nil(yy)) goto l117; } - l124:; + l125:; } - l117:; + l118:; yyprintf((stderr, " ok %s @ %s\n", "field_test", yy->__buf+yy->__pos)); return 1; - l116:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l117:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "field_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_numeric_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "numeric_test")); if (!yy_numeric_headers(yy)) goto l126; if (!yy_sp(yy)) goto l126; if (!yy_relational(yy)) goto l126; if (!yy_sp(yy)) goto l126; if (!yy_numeric_value(yy)) goto l126; + yyprintf((stderr, "%s\n", "numeric_test")); if (!yy_numeric_headers(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_relational(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_numeric_value(yy)) goto l127; yyprintf((stderr, " ok %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); return 1; - l126:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l127:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "string_test")); if (!yy_string_headers(yy)) goto l127; if (!yy_sp(yy)) goto l127; - { int yypos128= yy->__pos, yythunkpos128= yy->__thunkpos; if (!yy_relational(yy)) goto l129; if (!yy_sp(yy)) goto l129; if (!yy_string_value(yy)) goto l129; goto l128; - l129:; yy->__pos= yypos128; yy->__thunkpos= yythunkpos128; if (!yy_string_match(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_string_value(yy)) goto l127; + yyprintf((stderr, "%s\n", "string_test")); if (!yy_string_headers(yy)) goto l128; if (!yy_sp(yy)) goto l128; + { int yypos129= yy->__pos, yythunkpos129= yy->__thunkpos; if (!yy_relational(yy)) goto l130; if (!yy_sp(yy)) goto l130; if (!yy_string_value(yy)) goto l130; goto l129; + l130:; yy->__pos= yypos129; yy->__thunkpos= yythunkpos129; if (!yy_string_match(yy)) goto l128; if (!yy_sp(yy)) goto l128; if (!yy_string_value(yy)) goto l128; + { int yypos131= yy->__pos, yythunkpos131= yy->__thunkpos; if (!yy_string_match_mod(yy)) goto l131; goto l132; + l131:; yy->__pos= yypos131; yy->__thunkpos= yythunkpos131; } - l128:; + l132:; yyDo(yy, yy_1_string_test, yy->__begin, yy->__end); + } + l129:; yyprintf((stderr, " ok %s @ %s\n", "string_test", yy->__buf+yy->__pos)); return 1; - l127:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l128:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_close(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "close")); if (!yymatchChar(yy, ')')) goto l130; yyDo(yy, yy_1_close, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l130; + yyprintf((stderr, "%s\n", "close")); if (!yymatchChar(yy, ')')) goto l133; yyDo(yy, yy_1_close, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l133; yyprintf((stderr, " ok %s @ %s\n", "close", yy->__buf+yy->__pos)); return 1; - l130:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l133:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "close", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_open(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "open")); if (!yymatchChar(yy, '(')) goto l131; yyDo(yy, yy_1_open, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l131; + yyprintf((stderr, "%s\n", "open")); if (!yymatchChar(yy, '(')) goto l134; yyDo(yy, yy_1_open, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l134; yyprintf((stderr, " ok %s @ %s\n", "open", yy->__buf+yy->__pos)); return 1; - l131:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l134:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "open", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "test")); - { int yypos133= yy->__pos, yythunkpos133= yy->__thunkpos; if (!yy_string_test(yy)) goto l134; goto l133; - l134:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_numeric_test(yy)) goto l135; goto l133; - l135:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_field_test(yy)) goto l136; goto l133; - l136:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_ts_test(yy)) goto l137; goto l133; - l137:; yy->__pos= yypos133; yy->__thunkpos= yythunkpos133; if (!yy_boolean_test(yy)) goto l132; + { int yypos136= yy->__pos, yythunkpos136= yy->__thunkpos; if (!yy_string_test(yy)) goto l137; goto l136; + l137:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_numeric_test(yy)) goto l138; goto l136; + l138:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_field_test(yy)) goto l139; goto l136; + l139:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_ts_test(yy)) goto l140; goto l136; + l140:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_boolean_test(yy)) goto l135; } - l133:; yyDo(yy, yy_1_test, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l132; + l136:; yyDo(yy, yy_1_test, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l135; yyprintf((stderr, " ok %s @ %s\n", "test", yy->__buf+yy->__pos)); return 1; - l132:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l135:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_and(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "and")); if (!yymatchString(yy, "&&")) goto l138; yyText(yy, yy->__begin, yy->__end); { + yyprintf((stderr, "%s\n", "and")); if (!yymatchString(yy, "&&")) goto l141; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(cond_cnt(&yy->ctx))) goto l138; +if (!(cond_cnt(&yy->ctx))) goto l141; #undef yytext #undef yyleng - } yyDo(yy, yy_1_and, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l138; + } yyDo(yy, yy_1_and, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l141; yyprintf((stderr, " ok %s @ %s\n", "and", yy->__buf+yy->__pos)); return 1; - l138:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l141:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "and", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_expr(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "expr")); - { int yypos140= yy->__pos, yythunkpos140= yy->__thunkpos; if (!yy_test(yy)) goto l141; goto l140; - l141:; yy->__pos= yypos140; yy->__thunkpos= yythunkpos140; if (!yy_open(yy)) goto l139; if (!yy_ored(yy)) goto l139; if (!yy_close(yy)) goto l139; + { int yypos143= yy->__pos, yythunkpos143= yy->__thunkpos; if (!yy_test(yy)) goto l144; goto l143; + l144:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_open(yy)) goto l142; if (!yy_ored(yy)) goto l142; if (!yy_close(yy)) goto l142; } - l140:; + l143:; yyprintf((stderr, " ok %s @ %s\n", "expr", yy->__buf+yy->__pos)); return 1; - l139:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l142:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "expr", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_or(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "or")); if (!yymatchString(yy, "||")) goto l142; yyText(yy, yy->__begin, yy->__end); { + yyprintf((stderr, "%s\n", "or")); if (!yymatchString(yy, "||")) goto l145; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(cond_cnt(&yy->ctx))) goto l142; +if (!(cond_cnt(&yy->ctx))) goto l145; #undef yytext #undef yyleng - } yyDo(yy, yy_1_or, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l142; + } yyDo(yy, yy_1_or, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l145; yyprintf((stderr, " ok %s @ %s\n", "or", yy->__buf+yy->__pos)); return 1; - l142:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l145:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "or", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_anded(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "anded")); if (!yy_expr(yy)) goto l143; - l144:; - { int yypos145= yy->__pos, yythunkpos145= yy->__thunkpos; if (!yy_and(yy)) goto l145; if (!yy_expr(yy)) goto l145; goto l144; - l145:; yy->__pos= yypos145; yy->__thunkpos= yythunkpos145; - } if (!yy_sp(yy)) goto l143; + yyprintf((stderr, "%s\n", "anded")); if (!yy_expr(yy)) goto l146; + l147:; + { int yypos148= yy->__pos, yythunkpos148= yy->__thunkpos; if (!yy_and(yy)) goto l148; if (!yy_expr(yy)) goto l148; goto l147; + l148:; yy->__pos= yypos148; yy->__thunkpos= yythunkpos148; + } if (!yy_sp(yy)) goto l146; yyprintf((stderr, " ok %s @ %s\n", "anded", yy->__buf+yy->__pos)); return 1; - l143:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l146:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "anded", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_eol(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "eol")); - { int yypos147= yy->__pos, yythunkpos147= yy->__thunkpos; if (!yymatchDot(yy)) goto l147; goto l146; - l147:; yy->__pos= yypos147; yy->__thunkpos= yythunkpos147; + { int yypos150= yy->__pos, yythunkpos150= yy->__thunkpos; if (!yymatchDot(yy)) goto l150; goto l149; + l150:; yy->__pos= yypos150; yy->__thunkpos= yythunkpos150; } yyprintf((stderr, " ok %s @ %s\n", "eol", yy->__buf+yy->__pos)); return 1; - l146:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l149:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "eol", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_ored(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "ored")); if (!yy_anded(yy)) goto l148; - l149:; - { int yypos150= yy->__pos, yythunkpos150= yy->__thunkpos; if (!yy_or(yy)) goto l150; if (!yy_anded(yy)) goto l150; goto l149; - l150:; yy->__pos= yypos150; yy->__thunkpos= yythunkpos150; - } if (!yy_sp(yy)) goto l148; + yyprintf((stderr, "%s\n", "ored")); if (!yy_anded(yy)) goto l151; + l152:; + { int yypos153= yy->__pos, yythunkpos153= yy->__thunkpos; if (!yy_or(yy)) goto l153; if (!yy_anded(yy)) goto l153; goto l152; + l153:; yy->__pos= yypos153; yy->__thunkpos= yythunkpos153; + } if (!yy_sp(yy)) goto l151; yyprintf((stderr, " ok %s @ %s\n", "ored", yy->__buf+yy->__pos)); return 1; - l148:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l151:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "ored", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_sp(yycontext *yy) { yyprintf((stderr, "%s\n", "sp")); - l152:; - { int yypos153= yy->__pos, yythunkpos153= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\002\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l153; goto l152; - l153:; yy->__pos= yypos153; yy->__thunkpos= yythunkpos153; + l155:; + { int yypos156= yy->__pos, yythunkpos156= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\002\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l156; goto l155; + l156:; yy->__pos= yypos156; yy->__thunkpos= yythunkpos156; } yyprintf((stderr, " ok %s @ %s\n", "sp", yy->__buf+yy->__pos)); return 1; } YY_RULE(int) yy_match(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "match")); if (!yy_sp(yy)) goto l154; if (!yy_ored(yy)) goto l154; if (!yy_eol(yy)) goto l154; yyDo(yy, yy_1_match, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "match")); if (!yy_sp(yy)) goto l157; if (!yy_ored(yy)) goto l157; if (!yy_eol(yy)) goto l157; yyDo(yy, yy_1_match, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "match", yy->__buf+yy->__pos)); return 1; - l154:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l157:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "match", yy->__buf+yy->__pos)); return 0; } @@ -2151,6 +2248,7 @@ YY_PARSE(yycontext *) YYRELEASE(yycontext *yyctx) } #endif +#line 371 "heka_message_matcher_parser.leg" @@ -2185,10 +2283,10 @@ lsb_message_matcher* lsb_create_message_matcher(const char *exp) goto cleanup; } mm->nodes = yy.ctx.out.a; - mm->nodes[0].size = yy.ctx.out.pos; + mm->size = yy.ctx.out.pos; // turn the postfix stack into an executable tree - match_node **stack = calloc(sizeof(match_node *) * mm->nodes[0].size, 1); + match_node **stack = calloc(sizeof(match_node *) * mm->size, 1); if (!stack) { free(mm); mm = NULL; @@ -2196,7 +2294,7 @@ lsb_message_matcher* lsb_create_message_matcher(const char *exp) } int top = 0; - for (int i = mm->nodes[0].size - 1; i >= 0; --i) { + for (int i = mm->size - 1; i >= 0; --i) { if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { stack[top++] = &mm->nodes[i]; } else { diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index cab78b8..088c5fd 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -223,6 +223,15 @@ static void set_string_value(context *ctx, char *s) } +static void set_match_mod(context *ctx) +{ + if (ctx->mn.value_mod == '\0' + && strpbrk(ctx->mn.value.s, "^$*+?.[%-") == NULL) { // literal + ctx->mn.value_mod = '%'; + } +} + + static bool check_string_len(char *s) { int i, j; @@ -286,7 +295,7 @@ or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp open = '(' {push_op(&yy->ctx, OP_OPEN)} sp close = ')' {pop_to_paren(&yy->ctx)} sp -string_test = string_headers sp (relational sp string_value | string_match sp string_value) +string_test = string_headers sp (relational sp string_value | (string_match sp string_value string_match_mod?) {set_match_mod(&yy->ctx)}) string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} sp | "Logger" {yy->ctx.mn.id = LSB_PB_LOGGER} sp @@ -301,6 +310,8 @@ string_value = ( '"' < ('\\\"' | (!'"' .))* > '"' string_match = op_seq | op_sne +string_match_mod = "%" {yy->ctx.mn.value_mod = '%'} + numeric_headers = "Severity" {yy->ctx.mn.id = LSB_PB_SEVERITY} sp | "Pid" {yy->ctx.mn.id = LSB_PB_PID} sp @@ -391,10 +402,10 @@ lsb_message_matcher* lsb_create_message_matcher(const char *exp) goto cleanup; } mm->nodes = yy.ctx.out.a; - mm->nodes[0].size = yy.ctx.out.pos; + mm->size = yy.ctx.out.pos; // turn the postfix stack into an executable tree - match_node **stack = calloc(sizeof(match_node *) * mm->nodes[0].size, 1); + match_node **stack = calloc(sizeof(match_node *) * mm->size, 1); if (!stack) { free(mm); mm = NULL; @@ -402,7 +413,7 @@ lsb_message_matcher* lsb_create_message_matcher(const char *exp) } int top = 0; - for (int i = mm->nodes[0].size - 1; i >= 0; --i) { + for (int i = mm->size - 1; i >= 0; --i) { if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { stack[top++] = &mm->nodes[i]; } else { diff --git a/src/util/string_matcher.c b/src/util/string_matcher.c index ef44d83..e4fccec 100644 --- a/src/util/string_matcher.c +++ b/src/util/string_matcher.c @@ -26,9 +26,11 @@ #include "luasandbox/util/string_matcher.h" #include +#include /* macro to `unsign' a character */ #define uchar(c) ((unsigned char)(c)) +#define L_ESC '%' typedef struct MatchState { const char *src_init; /* init of source string */ @@ -36,10 +38,6 @@ typedef struct MatchState { } MatchState; -#define L_ESC '%' -#define SPECIALS "^$*+?.([%-" - - static const char* classend(const char *p) { switch (*p++) { @@ -258,6 +256,29 @@ static const char* match(MatchState *ms, const char *s, const char *p) } +static const char* lmemfind(const char *s1, size_t l1, + const char *s2, size_t l2) +{ + if (l2 == 0) return s1; /* empty strings are everywhere */ + else if (l2 > l1) return NULL; /* avoids a negative `l1' */ + else { + const char *init; /* to search for a `*s2' inside `s1' */ + l2--; /* 1st char will be checked by `memchr' */ + l1 = l1 - l2; /* `s2' cannot be found after that */ + while (l1 > 0 && (init = (const char *)memchr(s1, *s2, l1)) != NULL) { + init++; /* 1st char is already checked */ + if (memcmp(init, s2 + 1, l2) == 0) return init - 1; + else { /* correct `l1' and `s1' to try again */ + l1 -= init - s1; + s1 = init; + } + } + return NULL; /* not found */ + } +} + + + bool lsb_string_match(const char *s, size_t len, const char *p) { MatchState ms; @@ -273,3 +294,12 @@ bool lsb_string_match(const char *s, size_t len, const char *p) } while (s1++ < ms.src_end && !anchor); return false; } + + +bool lsb_string_find(const char *s, size_t ls, const char *p, size_t lp) +{ + if (lmemfind(s, ls, p, lp)) { + return true; + } + return false; +} diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 286de36..461a20b 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -16,9 +16,9 @@ #include "luasandbox/util/heka_message.h" #include "luasandbox/util/heka_message_matcher.h" -// {"Logger":"GoSpec","Uuid":"xxx","Pid":32157,"Severity":6,"EnvVersion":"0.8","Fields":[{""value":["bar"],"name":"foo","value_type":0},{"value":[64],"name":"number","value_type":2},{"value":["data"],"name":"bytes","value_type":1},{"value":[999,1024],"name":"int","value_type":2},{"value":[99.9],"name":"double","value_type":3},{"value":[true],"name":"bool","value_type":4},{"value":["alternate"],"name":"foo","value_type":0},{"value":["name=test;type=web;"],"name":"Payload","value_type":0},{"representation":"date-time","value":["Mon Jan 02 15:04:05 -0700 2006"],"name":"Timestamp","value_type":0},{"value":[0],"name":"zero","value_type":2},{"value":["43"],"name":"string","value_type":0}],"Payload":"Test Payload","Timestamp":1.428773426113e+18,"Hostname":"trink-x230","Type":"TEST"} +// {"Logger":"GoSpec","Uuid":"xxx","Pid":32157,"Severity":6,"EnvVersion":"0.8","Fields":[{""value":["bar"],"name":"foo","value_type":0},{"value":[64],"name":"number","value_type":2},{"value":["data"],"name":"bytes","value_type":1},{"value":[999,1024],"name":"int","value_type":2},{"value":[99.9],"name":"double","value_type":3},{"value":[true],"name":"bool","value_type":4},{"value":["alternate"],"name":"foo","value_type":0},{"value":["name=test;type=web;"],"name":"Payload","value_type":0},{"representation":"date-time","value":["Mon Jan 02 15:04:05 -0700 2006"],"name":"Timestamp","value_type":0},{"value":[0],"name":"zero","value_type":2},{"value":["43"],"name":"string","value_type":0}],"Payload":"Test Payload with a longer string to attempt to create a difference in pattern match time versus the string literal match time for a unique-item","Timestamp":1.428773426113e+18,"Hostname":"trink-x230","Type":"TEST"} -char pb[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x0c\x54\x65\x73\x74\x20\x50\x61\x79\x6c\x6f\x61\x64\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; +char pb[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x90\x01Test Payload with a longer string to attempt to create a difference in pattern match time versus the string literal match time for a unique-item\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; size_t pblen = sizeof(pb); @@ -63,7 +63,7 @@ static char* test_true_matcher() , "Timestamp > '2015-04-11T17:30:26Z'" // 1428773426000000000 , "Timestamp > '2015-04-11T17:30:26.112Z'" , "Timestamp < '2015-04-11T17:30:26.114Z'" - , "(Severity == 7 || Payload == 'Test Payload') && Type == 'TEST'" + , "(Severity == 7 || Logger == 'GoSpec') && Type == 'TEST'" , "EnvVersion == \"0.8\"" , "EnvVersion == '0.8'" , "EnvVersion != '0.9'" @@ -85,7 +85,7 @@ static char* test_true_matcher() , "Timestamp > 0" , "Type != 'test'" , "Type == 'TEST' && Severity == 6" - , "Type == 'test' && Severity == 7 || Payload == 'Test Payload'" + , "Type == 'test' && Severity == 7 || Logger == 'GoSpec'" , "Type == 'TEST'" , "Type == 'foo' || Type == 'bar' || Type == 'TEST'" , "Fields[foo] == 'bar'" @@ -105,6 +105,7 @@ static char* test_true_matcher() , "Fields[int][0][1] == 1024" , "Fields[double] == 99.9" , "Fields[bool] == TRUE" + , "Fields[bool] != FALSE" , "Fields[int] != NIL" , "Fields[int][0][1] != NIL" , "Fields[int][0][2] == NIL" @@ -122,6 +123,7 @@ static char* test_true_matcher() , "Fields[foo][255] == NIL" , "Fields[foo][0][255] == NIL" , T128 + , "Payload =~ 'unique-item'%" , NULL }; lsb_heka_message m; @@ -161,6 +163,7 @@ static char* test_false_matcher() , "Fields[foo][1] == 'bar'" , "Fields[foo][0][1] == 'bar'" , "Fields[bool] == FALSE" + , "Fields[bool] != TRUE" , "Fields[foo] > 'bara'" , "Fields[foo] >= 'bara'" , "Fields[foo] == 'bara'" @@ -181,6 +184,7 @@ static char* test_false_matcher() , "Type !~ 'ST$'" , "Logger =~ '.' && Type =~ '^anything'" , "Type == '" S255 "'" + , "Payload =~ 'not.found'%" , NULL }; lsb_heka_message m; @@ -227,6 +231,7 @@ static char* test_malformed_matcher() , "Type == '" S255 "5'" // string too long , "Fields[test][256] == 1" // field index out of bounds , "Fields[test][0][256] == 1" // array index out of bounds + , "Payload =~ 'foo'i" // invalid string match pattern modifier , NULL }; lsb_heka_message m; @@ -268,7 +273,11 @@ static char* benchmark_match() , "Fields[int] != NIL" , "Type =~ '^[Tt]EST' && Severity == 6" , "Payload =~ '^Test'" - , "Payload =~ 'load$'" + , "Payload =~ 'item$'" + , "Payload =~ 'unique%-item'" + , "Payload =~ 'unique-item'%" + , "Payload =~ 'unique'" + , "Payload =~ 'unique'%" , NULL }; lsb_heka_message m; diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c index 78d58b2..50289b1 100644 --- a/src/util/test/test_string_matcher.c +++ b/src/util/test/test_string_matcher.c @@ -69,6 +69,24 @@ static char* test_success() } +static char* test_find_success() +{ + char *tests[] = { + "foo." , "." + , "foobar", "foo" + , "foo-bar", "o-b" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(lsb_string_find(tests[i], strlen(tests[i]), tests[i + 1], strlen(tests[i + 1])), + "not found test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + return NULL; +} + + static char* test_failure() { char *tests[] = { @@ -102,6 +120,23 @@ static char* test_failure() } +static char* test_find_failure() +{ + char *tests[] = { + "foo" , "." + , "foobar" , "longer string" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(!lsb_string_find(tests[i], strlen(tests[i]), tests[i + 1], strlen(tests[i + 1])), + "find test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + return NULL; +} + + static char* test_invalid() { char *tests[] = { @@ -128,7 +163,9 @@ static char* all_tests() { mu_run_test(test_stub); mu_run_test(test_success); + mu_run_test(test_find_success); mu_run_test(test_failure); + mu_run_test(test_find_failure); mu_run_test(test_invalid); return NULL; } From d61292bc827d086b255481f5e7f39d2aceea0eff Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 10 Nov 2017 13:21:29 -0800 Subject: [PATCH 210/235] Issue 208 - Extend the grammar to include Fields --- src/util/heka_message_matcher_parser.c | 242 +++++++++++----------- src/util/heka_message_matcher_parser.leg | 6 +- src/util/test/test_heka_message_matcher.c | 2 + 3 files changed, 126 insertions(+), 124 deletions(-) diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index ff5ba97..eca9467 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -916,6 +916,20 @@ YY_ACTION(void) yy_1_string_match_mod(yycontext *yy, char *yytext, int yyleng) #undef yypos #undef yy } +YY_ACTION(void) yy_1_string_match(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_match\n")); + { +#line 311 + set_match_mod(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ @@ -1014,20 +1028,6 @@ YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) #undef yypos #undef yy } -YY_ACTION(void) yy_1_string_test(yycontext *yy, char *yytext, int yyleng) -{ -#define __ yy->__ -#define yypos yy->__pos -#define yythunkpos yy->__thunkpos - yyprintf((stderr, "do yy_1_string_test\n")); - { -#line 298 - set_match_mod(&yy->ctx); - } -#undef yythunkpos -#undef yypos -#undef yy -} YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ @@ -1778,7 +1778,11 @@ YY_RULE(int) yy_string_match(yycontext *yy) { int yypos69= yy->__pos, yythunkpos69= yy->__thunkpos; if (!yy_op_seq(yy)) goto l70; goto l69; l70:; yy->__pos= yypos69; yy->__thunkpos= yythunkpos69; if (!yy_op_sne(yy)) goto l68; } - l69:; + l69:; if (!yy_sp(yy)) goto l68; if (!yy_string_value(yy)) goto l68; + { int yypos71= yy->__pos, yythunkpos71= yy->__thunkpos; if (!yy_string_match_mod(yy)) goto l71; goto l72; + l71:; yy->__pos= yypos71; yy->__thunkpos= yythunkpos71; + } + l72:; yyDo(yy, yy_1_string_match, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "string_match", yy->__buf+yy->__pos)); return 1; l68:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; @@ -1788,280 +1792,276 @@ YY_RULE(int) yy_string_match(yycontext *yy) YY_RULE(int) yy_string_value(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_value")); - { int yypos72= yy->__pos, yythunkpos72= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l73; yyText(yy, yy->__begin, yy->__end); { + { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l75; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l73; +if (!(YY_BEGIN)) goto l75; #undef yytext #undef yyleng } - l74:; - { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; - { int yypos76= yy->__pos, yythunkpos76= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l77; goto l76; - l77:; yy->__pos= yypos76; yy->__thunkpos= yythunkpos76; - { int yypos78= yy->__pos, yythunkpos78= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l78; goto l75; - l78:; yy->__pos= yypos78; yy->__thunkpos= yythunkpos78; - } if (!yymatchDot(yy)) goto l75; + l76:; + { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; + { int yypos78= yy->__pos, yythunkpos78= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l79; goto l78; + l79:; yy->__pos= yypos78; yy->__thunkpos= yythunkpos78; + { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l80; goto l77; + l80:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; + } if (!yymatchDot(yy)) goto l77; } - l76:; goto l74; - l75:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; + l78:; goto l76; + l77:; yy->__pos= yypos77; yy->__thunkpos= yythunkpos77; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l73; +if (!(YY_END)) goto l75; #undef yytext #undef yyleng - } if (!yymatchChar(yy, '"')) goto l73; goto l72; - l73:; yy->__pos= yypos72; yy->__thunkpos= yythunkpos72; if (!yymatchChar(yy, '\'')) goto l71; yyText(yy, yy->__begin, yy->__end); { + } if (!yymatchChar(yy, '"')) goto l75; goto l74; + l75:; yy->__pos= yypos74; yy->__thunkpos= yythunkpos74; if (!yymatchChar(yy, '\'')) goto l73; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l71; +if (!(YY_BEGIN)) goto l73; #undef yytext #undef yyleng } - l79:; - { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; - { int yypos81= yy->__pos, yythunkpos81= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l82; goto l81; - l82:; yy->__pos= yypos81; yy->__thunkpos= yythunkpos81; - { int yypos83= yy->__pos, yythunkpos83= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l83; goto l80; - l83:; yy->__pos= yypos83; yy->__thunkpos= yythunkpos83; - } if (!yymatchDot(yy)) goto l80; + l81:; + { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; + { int yypos83= yy->__pos, yythunkpos83= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l84; goto l83; + l84:; yy->__pos= yypos83; yy->__thunkpos= yythunkpos83; + { int yypos85= yy->__pos, yythunkpos85= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l85; goto l82; + l85:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; + } if (!yymatchDot(yy)) goto l82; } - l81:; goto l79; - l80:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; + l83:; goto l81; + l82:; yy->__pos= yypos82; yy->__thunkpos= yythunkpos82; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l71; +if (!(YY_END)) goto l73; #undef yytext #undef yyleng - } if (!yymatchChar(yy, '\'')) goto l71; + } if (!yymatchChar(yy, '\'')) goto l73; } - l72:; yyText(yy, yy->__begin, yy->__end); { + l74:; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(check_string_len(yytext))) goto l71; +if (!(check_string_len(yytext))) goto l73; #undef yytext #undef yyleng } yyDo(yy, yy_1_string_value, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "string_value", yy->__buf+yy->__pos)); return 1; - l71:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l73:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_value", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_headers(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_headers")); - { int yypos85= yy->__pos, yythunkpos85= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l86; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l86; goto l85; - l86:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Logger")) goto l87; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l87; goto l85; - l87:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Hostname")) goto l88; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l88; goto l85; - l88:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "EnvVersion")) goto l89; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l89; goto l85; - l89:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Payload")) goto l90; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l90; goto l85; - l90:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; if (!yymatchString(yy, "Uuid")) goto l84; yyDo(yy, yy_6_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l84; - } - l85:; + { int yypos87= yy->__pos, yythunkpos87= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l88; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l88; goto l87; + l88:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Logger")) goto l89; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l89; goto l87; + l89:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Hostname")) goto l90; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l90; goto l87; + l90:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "EnvVersion")) goto l91; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l91; goto l87; + l91:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Payload")) goto l92; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l92; goto l87; + l92:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Uuid")) goto l86; yyDo(yy, yy_6_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l86; + } + l87:; yyprintf((stderr, " ok %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); return 1; - l84:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l86:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_boolean(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "boolean")); - { int yypos92= yy->__pos, yythunkpos92= yy->__thunkpos; if (!yy_true(yy)) goto l93; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l92; - l93:; yy->__pos= yypos92; yy->__thunkpos= yythunkpos92; if (!yy_false(yy)) goto l91; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); + { int yypos94= yy->__pos, yythunkpos94= yy->__thunkpos; if (!yy_true(yy)) goto l95; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l94; + l95:; yy->__pos= yypos94; yy->__thunkpos= yythunkpos94; if (!yy_false(yy)) goto l93; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); } - l92:; + l94:; yyprintf((stderr, " ok %s @ %s\n", "boolean", yy->__buf+yy->__pos)); return 1; - l91:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l93:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "boolean", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_false(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l94; if (!yy_sp(yy)) goto l94; + yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l96; if (!yy_sp(yy)) goto l96; yyprintf((stderr, " ok %s @ %s\n", "false", yy->__buf+yy->__pos)); return 1; - l94:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l96:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "false", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_true(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l95; if (!yy_sp(yy)) goto l95; + yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l97; if (!yy_sp(yy)) goto l97; yyprintf((stderr, " ok %s @ %s\n", "true", yy->__buf+yy->__pos)); return 1; - l95:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l97:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "true", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_relational(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "relational")); - { int yypos97= yy->__pos, yythunkpos97= yy->__thunkpos; if (!yy_op_eq(yy)) goto l98; goto l97; - l98:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_ne(yy)) goto l99; goto l97; - l99:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_gte(yy)) goto l100; goto l97; - l100:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_gt(yy)) goto l101; goto l97; - l101:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_lte(yy)) goto l102; goto l97; - l102:; yy->__pos= yypos97; yy->__thunkpos= yythunkpos97; if (!yy_op_lt(yy)) goto l96; - } - l97:; + { int yypos99= yy->__pos, yythunkpos99= yy->__thunkpos; if (!yy_op_eq(yy)) goto l100; goto l99; + l100:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_ne(yy)) goto l101; goto l99; + l101:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_gte(yy)) goto l102; goto l99; + l102:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_gt(yy)) goto l103; goto l99; + l103:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_lte(yy)) goto l104; goto l99; + l104:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_lt(yy)) goto l98; + } + l99:; yyprintf((stderr, " ok %s @ %s\n", "relational", yy->__buf+yy->__pos)); return 1; - l96:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l98:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "relational", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_lt(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l103; if (!yy_sp(yy)) goto l103; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); return 1; - l103:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_lte(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l104; if (!yy_sp(yy)) goto l104; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); return 1; - l104:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_gt(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); return 1; - l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_gte(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); return 1; - l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_sne(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l109; if (!yy_sp(yy)) goto l109; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); return 1; - l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_seq(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l110; if (!yy_sp(yy)) goto l110; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); return 1; - l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l110:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_ne(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l109; if (!yy_sp(yy)) goto l109; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l111; if (!yy_sp(yy)) goto l111; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); return 1; - l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l111:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_eq(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l110; if (!yy_sp(yy)) goto l110; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l112; if (!yy_sp(yy)) goto l112; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); return 1; - l110:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l112:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_boolean_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "boolean_test")); - { int yypos112= yy->__pos, yythunkpos112= yy->__thunkpos; if (!yy_true(yy)) goto l113; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l112; - l113:; yy->__pos= yypos112; yy->__thunkpos= yythunkpos112; if (!yy_false(yy)) goto l111; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); + { int yypos114= yy->__pos, yythunkpos114= yy->__thunkpos; if (!yy_true(yy)) goto l115; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l114; + l115:; yy->__pos= yypos114; yy->__thunkpos= yythunkpos114; if (!yy_false(yy)) goto l113; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); } - l112:; + l114:; yyprintf((stderr, " ok %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); return 1; - l111:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l113:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_ts_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "ts_test")); if (!yymatchString(yy, "Timestamp")) goto l114; if (!yy_sp(yy)) goto l114; if (!yy_relational(yy)) goto l114; if (!yy_sp(yy)) goto l114; - { int yypos115= yy->__pos, yythunkpos115= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l116; goto l115; - l116:; yy->__pos= yypos115; yy->__thunkpos= yythunkpos115; if (!yy_ts_quoted(yy)) goto l114; + yyprintf((stderr, "%s\n", "ts_test")); if (!yymatchString(yy, "Timestamp")) goto l116; if (!yy_sp(yy)) goto l116; if (!yy_relational(yy)) goto l116; if (!yy_sp(yy)) goto l116; + { int yypos117= yy->__pos, yythunkpos117= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l118; goto l117; + l118:; yy->__pos= yypos117; yy->__thunkpos= yythunkpos117; if (!yy_ts_quoted(yy)) goto l116; } - l115:; yyDo(yy, yy_1_ts_test, yy->__begin, yy->__end); + l117:; yyDo(yy, yy_1_ts_test, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); return 1; - l114:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l116:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_field_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l117; if (!yy_sp(yy)) goto l117; - { int yypos118= yy->__pos, yythunkpos118= yy->__thunkpos; if (!yy_relational(yy)) goto l119; if (!yy_sp(yy)) goto l119; - { int yypos120= yy->__pos, yythunkpos120= yy->__thunkpos; if (!yy_string_value(yy)) goto l121; goto l120; - l121:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; if (!yy_numeric_value(yy)) goto l119; + yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l119; if (!yy_sp(yy)) goto l119; + { int yypos120= yy->__pos, yythunkpos120= yy->__thunkpos; if (!yy_relational(yy)) goto l121; if (!yy_sp(yy)) goto l121; + { int yypos122= yy->__pos, yythunkpos122= yy->__thunkpos; if (!yy_string_value(yy)) goto l123; goto l122; + l123:; yy->__pos= yypos122; yy->__thunkpos= yythunkpos122; if (!yy_numeric_value(yy)) goto l121; } - l120:; goto l118; - l119:; yy->__pos= yypos118; yy->__thunkpos= yythunkpos118; if (!yy_string_match(yy)) goto l122; if (!yy_sp(yy)) goto l122; if (!yy_string_value(yy)) goto l122; goto l118; - l122:; yy->__pos= yypos118; yy->__thunkpos= yythunkpos118; - { int yypos123= yy->__pos, yythunkpos123= yy->__thunkpos; if (!yy_op_eq(yy)) goto l124; goto l123; - l124:; yy->__pos= yypos123; yy->__thunkpos= yythunkpos123; if (!yy_op_ne(yy)) goto l117; + l122:; goto l120; + l121:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; if (!yy_string_match(yy)) goto l124; goto l120; + l124:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; + { int yypos125= yy->__pos, yythunkpos125= yy->__thunkpos; if (!yy_op_eq(yy)) goto l126; goto l125; + l126:; yy->__pos= yypos125; yy->__thunkpos= yythunkpos125; if (!yy_op_ne(yy)) goto l119; } - l123:; if (!yy_sp(yy)) goto l117; - { int yypos125= yy->__pos, yythunkpos125= yy->__thunkpos; if (!yy_boolean(yy)) goto l126; goto l125; - l126:; yy->__pos= yypos125; yy->__thunkpos= yythunkpos125; if (!yy_nil(yy)) goto l117; + l125:; if (!yy_sp(yy)) goto l119; + { int yypos127= yy->__pos, yythunkpos127= yy->__thunkpos; if (!yy_boolean(yy)) goto l128; goto l127; + l128:; yy->__pos= yypos127; yy->__thunkpos= yythunkpos127; if (!yy_nil(yy)) goto l119; } - l125:; + l127:; } - l118:; + l120:; yyprintf((stderr, " ok %s @ %s\n", "field_test", yy->__buf+yy->__pos)); return 1; - l117:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l119:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "field_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_numeric_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "numeric_test")); if (!yy_numeric_headers(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_relational(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_numeric_value(yy)) goto l127; + yyprintf((stderr, "%s\n", "numeric_test")); if (!yy_numeric_headers(yy)) goto l129; if (!yy_sp(yy)) goto l129; if (!yy_relational(yy)) goto l129; if (!yy_sp(yy)) goto l129; if (!yy_numeric_value(yy)) goto l129; yyprintf((stderr, " ok %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); return 1; - l127:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l129:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "string_test")); if (!yy_string_headers(yy)) goto l128; if (!yy_sp(yy)) goto l128; - { int yypos129= yy->__pos, yythunkpos129= yy->__thunkpos; if (!yy_relational(yy)) goto l130; if (!yy_sp(yy)) goto l130; if (!yy_string_value(yy)) goto l130; goto l129; - l130:; yy->__pos= yypos129; yy->__thunkpos= yythunkpos129; if (!yy_string_match(yy)) goto l128; if (!yy_sp(yy)) goto l128; if (!yy_string_value(yy)) goto l128; - { int yypos131= yy->__pos, yythunkpos131= yy->__thunkpos; if (!yy_string_match_mod(yy)) goto l131; goto l132; - l131:; yy->__pos= yypos131; yy->__thunkpos= yythunkpos131; - } - l132:; yyDo(yy, yy_1_string_test, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "string_test")); if (!yy_string_headers(yy)) goto l130; if (!yy_sp(yy)) goto l130; + { int yypos131= yy->__pos, yythunkpos131= yy->__thunkpos; if (!yy_relational(yy)) goto l132; if (!yy_sp(yy)) goto l132; if (!yy_string_value(yy)) goto l132; goto l131; + l132:; yy->__pos= yypos131; yy->__thunkpos= yythunkpos131; if (!yy_string_match(yy)) goto l130; } - l129:; + l131:; yyprintf((stderr, " ok %s @ %s\n", "string_test", yy->__buf+yy->__pos)); return 1; - l128:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l130:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_test", yy->__buf+yy->__pos)); return 0; } diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index 088c5fd..9f02360 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -295,7 +295,7 @@ or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp open = '(' {push_op(&yy->ctx, OP_OPEN)} sp close = ')' {pop_to_paren(&yy->ctx)} sp -string_test = string_headers sp (relational sp string_value | (string_match sp string_value string_match_mod?) {set_match_mod(&yy->ctx)}) +string_test = string_headers sp (relational sp string_value | string_match) string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} sp | "Logger" {yy->ctx.mn.id = LSB_PB_LOGGER} sp @@ -308,7 +308,7 @@ string_value = ( '"' < ('\\\"' | (!'"' .))* > '"' | "'" < ("\\\'" | (!"'" .))* > "'" ) &{check_string_len(yytext)} {set_string_value(&yy->ctx, yytext)} -string_match = op_seq | op_sne +string_match = (op_seq | op_sne) sp string_value string_match_mod? {set_match_mod(&yy->ctx)} string_match_mod = "%" {yy->ctx.mn.value_mod = '%'} @@ -324,7 +324,7 @@ decimal = "." [0-9]+ exponent = [eE] sign? [0-9]+ field_test = fields sp ((relational sp (string_value | numeric_value)) - | string_match sp string_value + | string_match | (op_eq | op_ne) sp (boolean | nil)) fields = "Fields[" < [^\]]* > "]" {set_field(&yy->ctx, yytext)} f:index? {yy->ctx.mn.fi = f} a:index? {yy->ctx.mn.ai = a} index = "[" < zero_to_255 > "]" {$$ = atoi(yytext)} diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 461a20b..88faacd 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -124,6 +124,7 @@ static char* test_true_matcher() , "Fields[foo][0][255] == NIL" , T128 , "Payload =~ 'unique-item'%" + , "Fields[Timestamp] =~ ' -0700'%" , NULL }; lsb_heka_message m; @@ -185,6 +186,7 @@ static char* test_false_matcher() , "Logger =~ '.' && Type =~ '^anything'" , "Type == '" S255 "'" , "Payload =~ 'not.found'%" + , "Fields[foo][1] =~ 'not.found'%" , NULL }; lsb_heka_message m; From 554fe51d291b35ce8db7222951bdc35ed203ec66 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 17 Nov 2017 09:08:52 -0800 Subject: [PATCH 211/235] Issue 201 - Add an input_limit configuration variable to the sandbox --- CMakeLists.txt | 2 +- docs/sandbox.md | 3 +++ include/luasandbox.h | 1 + src/luasandbox.c | 3 +++ src/test/lua/read_config.lua | 1 + src/test/test_generic_sandbox.c | 3 +++ 6 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 65fb963..c71e32a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.7 LANGUAGES C) +project(luasandbox VERSION 1.2.8 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/sandbox.md b/docs/sandbox.md index 813c61f..ab87a0d 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -2,6 +2,9 @@ ## Configuration +* **input_limit** - the largest input string that is allowed to be processed + (bytes (unsigned), default 65536, 0 for unlimited*). This is not directly used + by the sandbox it is made availble to modules to standardize the configuration. * **output_limit** - the largest output string an input or analysis plugin can inject into the host (bytes (unsigned), default 65536, 0 for unlimited*) * **memory_limit** - the maximum amount of memory a plugin can use before being diff --git a/include/luasandbox.h b/include/luasandbox.h index cd89842..32f8744 100644 --- a/include/luasandbox.h +++ b/include/luasandbox.h @@ -32,6 +32,7 @@ #define LSB_THIS_PTR "lsb_this_ptr" #define LSB_MEMORY_LIMIT "memory_limit" #define LSB_INSTRUCTION_LIMIT "instruction_limit" +#define LSB_INPUT_LIMIT "input_limit" #define LSB_OUTPUT_LIMIT "output_limit" #define LSB_LOG_LEVEL "log_level" #define LSB_LUA_PATH "path" diff --git a/src/luasandbox.c b/src/luasandbox.c index 85ebe4f..d6ff41e 100644 --- a/src/luasandbox.c +++ b/src/luasandbox.c @@ -310,6 +310,9 @@ static lua_State* load_sandbox_config(const char *cfg, lsb_logger *logger) int ret = luaL_dostring(L, cfg); if (ret) goto cleanup; + ret = check_size(L, LUA_GLOBALSINDEX, LSB_INPUT_LIMIT, 64 * 1024); + if (ret) goto cleanup; + ret = check_size(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); if (ret) goto cleanup; diff --git a/src/test/lua/read_config.lua b/src/test/lua/read_config.lua index a72779e..30d1c8d 100644 --- a/src/test/lua/read_config.lua +++ b/src/test/lua/read_config.lua @@ -2,6 +2,7 @@ assert(type(read_config) == "function") assert(read_config("memory_limit") == 65765) assert(read_config("instruction_limit") == 1000) assert(read_config("output_limit") == 1024) +assert(read_config("input_limit") == 1024 * 64) local array = read_config("array") assert(type(array) == "table") diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c index 30e7a15..b16d2e2 100644 --- a/src/test/test_generic_sandbox.c +++ b/src/test/test_generic_sandbox.c @@ -201,6 +201,9 @@ static char* test_create_error() lsb_lua_sandbox *sb = lsb_create(NULL, NULL, NULL, NULL); mu_assert(!sb, "lsb_create() null lua_file"); + sb = lsb_create(NULL, "lua/counter.lua", "input_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + sb = lsb_create(NULL, "lua/counter.lua", "output_limit = 'aaa'", NULL); mu_assert(!sb, "lsb_create() invalid config"); From 7e847060bcb0d54d1cd6dfa59aa32f44660f4eed Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 17 Nov 2017 10:05:18 -0800 Subject: [PATCH 212/235] Issue 213 - Message matcher improvements --- src/util/heka_message_matcher.c | 133 ++++---- src/util/heka_message_matcher_impl.h | 42 ++- src/util/heka_message_matcher_parser.c | 392 ++++++++++++++-------- src/util/heka_message_matcher_parser.leg | 294 +++++++++++----- src/util/test/test_heka_message_matcher.c | 46 ++- 5 files changed, 610 insertions(+), 297 deletions(-) diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c index 3057c51..c1c7125 100644 --- a/src/util/heka_message_matcher.c +++ b/src/util/heka_message_matcher.c @@ -20,50 +20,51 @@ static bool string_test(match_node *mn, lsb_const_string *val) { + const char *mn_val = mn->data + mn->var_len; switch (mn->op) { case OP_EQ: - if (val->len != mn->value_len) return false; - return strncmp(val->s, mn->value.s, val->len) == 0; + if (val->len != mn->val_len) return false; + return strncmp(val->s, mn_val, val->len) == 0; case OP_NE: - if (val->len != mn->value_len) return true; - return strncmp(val->s, mn->value.s, val->len) != 0; + if (val->len != mn->val_len) return true; + return strncmp(val->s, mn_val, val->len) != 0; case OP_LT: { - int cmp = strncmp(val->s, mn->value.s, val->len); + int cmp = strncmp(val->s, mn_val, val->len); if (cmp == 0) { - return val->len < mn->value_len; + return val->len < mn->val_len; } return cmp < 0; } case OP_LTE: - return strncmp(val->s, mn->value.s, val->len) <= 0; + return strncmp(val->s, mn_val, val->len) <= 0; case OP_GT: { - int cmp = strncmp(val->s, mn->value.s, val->len); + int cmp = strncmp(val->s, mn_val, val->len); if (cmp == 0) { - return val->len > mn->value_len; + return val->len > mn->val_len; } return cmp > 0; } case OP_GTE: { - int cmp = strncmp(val->s, mn->value.s, val->len); + int cmp = strncmp(val->s, mn_val, val->len); if (cmp == 0) { - return val->len >= mn->value_len; + return val->len >= mn->val_len; } return cmp > 0; } case OP_RE: - if (mn->value_mod == '%') { - return lsb_string_find(val->s, val->len, mn->value.s, mn->value_len); + if (mn->val_mod == PATTERN_MOD_ESC) { + return lsb_string_find(val->s, val->len, mn_val, mn->val_len); } else { - return lsb_string_match(val->s, val->len, mn->value.s); + return lsb_string_match(val->s, val->len, mn_val); } case OP_NRE: - if (mn->value_mod == '%') { - return !lsb_string_find(val->s, val->len, mn->value.s, mn->value_len); + if (mn->val_mod == PATTERN_MOD_ESC) { + return !lsb_string_find(val->s, val->len, mn_val, mn->val_len); } else { - return !lsb_string_match(val->s, val->len, mn->value.s); + return !lsb_string_match(val->s, val->len, mn_val); } default: break; @@ -74,19 +75,21 @@ static bool string_test(match_node *mn, lsb_const_string *val) static bool numeric_test(match_node *mn, double val) { + double d; + memcpy(&d, mn->data + mn->var_len, sizeof(double)); switch (mn->op) { case OP_EQ: - return val == mn->value.d; + return val == d; case OP_NE: - return val != mn->value.d; + return val != d; case OP_LT: - return val < mn->value.d; + return val < d; case OP_LTE: - return val <= mn->value.d; + return val <= d; case OP_GT: - return val > mn->value.d; + return val > d; case OP_GTE: - return val >= mn->value.d; + return val >= d; default: break; } @@ -102,7 +105,7 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) case OP_FALSE: return false; default: - switch (mn->id) { + switch (mn->field_id) { case LSB_PB_TIMESTAMP: return numeric_test(mn, (double)m->timestamp); case LSB_PB_TYPE: @@ -124,17 +127,17 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) default: { lsb_read_value val; - lsb_const_string variable = { .s = mn->variable, - .len = mn->variable_len }; + lsb_const_string variable = { .s = mn->data, .len = mn->var_len }; - if (!lsb_read_heka_field(m, &variable, mn->fi, mn->ai, &val)) { - if (mn->value_type == TYPE_NIL) { + if (!lsb_read_heka_field(m, &variable, mn->u.idx.f, mn->u.idx.a, + &val)) { + if (mn->val_type == TYPE_NIL) { return mn->op == OP_EQ; } return false; } - switch (mn->value_type) { + switch (mn->val_type) { case TYPE_STRING: if (val.type == LSB_READ_STRING) { return string_test(mn, &val.u.s); @@ -145,9 +148,14 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) return numeric_test(mn, val.u.d); } break; - case TYPE_BOOLEAN: + case TYPE_TRUE: if (val.type == LSB_READ_BOOL || val.type == LSB_READ_NUMERIC) { - return numeric_test(mn, val.u.d); + return mn->op == OP_EQ ? val.u.d == true : val.u.d != true; + } + break; + case TYPE_FALSE: + if (val.type == LSB_READ_BOOL || val.type == LSB_READ_NUMERIC) { + return mn->op == OP_EQ ? val.u.d == false: val.u.d != false; } break; case TYPE_NIL: @@ -162,45 +170,9 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) } -static bool eval_tree(match_node *root, match_node *mn, lsb_heka_message *m) -{ - bool match; - if (mn->id == 0 && mn->fi) { - match = eval_tree(root, root + mn->fi, m); - } else { - match = eval_node(mn, m); - } - - if (match && mn->op == OP_OR) { - return match; // short circuit - } - - if (!match && mn->op == OP_AND) { - return match; // short circuit - } - - if (mn->id == 0 && mn->ai) { - match = eval_tree(root, root + mn->ai, m); - } - return match; -} - - void lsb_destroy_message_matcher(lsb_message_matcher *mm) { if (!mm) return; - - for (int i = 0; i < mm->size; ++i) { - free(mm->nodes[i].variable); - switch (mm->nodes[i].value_type) { - case TYPE_STRING: - free(mm->nodes[i].value.s); - break; - default: - // no action required - break; - } - } free(mm->nodes); free(mm); } @@ -208,5 +180,30 @@ void lsb_destroy_message_matcher(lsb_message_matcher *mm) bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m) { - return eval_tree(mm->nodes, mm->nodes, m); + bool match = false; + if (!mm) return match; + + match_node *s = mm->nodes; + match_node *e = mm->nodes + (mm->bytes / sizeof(match_node)); + for (match_node *p = mm->nodes; p < e;) { + switch (p->op) { + case OP_OR: + if (match) { + p = s + p->u.off; // short circuit + continue; + } + break; + case OP_AND: + if (!match) { + p = s + p->u.off; // short circuit + continue; + } + break; + default: + match = eval_node(p, m); + break; + } + p += p->units; + } + return match; } diff --git a/src/util/heka_message_matcher_impl.h b/src/util/heka_message_matcher_impl.h index 21eb36d..ebf3543 100644 --- a/src/util/heka_message_matcher_impl.h +++ b/src/util/heka_message_matcher_impl.h @@ -10,6 +10,7 @@ #define luasandbox_util_heka_message_matcher_impl_h_ #include +#include typedef enum { OP_EQ, @@ -32,30 +33,41 @@ typedef enum { TYPE_NIL, TYPE_STRING, TYPE_NUMERIC, - TYPE_BOOLEAN, + TYPE_FALSE, + TYPE_TRUE, } match_type; -typedef struct match_node { - unsigned char id; - unsigned char op; - unsigned char value_mod; - unsigned char value_type; - unsigned char value_len; - unsigned char variable_len; - unsigned char fi; // left node index for logical op - unsigned char ai; // right node index for logical op - char *variable; +typedef enum { + PATTERN_MOD_NONE, + PATTERN_MOD_ESC, +} match_pattern_mod; + +struct indices { + uint8_t f; + uint8_t a; +}; + + +typedef struct match_node { + uint8_t op; + uint8_t units; + uint8_t var_len; + uint8_t val_len; + uint8_t field_id; + uint8_t val_type : 4; + uint8_t val_mod : 4; union { - char *s; - double d; - } value; + struct indices idx; + uint16_t off; + } u; + char data[]; // inlined field variable and value data (when necessary) } match_node; struct lsb_message_matcher { - unsigned char size; + size_t bytes; match_node *nodes; }; diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index eca9467..c7e8ce3 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -31,10 +31,28 @@ #pragma warning( disable : 4267 4244 ) #endif +typedef struct match_node_tmp { + uint8_t id; + uint8_t op; + uint8_t val_mod; + uint8_t val_type; + uint8_t val_len; + uint8_t var_len; + uint8_t fi; // left node index for logical op + uint8_t ai; // right node index for logical op + char *var; + + union { + char *s; + double d; + } val; +} match_node_tmp; + + typedef struct match_node_array { - match_node *a; - int pos; - int size; + match_node_tmp *a; + int pos; + int size; } match_node_array; @@ -48,7 +66,7 @@ typedef struct input_string { typedef struct context { match_node_array out; match_node_array ops; - match_node mn; + match_node_tmp mn; struct tm tms; int cond_cnt; input_string is; @@ -77,23 +95,23 @@ typedef struct context { } -static void init_match_node(match_node *mn) +static void init_match_node(match_node_tmp *mn) { - memset(mn, 0, sizeof(match_node)); + memset(mn, 0, sizeof(match_node_tmp)); } -static void move_match_node(match_node *dest, match_node *src) +static void move_match_node(match_node_tmp *dest, match_node_tmp *src) { - memcpy(dest, src, sizeof(match_node)); + memcpy(dest, src, sizeof(match_node_tmp)); init_match_node(src); // dest now owns the memory, wipe the pointers } static void realloc_mna(match_node_array *mna) { - size_t bytes = sizeof(match_node) * ++mna->size; - match_node *tmp = realloc(mna->a, bytes); + size_t bytes = sizeof(match_node_tmp) * ++mna->size; + match_node_tmp *tmp = realloc(mna->a, bytes); if (tmp) { mna->a = tmp; init_match_node(&mna->a[mna->size - 1]); @@ -104,7 +122,7 @@ static void realloc_mna(match_node_array *mna) } -static void push_output(context *ctx, match_node *mn) +static void push_output(context *ctx, match_node_tmp *mn) { if (!ctx->out.a || ctx->out.pos == ctx->out.size) { realloc_mna(&ctx->out); @@ -136,7 +154,7 @@ static void push_op(context *ctx, match_operation op) static void pop_to_paren(context *ctx) { for (; ctx->ops.pos > 0; --ctx->ops.pos) { - match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; if (op->op == OP_OPEN) break; push_output(ctx, op); } @@ -146,7 +164,7 @@ static void pop_to_paren(context *ctx) static void pop_all_ops(context *ctx) { for (; ctx->ops.pos > 0; --ctx->ops.pos) { - match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; if (op->op == OP_OPEN) continue; push_output(ctx, op); } @@ -172,30 +190,30 @@ static void update_time(context *ctx, int hour, int minute, int sec) static void update_offset(context *ctx, char sign, int hour, int minute) { - ctx->mn.value.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); + ctx->mn.val.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); } static void set_field(context *ctx, char *name) { ctx->mn.id = LSB_PB_FIELDS; - ctx->mn.variable_len = strlen(name); - ctx->mn.variable = malloc(ctx->mn.variable_len + 1); - if (!ctx->mn.variable) { + ctx->mn.var_len = strlen(name); + ctx->mn.var = malloc(ctx->mn.var_len + 1); + if (!ctx->mn.var) { fprintf(stderr, "malloc failed\n"); exit(1); } - memcpy(ctx->mn.variable, name, ctx->mn.variable_len + 1); + memcpy(ctx->mn.var, name, ctx->mn.var_len + 1); } static void set_timestamp(context *ctx) { ctx->mn.id = LSB_PB_TIMESTAMP; - ctx->mn.value_type = TYPE_NUMERIC; + ctx->mn.val_type = TYPE_NUMERIC; if (ctx->tms.tm_isdst == -1) { - ctx->mn.value.d += mktime(&ctx->tms); - ctx->mn.value.d *= 1e9; + ctx->mn.val.d += mktime(&ctx->tms); + ctx->mn.val.d *= 1e9; } memset(&ctx->tms, 0, sizeof(struct tm)); } @@ -203,14 +221,14 @@ static void set_timestamp(context *ctx) static void set_numeric_value(context *ctx, char *s) { - ctx->mn.value_type = TYPE_NUMERIC; - ctx->mn.value.d = strtod(s, NULL); + ctx->mn.val_type = TYPE_NUMERIC; + ctx->mn.val.d = strtod(s, NULL); } static void set_string_value(context *ctx, char *s) { - ctx->mn.value_type = TYPE_STRING; + ctx->mn.val_type = TYPE_STRING; int i, j; for (i = 0, j = 0; s[i]; ++i, ++j) { if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { @@ -220,21 +238,21 @@ static void set_string_value(context *ctx, char *s) } s[j] = 0; - ctx->mn.value_len = j; - ctx->mn.value.s = malloc(j + 1); - if (!ctx->mn.value.s) { + ctx->mn.val_len = j; + ctx->mn.val.s = malloc(j + 1); + if (!ctx->mn.val.s) { fprintf(stderr, "malloc failed\n"); exit(1); } - memcpy(ctx->mn.value.s, s, j + 1); + memcpy(ctx->mn.val.s, s, j + 1); } static void set_match_mod(context *ctx) { - if (ctx->mn.value_mod == '\0' - && strpbrk(ctx->mn.value.s, "^$*+?.[%-") == NULL) { // literal - ctx->mn.value_mod = '%'; + if (ctx->mn.val_mod == PATTERN_MOD_NONE + && strpbrk(ctx->mn.val.s, "^$*+?.[%-") == NULL) { // literal + ctx->mn.val_mod = PATTERN_MOD_ESC; } } @@ -251,13 +269,6 @@ static bool check_string_len(char *s) } -static void set_boolean_value(context *ctx, bool b) -{ - ctx->mn.value_type = TYPE_BOOLEAN; - ctx->mn.value.d = b; -} - - static int cond_cnt(context *ctx) { return (++ctx->cond_cnt * 2 + 1 > UCHAR_MAX) ? 0 : 1; @@ -597,8 +608,8 @@ YY_ACTION(void) yy_1_nil(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_nil\n")); { -#line 365 - yy->ctx.mn.value_type = TYPE_NIL;; +#line 376 + yy->ctx.mn.val_type = TYPE_NIL;; } #undef yythunkpos #undef yypos @@ -611,8 +622,8 @@ YY_ACTION(void) yy_1_second_frac(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second_frac\n")); { -#line 363 - yy->ctx.mn.value.d += strtod(yytext, NULL); +#line 374 + yy->ctx.mn.val.d += strtod(yytext, NULL); } #undef yythunkpos #undef yypos @@ -625,7 +636,7 @@ YY_ACTION(void) yy_1_second(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second\n")); { -#line 362 +#line 373 __ = atoi(yytext); } #undef yythunkpos @@ -639,7 +650,7 @@ YY_ACTION(void) yy_1_minute(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_minute\n")); { -#line 358 +#line 369 __ = atoi(yytext); } #undef yythunkpos @@ -653,7 +664,7 @@ YY_ACTION(void) yy_1_hour(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_hour\n")); { -#line 357 +#line 368 __ = atoi(yytext); } #undef yythunkpos @@ -669,7 +680,7 @@ YY_ACTION(void) yy_2_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_timeoffset\n")); { -#line 353 +#line 364 update_offset(&yy->ctx, yytext[0], h, m); } #undef yythunkpos @@ -687,7 +698,7 @@ YY_ACTION(void) yy_1_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_timeoffset\n")); { -#line 352 +#line 363 update_offset(&yy->ctx, '+', 0, 0); } #undef yythunkpos @@ -706,7 +717,7 @@ YY_ACTION(void) yy_1_partialtime(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_partialtime\n")); { -#line 351 +#line 362 update_time(&yy->ctx, h, m, s); } #undef yythunkpos @@ -723,7 +734,7 @@ YY_ACTION(void) yy_1_day(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_day\n")); { -#line 347 +#line 358 __ = atoi(yytext); } #undef yythunkpos @@ -737,7 +748,7 @@ YY_ACTION(void) yy_1_month(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_month\n")); { -#line 342 +#line 353 __ = atoi(yytext); } #undef yythunkpos @@ -751,7 +762,7 @@ YY_ACTION(void) yy_1_year(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_year\n")); { -#line 339 +#line 350 __ = atoi(yytext); } #undef yythunkpos @@ -768,7 +779,7 @@ YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fulldate\n")); { -#line 338 +#line 349 update_date(&yy->ctx, y, m, d); } #undef yythunkpos @@ -785,7 +796,7 @@ YY_ACTION(void) yy_1_ts_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_ts_test\n")); { -#line 336 +#line 347 set_timestamp(&yy->ctx); } #undef yythunkpos @@ -799,7 +810,7 @@ YY_ACTION(void) yy_1_index(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_index\n")); { -#line 330 +#line 341 __ = atoi(yytext); } #undef yythunkpos @@ -815,7 +826,7 @@ YY_ACTION(void) yy_3_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_fields\n")); { -#line 329 +#line 340 yy->ctx.mn.ai = a; } #undef yythunkpos @@ -833,7 +844,7 @@ YY_ACTION(void) yy_2_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_fields\n")); { -#line 329 +#line 340 yy->ctx.mn.fi = f; } #undef yythunkpos @@ -851,7 +862,7 @@ YY_ACTION(void) yy_1_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fields\n")); { -#line 329 +#line 340 set_field(&yy->ctx, yytext); } #undef yythunkpos @@ -867,7 +878,7 @@ YY_ACTION(void) yy_1_numeric_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_numeric_value\n")); { -#line 319 +#line 330 set_numeric_value(&yy->ctx, yytext); } #undef yythunkpos @@ -881,7 +892,7 @@ YY_ACTION(void) yy_2_numeric_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_numeric_headers\n")); { -#line 316 +#line 327 yy->ctx.mn.id = LSB_PB_PID; } #undef yythunkpos @@ -895,7 +906,7 @@ YY_ACTION(void) yy_1_numeric_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_numeric_headers\n")); { -#line 315 +#line 326 yy->ctx.mn.id = LSB_PB_SEVERITY; } #undef yythunkpos @@ -909,8 +920,8 @@ YY_ACTION(void) yy_1_string_match_mod(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_match_mod\n")); { -#line 313 - yy->ctx.mn.value_mod = '%'; +#line 324 + yy->ctx.mn.val_mod = PATTERN_MOD_ESC; } #undef yythunkpos #undef yypos @@ -923,7 +934,7 @@ YY_ACTION(void) yy_1_string_match(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_match\n")); { -#line 311 +#line 322 set_match_mod(&yy->ctx); } #undef yythunkpos @@ -937,7 +948,7 @@ YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_value\n")); { -#line 309 +#line 320 set_string_value(&yy->ctx, yytext); } #undef yythunkpos @@ -951,7 +962,7 @@ YY_ACTION(void) yy_6_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_6_string_headers\n")); { -#line 305 +#line 316 yy->ctx.mn.id = LSB_PB_UUID; } #undef yythunkpos @@ -965,7 +976,7 @@ YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_5_string_headers\n")); { -#line 304 +#line 315 yy->ctx.mn.id = LSB_PB_PAYLOAD; } #undef yythunkpos @@ -979,7 +990,7 @@ YY_ACTION(void) yy_4_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_4_string_headers\n")); { -#line 303 +#line 314 yy->ctx.mn.id = LSB_PB_ENV_VERSION; } #undef yythunkpos @@ -993,7 +1004,7 @@ YY_ACTION(void) yy_3_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_string_headers\n")); { -#line 302 +#line 313 yy->ctx.mn.id = LSB_PB_HOSTNAME; } #undef yythunkpos @@ -1007,7 +1018,7 @@ YY_ACTION(void) yy_2_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_string_headers\n")); { -#line 301 +#line 312 yy->ctx.mn.id = LSB_PB_LOGGER; } #undef yythunkpos @@ -1021,7 +1032,7 @@ YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_headers\n")); { -#line 300 +#line 311 yy->ctx.mn.id = LSB_PB_TYPE; } #undef yythunkpos @@ -1035,7 +1046,7 @@ YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_close\n")); { -#line 296 +#line 307 pop_to_paren(&yy->ctx); } #undef yythunkpos @@ -1049,7 +1060,7 @@ YY_ACTION(void) yy_1_open(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_open\n")); { -#line 295 +#line 306 push_op(&yy->ctx, OP_OPEN); } #undef yythunkpos @@ -1063,7 +1074,7 @@ YY_ACTION(void) yy_1_or(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_or\n")); { -#line 294 +#line 305 push_op(&yy->ctx, OP_OR); } #undef yythunkpos @@ -1077,7 +1088,7 @@ YY_ACTION(void) yy_1_and(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_and\n")); { -#line 293 +#line 304 push_op(&yy->ctx, OP_AND); } #undef yythunkpos @@ -1091,8 +1102,8 @@ YY_ACTION(void) yy_2_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean\n")); { -#line 291 - set_boolean_value(&yy->ctx, 0); +#line 302 + yy->ctx.mn.val_type = TYPE_FALSE; } #undef yythunkpos #undef yypos @@ -1105,8 +1116,8 @@ YY_ACTION(void) yy_1_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean\n")); { -#line 290 - set_boolean_value(&yy->ctx, 1); +#line 301 + yy->ctx.mn.val_type = TYPE_TRUE; } #undef yythunkpos #undef yypos @@ -1119,7 +1130,7 @@ YY_ACTION(void) yy_2_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean_test\n")); { -#line 289 +#line 300 yy->ctx.mn.op = OP_FALSE; } #undef yythunkpos @@ -1133,7 +1144,7 @@ YY_ACTION(void) yy_1_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean_test\n")); { -#line 288 +#line 299 yy->ctx.mn.op = OP_TRUE; } #undef yythunkpos @@ -1147,7 +1158,7 @@ YY_ACTION(void) yy_1_op_lt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lt\n")); { -#line 279 +#line 290 yy->ctx.mn.op = OP_LT; } #undef yythunkpos @@ -1161,7 +1172,7 @@ YY_ACTION(void) yy_1_op_lte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lte\n")); { -#line 278 +#line 289 yy->ctx.mn.op = OP_LTE; } #undef yythunkpos @@ -1175,7 +1186,7 @@ YY_ACTION(void) yy_1_op_gt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gt\n")); { -#line 277 +#line 288 yy->ctx.mn.op = OP_GT; } #undef yythunkpos @@ -1189,7 +1200,7 @@ YY_ACTION(void) yy_1_op_gte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gte\n")); { -#line 276 +#line 287 yy->ctx.mn.op = OP_GTE; } #undef yythunkpos @@ -1203,7 +1214,7 @@ YY_ACTION(void) yy_1_op_sne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_sne\n")); { -#line 275 +#line 286 yy->ctx.mn.op = OP_NRE; } #undef yythunkpos @@ -1217,7 +1228,7 @@ YY_ACTION(void) yy_1_op_seq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_seq\n")); { -#line 274 +#line 285 yy->ctx.mn.op = OP_RE; } #undef yythunkpos @@ -1231,7 +1242,7 @@ YY_ACTION(void) yy_1_op_ne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_ne\n")); { -#line 273 +#line 284 yy->ctx.mn.op = OP_NE; } #undef yythunkpos @@ -1245,7 +1256,7 @@ YY_ACTION(void) yy_1_op_eq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_eq\n")); { -#line 272 +#line 283 yy->ctx.mn.op = OP_EQ; } #undef yythunkpos @@ -1259,7 +1270,7 @@ YY_ACTION(void) yy_1_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_test\n")); { -#line 270 +#line 281 push_output(&yy->ctx, &yy->ctx.mn); } #undef yythunkpos @@ -1273,7 +1284,7 @@ YY_ACTION(void) yy_1_match(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_match\n")); { -#line 261 +#line 272 pop_all_ops(&yy->ctx); } #undef yythunkpos @@ -2248,64 +2259,183 @@ YY_PARSE(yycontext *) YYRELEASE(yycontext *yyctx) } #endif -#line 371 "heka_message_matcher_parser.leg" - +#line 382 "heka_message_matcher_parser.leg" -lsb_message_matcher* lsb_create_message_matcher(const char *exp) +static match_node* copy_node(unsigned char parent, match_node *mn, + match_node_tmp *mnt) { - if (!exp) { - return NULL; + mn->op = mnt->op; + mn->val_len = 0; + mn->var_len = 0; + mn->field_id = mnt->id; + mn->val_mod = mnt->val_mod; + mn->val_type = mnt->val_type; + + if (mn->op == OP_AND || mn->op == OP_OR) { + mn->u.off = parent; + } else { + mn->u.idx.f = mnt->fi; + mn->u.idx.a = mnt->ai; + } + + if (mnt->id == LSB_PB_FIELDS) { + mn->var_len = mnt->var_len; + } + if (mn->var_len) { + memcpy(mn->data, mnt->var, mn->var_len); // no NUL terminator + free(mnt->var); + mnt->var = NULL; + mnt->var_len = 0; + } + + size_t val_len = 0; + switch (mnt->val_type) { + case TYPE_STRING: + val_len = mnt->val_len + 1; + memcpy(mn->data + mn->var_len, mnt->val.s, val_len); + free(mnt->val.s); + mnt->val.s = NULL; + mnt->val_len = 0; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + memcpy(mn->data + mn->var_len, &mnt->val.d, val_len); + break; + default: + break; + } + + mn->units = 1 + ((sizeof(match_node) - 1 + mn->var_len + val_len) + / sizeof(match_node)); + if (val_len && mnt->val_type == TYPE_STRING) { + mn->val_len = val_len - 1; + } else { + mn->val_len = val_len; } + return mn + mn->units; +} - lsb_message_matcher *mm = NULL; - yycontext yy; - memset(&yy, 0, sizeof(yy)); - yy.ctx.is.s = exp; - yy.ctx.is.size = strlen(exp); - int ret = yyparse(&yy); - if (!ret) { - free(yy.ctx.out.a); - goto cleanup; + +static match_node* inorder_traverse(unsigned char parent, match_node **offsets, + match_node *mn, match_node_tmp *root, + match_node_tmp *mnt) +{ + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + unsigned char idx = mnt->val_len; + mnt->val_len = 0; + mn = inorder_traverse(idx, offsets, mn, root, root + mnt->fi); + offsets[idx] = mn; + mn = copy_node(parent, mn, mnt); + } else { + mn = copy_node(parent, mn, mnt); + } + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + mn = inorder_traverse(parent, offsets, mn, root, root + mnt->ai); } + return mn; +} - // reverse the order so the root node will be first - match_node *s = yy.ctx.out.a; - match_node *e = yy.ctx.out.a + yy.ctx.out.pos; - for (--e; s < e; ++s, --e) { - move_match_node(&yy.ctx.mn, s); - move_match_node(s, e); - move_match_node(e, &yy.ctx.mn); + +static size_t get_matcher_bytes(match_node_tmp nodes[], size_t size) +{ + size_t len = 0; + for (unsigned i = 0; i < size; ++i) { + size_t val_len = 0; + switch (nodes[i].val_type) { + case TYPE_STRING: + val_len = nodes[i].val_len + 1; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + break; + default: + break; + } + + size_t var_len = 0; + if (nodes[i].id == LSB_PB_FIELDS) { + var_len = nodes[i].var_len; + } + + len += (sizeof(match_node) * 2 + val_len + var_len - 1) + / sizeof(match_node) * sizeof(match_node); + + if (nodes[i].op == OP_OR || nodes[i].op == OP_AND) { + // squirrel away the position for the short-circuit calculation + nodes[i].val_len = i; + } } + return len; +} - mm = malloc(sizeof(lsb_message_matcher)); - if (!mm) { - goto cleanup; + +static void make_tree(match_node_tmp nodes[], size_t size) +{ + // turn the postfix stack into a traversable tree + match_node_tmp *stack[size]; + memset(stack, 0, sizeof(stack)); + int top = 0; + for (unsigned i = 0; i < size; ++i) { + if (nodes[i].op != OP_AND && nodes[i].op != OP_OR) { + stack[top++] = &nodes[i]; + } else { + nodes[i].ai = stack[--top] - nodes; + nodes[i].fi = stack[--top] - nodes; + stack[top++] = &nodes[i]; + } } - mm->nodes = yy.ctx.out.a; - mm->size = yy.ctx.out.pos; +} + + +static lsb_message_matcher* make_matcher(match_node_tmp nodes[], size_t size) +{ + lsb_message_matcher *mm = malloc(sizeof(lsb_message_matcher)); + if (!mm) { return NULL; } - // turn the postfix stack into an executable tree - match_node **stack = calloc(sizeof(match_node *) * mm->size, 1); - if (!stack) { + mm->bytes = get_matcher_bytes(nodes, size); + mm->nodes = calloc(mm->bytes, 1); + if (!mm->nodes) { free(mm); - mm = NULL; - goto cleanup; + return NULL; } - int top = 0; - for (int i = mm->size - 1; i >= 0; --i) { - if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { - stack[top++] = &mm->nodes[i]; - } else { - mm->nodes[i].ai = stack[--top] - mm->nodes; - mm->nodes[i].fi = stack[--top] - mm->nodes; - stack[top++] = &mm->nodes[i]; + match_node *offsets[size]; + memset(offsets, 0, sizeof(offsets)); + + inorder_traverse(size, offsets, mm->nodes, nodes, nodes + (size - 1)); + + // populate the short-circuit offsets + match_node *e = mm->nodes + (mm->bytes / sizeof(match_node)); + for (match_node *p = mm->nodes; p < e;){ + if (p->op == OP_AND || p->op == OP_OR) { + if (p->u.off < size) { + p->u.off = offsets[p->u.off] - mm->nodes; + } else { + p->u.off = mm->bytes / sizeof(match_node); + } } + p += p->units; } - free(stack); + return mm; +} + + +lsb_message_matcher* lsb_create_message_matcher(const char *exp) +{ + if (!exp) { return NULL; } -cleanup: + lsb_message_matcher *mm = NULL; + yycontext yy; + memset(&yy, 0, sizeof(yy)); + yy.ctx.is.s = exp; + yy.ctx.is.size = strlen(exp); + int ret = yyparse(&yy); + if (ret) { + make_tree(yy.ctx.out.a, yy.ctx.out.pos); + mm = make_matcher(yy.ctx.out.a, yy.ctx.out.pos); + } + free(yy.ctx.out.a); free(yy.ctx.ops.a); yyrelease(&yy); return mm; diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index 9f02360..be76063 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -24,10 +24,28 @@ #pragma warning( disable : 4267 4244 ) #endif +typedef struct match_node_tmp { + uint8_t id; + uint8_t op; + uint8_t val_mod; + uint8_t val_type; + uint8_t val_len; + uint8_t var_len; + uint8_t fi; // left node index for logical op + uint8_t ai; // right node index for logical op + char *var; + + union { + char *s; + double d; + } val; +} match_node_tmp; + + typedef struct match_node_array { - match_node *a; - int pos; - int size; + match_node_tmp *a; + int pos; + int size; } match_node_array; @@ -41,7 +59,7 @@ typedef struct input_string { typedef struct context { match_node_array out; match_node_array ops; - match_node mn; + match_node_tmp mn; struct tm tms; int cond_cnt; input_string is; @@ -70,23 +88,23 @@ typedef struct context { } -static void init_match_node(match_node *mn) +static void init_match_node(match_node_tmp *mn) { - memset(mn, 0, sizeof(match_node)); + memset(mn, 0, sizeof(match_node_tmp)); } -static void move_match_node(match_node *dest, match_node *src) +static void move_match_node(match_node_tmp *dest, match_node_tmp *src) { - memcpy(dest, src, sizeof(match_node)); + memcpy(dest, src, sizeof(match_node_tmp)); init_match_node(src); // dest now owns the memory, wipe the pointers } static void realloc_mna(match_node_array *mna) { - size_t bytes = sizeof(match_node) * ++mna->size; - match_node *tmp = realloc(mna->a, bytes); + size_t bytes = sizeof(match_node_tmp) * ++mna->size; + match_node_tmp *tmp = realloc(mna->a, bytes); if (tmp) { mna->a = tmp; init_match_node(&mna->a[mna->size - 1]); @@ -97,7 +115,7 @@ static void realloc_mna(match_node_array *mna) } -static void push_output(context *ctx, match_node *mn) +static void push_output(context *ctx, match_node_tmp *mn) { if (!ctx->out.a || ctx->out.pos == ctx->out.size) { realloc_mna(&ctx->out); @@ -129,7 +147,7 @@ static void push_op(context *ctx, match_operation op) static void pop_to_paren(context *ctx) { for (; ctx->ops.pos > 0; --ctx->ops.pos) { - match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; if (op->op == OP_OPEN) break; push_output(ctx, op); } @@ -139,7 +157,7 @@ static void pop_to_paren(context *ctx) static void pop_all_ops(context *ctx) { for (; ctx->ops.pos > 0; --ctx->ops.pos) { - match_node *op = &ctx->ops.a[ctx->ops.pos - 1]; + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; if (op->op == OP_OPEN) continue; push_output(ctx, op); } @@ -165,30 +183,30 @@ static void update_time(context *ctx, int hour, int minute, int sec) static void update_offset(context *ctx, char sign, int hour, int minute) { - ctx->mn.value.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); + ctx->mn.val.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); } static void set_field(context *ctx, char *name) { ctx->mn.id = LSB_PB_FIELDS; - ctx->mn.variable_len = strlen(name); - ctx->mn.variable = malloc(ctx->mn.variable_len + 1); - if (!ctx->mn.variable) { + ctx->mn.var_len = strlen(name); + ctx->mn.var = malloc(ctx->mn.var_len + 1); + if (!ctx->mn.var) { fprintf(stderr, "malloc failed\n"); exit(1); } - memcpy(ctx->mn.variable, name, ctx->mn.variable_len + 1); + memcpy(ctx->mn.var, name, ctx->mn.var_len + 1); } static void set_timestamp(context *ctx) { ctx->mn.id = LSB_PB_TIMESTAMP; - ctx->mn.value_type = TYPE_NUMERIC; + ctx->mn.val_type = TYPE_NUMERIC; if (ctx->tms.tm_isdst == -1) { - ctx->mn.value.d += mktime(&ctx->tms); - ctx->mn.value.d *= 1e9; + ctx->mn.val.d += mktime(&ctx->tms); + ctx->mn.val.d *= 1e9; } memset(&ctx->tms, 0, sizeof(struct tm)); } @@ -196,14 +214,14 @@ static void set_timestamp(context *ctx) static void set_numeric_value(context *ctx, char *s) { - ctx->mn.value_type = TYPE_NUMERIC; - ctx->mn.value.d = strtod(s, NULL); + ctx->mn.val_type = TYPE_NUMERIC; + ctx->mn.val.d = strtod(s, NULL); } static void set_string_value(context *ctx, char *s) { - ctx->mn.value_type = TYPE_STRING; + ctx->mn.val_type = TYPE_STRING; int i, j; for (i = 0, j = 0; s[i]; ++i, ++j) { if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { @@ -213,21 +231,21 @@ static void set_string_value(context *ctx, char *s) } s[j] = 0; - ctx->mn.value_len = j; - ctx->mn.value.s = malloc(j + 1); - if (!ctx->mn.value.s) { + ctx->mn.val_len = j; + ctx->mn.val.s = malloc(j + 1); + if (!ctx->mn.val.s) { fprintf(stderr, "malloc failed\n"); exit(1); } - memcpy(ctx->mn.value.s, s, j + 1); + memcpy(ctx->mn.val.s, s, j + 1); } static void set_match_mod(context *ctx) { - if (ctx->mn.value_mod == '\0' - && strpbrk(ctx->mn.value.s, "^$*+?.[%-") == NULL) { // literal - ctx->mn.value_mod = '%'; + if (ctx->mn.val_mod == PATTERN_MOD_NONE + && strpbrk(ctx->mn.val.s, "^$*+?.[%-") == NULL) { // literal + ctx->mn.val_mod = PATTERN_MOD_ESC; } } @@ -244,13 +262,6 @@ static bool check_string_len(char *s) } -static void set_boolean_value(context *ctx, bool b) -{ - ctx->mn.value_type = TYPE_BOOLEAN; - ctx->mn.value.d = b; -} - - static int cond_cnt(context *ctx) { return (++ctx->cond_cnt * 2 + 1 > UCHAR_MAX) ? 0 : 1; @@ -287,8 +298,8 @@ relational = op_eq boolean_test = true {yy->ctx.mn.op = OP_TRUE} | false {yy->ctx.mn.op = OP_FALSE} -boolean = true {set_boolean_value(&yy->ctx, 1)} - | false {set_boolean_value(&yy->ctx, 0)} +boolean = true {yy->ctx.mn.val_type = TYPE_TRUE} + | false {yy->ctx.mn.val_type = TYPE_FALSE} and = "&&" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_AND)} sp or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp @@ -310,7 +321,7 @@ string_value = ( '"' < ('\\\"' | (!'"' .))* > '"' string_match = (op_seq | op_sne) sp string_value string_match_mod? {set_match_mod(&yy->ctx)} -string_match_mod = "%" {yy->ctx.mn.value_mod = '%'} +string_match_mod = "%" {yy->ctx.mn.val_mod = PATTERN_MOD_ESC} numeric_headers = "Severity" {yy->ctx.mn.id = LSB_PB_SEVERITY} sp | "Pid" {yy->ctx.mn.id = LSB_PB_PID} sp @@ -360,9 +371,9 @@ second = ( < [0-5] [0-9] > | < "60" > ) {$$ = atoi(yytext)} -second_frac = < decimal > {yy->ctx.mn.value.d += strtod(yytext, NULL)} +second_frac = < decimal > {yy->ctx.mn.val.d += strtod(yytext, NULL)} -nil = "NIL" {yy->ctx.mn.value_type = TYPE_NIL;} sp +nil = "NIL" {yy->ctx.mn.val_type = TYPE_NIL;} sp true = "TRUE" sp false = "FALSE" sp sp = [ \t]* @@ -370,61 +381,180 @@ eol = !. %% - -lsb_message_matcher* lsb_create_message_matcher(const char *exp) +static match_node* copy_node(unsigned char parent, match_node *mn, + match_node_tmp *mnt) { - if (!exp) { - return NULL; + mn->op = mnt->op; + mn->val_len = 0; + mn->var_len = 0; + mn->field_id = mnt->id; + mn->val_mod = mnt->val_mod; + mn->val_type = mnt->val_type; + + if (mn->op == OP_AND || mn->op == OP_OR) { + mn->u.off = parent; + } else { + mn->u.idx.f = mnt->fi; + mn->u.idx.a = mnt->ai; } - lsb_message_matcher *mm = NULL; - yycontext yy; - memset(&yy, 0, sizeof(yy)); - yy.ctx.is.s = exp; - yy.ctx.is.size = strlen(exp); - int ret = yyparse(&yy); - if (!ret) { - free(yy.ctx.out.a); - goto cleanup; + if (mnt->id == LSB_PB_FIELDS) { + mn->var_len = mnt->var_len; + } + if (mn->var_len) { + memcpy(mn->data, mnt->var, mn->var_len); // no NUL terminator + free(mnt->var); + mnt->var = NULL; + mnt->var_len = 0; } - // reverse the order so the root node will be first - match_node *s = yy.ctx.out.a; - match_node *e = yy.ctx.out.a + yy.ctx.out.pos; - for (--e; s < e; ++s, --e) { - move_match_node(&yy.ctx.mn, s); - move_match_node(s, e); - move_match_node(e, &yy.ctx.mn); + size_t val_len = 0; + switch (mnt->val_type) { + case TYPE_STRING: + val_len = mnt->val_len + 1; + memcpy(mn->data + mn->var_len, mnt->val.s, val_len); + free(mnt->val.s); + mnt->val.s = NULL; + mnt->val_len = 0; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + memcpy(mn->data + mn->var_len, &mnt->val.d, val_len); + break; + default: + break; } - mm = malloc(sizeof(lsb_message_matcher)); - if (!mm) { - goto cleanup; + mn->units = 1 + ((sizeof(match_node) - 1 + mn->var_len + val_len) + / sizeof(match_node)); + if (val_len && mnt->val_type == TYPE_STRING) { + mn->val_len = val_len - 1; + } else { + mn->val_len = val_len; } - mm->nodes = yy.ctx.out.a; - mm->size = yy.ctx.out.pos; + return mn + mn->units; +} - // turn the postfix stack into an executable tree - match_node **stack = calloc(sizeof(match_node *) * mm->size, 1); - if (!stack) { - free(mm); - mm = NULL; - goto cleanup; + +static match_node* inorder_traverse(unsigned char parent, match_node **offsets, + match_node *mn, match_node_tmp *root, + match_node_tmp *mnt) +{ + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + unsigned char idx = mnt->val_len; + mnt->val_len = 0; + mn = inorder_traverse(idx, offsets, mn, root, root + mnt->fi); + offsets[idx] = mn; + mn = copy_node(parent, mn, mnt); + } else { + mn = copy_node(parent, mn, mnt); + } + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + mn = inorder_traverse(parent, offsets, mn, root, root + mnt->ai); + } + return mn; +} + + +static size_t get_matcher_bytes(match_node_tmp nodes[], size_t size) +{ + size_t len = 0; + for (unsigned i = 0; i < size; ++i) { + size_t val_len = 0; + switch (nodes[i].val_type) { + case TYPE_STRING: + val_len = nodes[i].val_len + 1; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + break; + default: + break; + } + + size_t var_len = 0; + if (nodes[i].id == LSB_PB_FIELDS) { + var_len = nodes[i].var_len; + } + + len += (sizeof(match_node) * 2 + val_len + var_len - 1) + / sizeof(match_node) * sizeof(match_node); + + if (nodes[i].op == OP_OR || nodes[i].op == OP_AND) { + // squirrel away the position for the short-circuit calculation + nodes[i].val_len = i; + } } + return len; +} + +static void make_tree(match_node_tmp nodes[], size_t size) +{ + // turn the postfix stack into a traversable tree + match_node_tmp *stack[size]; + memset(stack, 0, sizeof(stack)); int top = 0; - for (int i = mm->size - 1; i >= 0; --i) { - if (mm->nodes[i].op != OP_AND && mm->nodes[i].op != OP_OR) { - stack[top++] = &mm->nodes[i]; + for (unsigned i = 0; i < size; ++i) { + if (nodes[i].op != OP_AND && nodes[i].op != OP_OR) { + stack[top++] = &nodes[i]; } else { - mm->nodes[i].ai = stack[--top] - mm->nodes; - mm->nodes[i].fi = stack[--top] - mm->nodes; - stack[top++] = &mm->nodes[i]; + nodes[i].ai = stack[--top] - nodes; + nodes[i].fi = stack[--top] - nodes; + stack[top++] = &nodes[i]; + } + } +} + + +static lsb_message_matcher* make_matcher(match_node_tmp nodes[], size_t size) +{ + lsb_message_matcher *mm = malloc(sizeof(lsb_message_matcher)); + if (!mm) { return NULL; } + + mm->bytes = get_matcher_bytes(nodes, size); + mm->nodes = calloc(mm->bytes, 1); + if (!mm->nodes) { + free(mm); + return NULL; + } + + match_node *offsets[size]; + memset(offsets, 0, sizeof(offsets)); + + inorder_traverse(size, offsets, mm->nodes, nodes, nodes + (size - 1)); + + // populate the short-circuit offsets + match_node *e = mm->nodes + (mm->bytes / sizeof(match_node)); + for (match_node *p = mm->nodes; p < e;){ + if (p->op == OP_AND || p->op == OP_OR) { + if (p->u.off < size) { + p->u.off = offsets[p->u.off] - mm->nodes; + } else { + p->u.off = mm->bytes / sizeof(match_node); + } } + p += p->units; } - free(stack); + return mm; +} + + +lsb_message_matcher* lsb_create_message_matcher(const char *exp) +{ + if (!exp) { return NULL; } -cleanup: + lsb_message_matcher *mm = NULL; + yycontext yy; + memset(&yy, 0, sizeof(yy)); + yy.ctx.is.s = exp; + yy.ctx.is.size = strlen(exp); + int ret = yyparse(&yy); + if (ret) { + make_tree(yy.ctx.out.a, yy.ctx.out.pos); + mm = make_matcher(yy.ctx.out.a, yy.ctx.out.pos); + } + free(yy.ctx.out.a); free(yy.ctx.ops.a); yyrelease(&yy); return mm; diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 88faacd..86b6c5d 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -247,6 +247,7 @@ static char* test_malformed_matcher() return NULL; } + static char* benchmark_matcher_create() { int iter = 100000; @@ -264,11 +265,53 @@ static char* benchmark_matcher_create() return NULL; } + +static char* benchmark_match_hs() +{ + // see what a single sample looks like for a better comparison with Hindsight + char *tests[] = { + "TRUE" + , "Type == 'TEST' && Severity == 6" + , "Fields[foo] == 'bar' && Severity == 6" + , "Fields[number] == 64 && Severity == 6" + , "Fields[missing] == NIL" + , "Fields[int] != NIL" + , "Type =~ '^[Tt]EST' && Severity == 6" + , "Payload =~ '^Test'" + , "Payload =~ 'item$'" + , "Payload =~ 'unique%-item'" + , "Payload =~ 'unique-item'%" + , "Payload =~ 'unique'" + , "Payload =~ 'unique'%" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + + unsigned long long start, end; + for (int i = 0; tests[i]; i++) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "lsb_create_message_matcher failed: %s", tests[i]); + start = lsb_get_time(); + mu_assert(lsb_eval_message_matcher(mm, &m), + "lsb_eval_message_matcher failed"); + end = lsb_get_time(); + lsb_destroy_message_matcher(mm); + printf("matcher (lsb_get_time): '%s': %g\n", tests[i], + (double)(end - start) / 1e9); + } + lsb_free_heka_message(&m); + return NULL; +} + + static char* benchmark_match() { int iter = 1000000; char *tests[] = { - "Type == 'TEST' && Severity == 6" + "TRUE" + , "Type == 'TEST' && Severity == 6" , "Fields[foo] == 'bar' && Severity == 6" , "Fields[number] == 64 && Severity == 6" , "Fields[missing] == NIL" @@ -312,6 +355,7 @@ static char* all_tests() mu_run_test(test_false_matcher); mu_run_test(test_malformed_matcher); + mu_run_test(benchmark_match_hs); mu_run_test(benchmark_matcher_create); mu_run_test(benchmark_match); return NULL; From 5569eee61f31b108eeb8b85a5c8f17167935021c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 22 Nov 2017 16:05:08 -0800 Subject: [PATCH 213/235] Fix the out of bounds NUL terminator write in lsb_outputfd --- src/util/output_buffer.c | 2 +- src/util/test/test_output_buffer.c | 66 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c index 3bede46..8eff264 100644 --- a/src/util/output_buffer.c +++ b/src/util/output_buffer.c @@ -231,7 +231,7 @@ lsb_err_value lsb_outputfd(lsb_output_buffer *b, double d) number /= 10; } while (number > 0); - lsb_err_value ret = lsb_expand_output_buffer(b, (p - buffer) + negative); + lsb_err_value ret = lsb_expand_output_buffer(b, (p - buffer) + negative + 1); if (!ret) { if (negative) { b->buf[b->pos++] = '-'; diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c index b79f732..c1b9bb9 100644 --- a/src/util/test/test_output_buffer.c +++ b/src/util/test/test_output_buffer.c @@ -185,6 +185,66 @@ static char* test_outputd() } +static char* test_outputc_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputc(&b, 'a'); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputf_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputf(&b, "%s", "f"); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputs_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputs(&b, "s", 1); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputd_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputd(&b, 1); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + static char* all_tests() { mu_run_test(test_stub); @@ -197,6 +257,12 @@ static char* all_tests() mu_run_test(test_outputf); mu_run_test(test_outputs); mu_run_test(test_outputd); + + // make sure the terminating NUL is included in the size of the buffer + mu_run_test(test_outputc_full); + mu_run_test(test_outputf_full); + mu_run_test(test_outputs_full); + mu_run_test(test_outputd_full); return NULL; } From 6a8c0fda1f3b18f1de3dc934dcd4f4bc841ab57c Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 3 Jan 2018 11:11:52 -0800 Subject: [PATCH 214/235] Add TOC generation to the documentation --- gen_gh_pages.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua index 57b2825..aff0b99 100644 --- a/gen_gh_pages.lua +++ b/gen_gh_pages.lua @@ -32,6 +32,11 @@ local function main() local rv = os.execute(string.format("rsync -rav docs/ %s/", output_dir)) if rv ~= 0 then error"rsync setup" end + local fh = assert(io.open(string.format("%s/book.json", output_dir), "w")) + fh:write([[{"plugins" : ["collapsible-menu", "navigator"]}]]) + fh:close() + + os.execute(string.format("cd %s;gitbook install", output_dir)) os.execute(string.format("mv %s/index.md %s/README.md", output_dir, output_dir)) output_menu(output_dir, args[1]) os.execute(string.format("gitbook build %s", output_dir)) From 053a32744d1b492d2c908080199283bd1edcbb0b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 16 Jan 2018 09:30:19 -0800 Subject: [PATCH 215/235] Update cmake for the ubuntu:latest TravisCI build --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 14eaedc..e799383 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: - DOCKER_IMAGE=debian:stretch - DOCKER_IMAGE=fedora:latest - DOCKER_IMAGE=ubuntu:12.04 CMAKE_URL=auto - - DOCKER_IMAGE=ubuntu:latest # LTS + - DOCKER_IMAGE=ubuntu:latest CMAKE_URL=auto # LTS - DOCKER_IMAGE=ubuntu:devel script: From 26381fa398dc3ebe3c240f24dd1a93f15bb6a6db Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 23 Jan 2018 10:23:59 -0800 Subject: [PATCH 216/235] Prevent dynamic messages from being passed as the log fmt specifier --- .travis.yml | 4 ++-- CMakeLists.txt | 4 ++-- src/heka/sandbox.c | 6 +++--- src/util/heka_message.c | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index e799383..1f1a0dd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ env: - DOCKER_IMAGE=centos:6 CMAKE_URL=auto - DOCKER_IMAGE=centos:7 CMAKE_URL=auto - DOCKER_IMAGE=debian:7 CMAKE_URL=auto - - DOCKER_IMAGE=debian:8 - - DOCKER_IMAGE=debian:8 CC=clang + - DOCKER_IMAGE=debian:8 CMAKE_URL=auto + - DOCKER_IMAGE=debian:8 CC=clang CMAKE_URL=auto - DOCKER_IMAGE=debian:stretch - DOCKER_IMAGE=fedora:latest - DOCKER_IMAGE=ubuntu:12.04 CMAKE_URL=auto diff --git a/CMakeLists.txt b/CMakeLists.txt index c71e32a..add17bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,8 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -cmake_minimum_required(VERSION 3.0 FATAL_ERROR) -project(luasandbox VERSION 1.2.8 LANGUAGES C) +cmake_minimum_required(VERSION 3.6 FATAL_ERROR) +project(luasandbox VERSION 1.2.9 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index d8ad626..436b74d 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -513,7 +513,7 @@ lsb_heka_sandbox* lsb_heka_create_input(void *parent, if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { - logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); + logger->cb(logger->context, hsb->name, 3, "%s", lsb_get_error(hsb->lsb)); } lsb_destroy(hsb->lsb); free(hsb->hostname); @@ -710,7 +710,7 @@ lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { - logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); + logger->cb(logger->context, hsb->name, 3, "%s", lsb_get_error(hsb->lsb)); } lsb_destroy(hsb->lsb); free(hsb->hostname); @@ -929,7 +929,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, if (lsb_init(hsb->lsb, state_file)) { if (logger && logger->cb) { - logger->cb(logger->context, hsb->name, 3, lsb_get_error(hsb->lsb)); + logger->cb(logger->context, hsb->name, 3, "%s", lsb_get_error(hsb->lsb)); } lsb_destroy(hsb->lsb); free(hsb->hostname); diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 64398e6..fdae5ff 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -215,7 +215,7 @@ bool lsb_decode_heka_message(lsb_heka_message *m, { if (!m || !buf || len == 0) { if (logger && logger->cb) { - logger->cb(logger->context, __func__, 4, LSB_ERR_UTIL_NULL); + logger->cb(logger->context, __func__, 4, "%s", LSB_ERR_UTIL_NULL); } return false; } @@ -315,14 +315,14 @@ bool lsb_decode_heka_message(lsb_heka_message *m, if (!m->uuid.s) { if (logger && logger->cb) { - logger->cb(logger->context, __func__, 4, "missing " LSB_UUID); + logger->cb(logger->context, __func__, 4, "%s", "missing " LSB_UUID); } return false; } if (!timestamp) { if (logger && logger->cb) { - logger->cb(logger->context, __func__, 4, "missing " LSB_TIMESTAMP); + logger->cb(logger->context, __func__, 4, "%s", "missing " LSB_TIMESTAMP); } return false; } @@ -341,7 +341,7 @@ bool lsb_find_heka_message(lsb_heka_message *m, { if (!m || !ib || !discarded_bytes) { if (logger && logger->cb) { - logger->cb(logger->context, __func__, 4, LSB_ERR_UTIL_NULL); + logger->cb(logger->context, __func__, 4, "%s", LSB_ERR_UTIL_NULL); } return false; } From 7df9319a4adb6265e18a9f4533ffa4e090a715e6 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 5 Mar 2018 06:55:54 -0800 Subject: [PATCH 217/235] Update build for the downstream integration tests - Pass the docker image name along for distro specific setup --- build/functions.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/functions.sh b/build/functions.sh index 9289ac7..4ea418a 100755 --- a/build/functions.sh +++ b/build/functions.sh @@ -21,6 +21,7 @@ usage() { echo " - CXX: C++ compiler (default: g++)" >&2 } + # ================================================================= # Run command within container docker_run() { @@ -38,6 +39,7 @@ docker_run() { --env "CMAKE_SHA256=${CMAKE_SHA256}" \ --env "CC=${CC}" \ --env "CXX=${CXX}" \ + --env "DISTRO=${DOCKER_IMAGE}" \ "$DOCKER_IMAGE" \ $@ ) From 57f5db42f294a5f6f2495726d8a002a76dcd3f09 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 3 Apr 2018 13:52:11 -0700 Subject: [PATCH 218/235] Issue 221 - Fix possible out of order field array encoding --- CMakeLists.txt | 2 +- src/heka/message.c | 37 ++++++++++++++++++------------------- 2 files changed, 19 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index add17bd..da39c53 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.2.9 LANGUAGES C) +project(luasandbox VERSION 1.2.10 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/heka/message.c b/src/heka/message.c index ef8b96b..4b02814 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -302,9 +302,12 @@ encode_int(lua_State *lua, lsb_output_buffer *ob, char tag, const char *name, * @param lsb Pointer to the sandbox. * @param lua Pointer to the lua_State. * @param ob Pointer to the output data buffer. - * @param first Boolean indicator used to add addition protobuf data in the - * correct order. - * @param representation String representation of the field i.e., "ms" + * @param first Flag set on the first field value to add + * additional protobuf data in the correct order. + * In the case of arrays the value should contain + * the number of items in the array. + * @param representation String representation of the field + * i.e., "ms" * @param value_type Protobuf value type * * @return lsb_err_value NULL on success error message on failure @@ -329,26 +332,21 @@ static lsb_err_value encode_field_array(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, int t, const char *representation, int value_type) { - lsb_err_value ret = NULL; - int first = (int)lua_objlen(lua, -1); - int multiple = first > 1 ? first : 0; - size_t len_pos = 0; - lua_checkstack(lua, 2); - lua_pushnil(lua); - while (!ret && lua_next(lua, -2) != 0) { + int alen = (int)lua_objlen(lua, -2); + lsb_err_value ret = encode_field_value(lsb, lua, ob, alen, representation, + value_type); + size_t len_pos = ob->pos; + lua_pop(lua, 1); + for (int idx = 2; !ret && idx <= alen; ++idx) { + lua_rawgeti(lua, -1, idx); if (lua_type(lua, -1) != t) { snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); return LSB_ERR_HEKA_INPUT; } - ret = encode_field_value(lsb, lua, ob, first, representation, value_type); - if (first) { - len_pos = ob->pos; - first = 0; - } - lua_pop(lua, 1); // Remove the value leaving the key on top for - // the next interation. + ret = encode_field_value(lsb, lua, ob, 0, representation, value_type); + lua_pop(lua, 1); } - if (!ret && multiple && value_type == LSB_PB_INTEGER) { + if (!ret && alen > 1 && value_type == LSB_PB_INTEGER) { // fix up the varint packed length size_t i = len_pos - 2; int y = 0; @@ -523,9 +521,9 @@ encode_field_value(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, { lua_rawgeti(lua, -1, 1); int t = lua_type(lua, -1); - lua_pop(lua, 1); // remove the array test value switch (t) { case LUA_TNIL: + lua_pop(lua, 1); // remove the array test value ret = encode_field_object(lsb, lua, ob); break; case LUA_TNUMBER: @@ -534,6 +532,7 @@ encode_field_value(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, ret = encode_field_array(lsb, lua, ob, t, representation, value_type); break; default: + lua_pop(lua, 1); // remove the array test value snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported array type: %s", lua_typename(lua, t)); return LSB_ERR_LUA; From 7af1c9c883e0aa0089196224fbd595e045dbc6b3 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 19 Apr 2018 07:19:44 -0700 Subject: [PATCH 219/235] Add Uuid, Timestamp and Pid as restricted headers --- CMakeLists.txt | 2 +- docs/heka/analysis.md | 12 +++--- docs/heka/input.md | 10 ++--- docs/heka/message.md | 10 ++--- src/heka/message.c | 47 +++++++++++++---------- src/heka/sandbox.c | 4 ++ src/heka/sandbox_impl.h | 1 + src/heka/test/lua/aim.lua | 4 +- src/heka/test/lua/encode_message.lua | 24 ++++++------ src/heka/test/test_heka_sandbox.c | 57 ++++++++++++---------------- 10 files changed, 89 insertions(+), 82 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index da39c53..fe76408 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.2.10 LANGUAGES C) +project(luasandbox VERSION 1.2.11 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md index d38481b..bae001a 100644 --- a/docs/heka/analysis.md +++ b/docs/heka/analysis.md @@ -117,12 +117,12 @@ message especially when using an LPeg grammar transformation. ### inject_message Creates a new Heka protocol buffer message using the contents of the specified -Lua table (overwriting whatever is in the payload buffer). `Logger` and -`Hostname` are restricted header values. An override configuration option is -provided `restricted_headers`; when true (default) the headers are always set -to the configuration values; when false the headers are set to the values -provide in the message table, if no value is provided it defaults to the -appropriate configuration value. +Lua table (overwriting whatever is in the payload buffer). `Timestamp`, +`Logger`, `Hostname` and `Pid` are restricted header values. An override +configuration option is provided `restricted_headers`; when true (default) the +headers are always set to the configuration values; when false the headers are +set to the values provided in the message table, if no value is provided it +defaults to the appropriate value. *Arguments* * msg ([Heka message table](message.md)) diff --git a/docs/heka/input.md b/docs/heka/input.md index b626638..8b33812 100644 --- a/docs/heka/input.md +++ b/docs/heka/input.md @@ -62,11 +62,11 @@ Converts a Heka protobuf encoded message string into a Lua table. See ### inject_message Sends a Heka protocol buffer message into the host. For the Heka message table -argument `Logger` and `Hostname` are restricted header values. An override -configuration option is provided `restricted_headers`; when true the headers are -always set to the configuration values; when false (default) the headers are set -to the values provide in the message table, if no value is provided it defaults -to the appropriate configuration value. +arguments `Timestamp`, `Logger`, `Hostname` and `Pid` are restricted header +values. An override configuration option is provided `restricted_headers`; when +true the headers are always set to the configuration values; when false +(default) the headers are set to the values provide in the message table, +if no value is provided it defaults to the appropriate value. *Arguments* * msg ([Heka message table](message.md), diff --git a/docs/heka/message.md b/docs/heka/message.md index 463e36f..6504c85 100644 --- a/docs/heka/message.md +++ b/docs/heka/message.md @@ -4,14 +4,14 @@ ```lua { -Uuid = "data", -- auto generated if not a 16 byte raw binary UUID or a 36 character human readable UUID -Logger = "nginx", -- defaults to the Logger configuration value but can be overridden with the `restricted_headers` configuration -Hostname = "example.com", -- defaults to the Hostname configuration value but can be overridden with the `restricted_headers` configuration -Timestamp = 1e9, -- auto generated if not a number +Uuid = "data", -- restricted header, auto generated if not a 16 byte raw binary UUID or a 36 character human readable UUID +Logger = "nginx", -- restricted header, defaults to the Logger configuration value +Hostname = "example.com", -- restricted header, defaults to the Hostname configuration value +Timestamp = 1e9, -- restricted header, defaults to the current time Type = "TEST", Payload = "Test Payload", EnvVersion = "0.8", -Pid = 1234, +Pid = 1234, -- restricted header, defaults to the Pid configuration value Severity = 6, Fields = { http_status = 200, -- encoded as a double diff --git a/src/heka/message.c b/src/heka/message.c index 4b02814..fef1a17 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -41,7 +41,15 @@ static void set_missing_headers(lua_State *lua, int idx, lsb_heka_sandbox *hsb) lua_pop(lua, 1); if (t == LUA_TNIL && hsb->hostname) { lua_pushstring(lua, hsb->hostname); - lua_setfield(lua, 1, LSB_HOSTNAME); + lua_setfield(lua, idx, LSB_HOSTNAME); + } + + lua_getfield(lua, idx, LSB_PID); + t = lua_type(lua, -1); + lua_pop(lua, 1); + if (t == LUA_TNIL) { + lua_pushinteger(lua, hsb->pid); + lua_setfield(lua, idx, LSB_PID); } } @@ -894,35 +902,36 @@ int heka_encode_message(lua_State *lua) lsb_err_value heka_encode_message_table(lsb_lua_sandbox *lsb, lua_State *lua, int idx) { + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); lsb_err_value ret = NULL; lsb_output_buffer *ob = &lsb->output; ob->pos = 0; - // use existing or create a type 4 uuid - lua_getfield(lua, idx, LSB_UUID); - size_t len; - const char *uuid = lua_tolstring(lua, -1, &len); - ret = lsb_write_heka_uuid(ob, uuid, len); - lua_pop(lua, 1); // remove uuid - if (ret) return ret; - - // use existing or create a timestamp - lua_getfield(lua, idx, LSB_TIMESTAMP); long long ts; - if (lua_isnumber(lua, -1)) { - ts = (long long)lua_tonumber(lua, -1); - } else { - ts = lsb_get_timestamp(); - } - lua_pop(lua, 1); // remove timestamp - - lsb_heka_sandbox *hsb = lsb_get_parent(lsb); if (hsb->restricted_headers) { + ret = lsb_write_heka_uuid(ob, NULL, 0); + if (ret) return ret; + ts = lsb_get_timestamp(); lua_pushstring(lua, hsb->name); lua_setfield(lua, idx, LSB_LOGGER); lua_pushstring(lua, hsb->hostname); lua_setfield(lua, idx, LSB_HOSTNAME); + lua_pushinteger(lua, hsb->pid); + lua_setfield(lua, idx, LSB_PID); } else { + lua_getfield(lua, idx, LSB_UUID); + size_t len; + const char *uuid = lua_tolstring(lua, -1, &len); + ret = lsb_write_heka_uuid(ob, uuid, len); + lua_pop(lua, 1); // remove uuid + + lua_getfield(lua, idx, LSB_TIMESTAMP); + if (lua_isnumber(lua, -1)) { + ts = (long long)lua_tonumber(lua, -1); + } else { + ts = lsb_get_timestamp(); + } + lua_pop(lua, 1); // remove timestamp set_missing_headers(lua, idx, hsb); } diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 436b74d..69ad06a 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -448,6 +448,10 @@ static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) } lua_pop(lua, 1); // remove the Hostname + lua_getfield(lua, 1, LSB_PID); + hsb->pid = (int)lua_tointeger(lua, -1); + lua_pop(lua, 1); + lua_getfield(lua, 1, "restricted_headers"); if (lua_type(lua, -1) == LUA_TBOOLEAN) { hsb->restricted_headers = lua_toboolean(lua, -1); diff --git a/src/heka/sandbox_impl.h b/src/heka/sandbox_impl.h index 71413be..c15a740 100644 --- a/src/heka/sandbox_impl.h +++ b/src/heka/sandbox_impl.h @@ -40,6 +40,7 @@ struct lsb_heka_sandbox { struct heka_stats stats; char type; bool restricted_headers; + int pid; }; #endif diff --git a/src/heka/test/lua/aim.lua b/src/heka/test/lua/aim.lua index 0433676..e8a3b46 100644 --- a/src/heka/test/lua/aim.lua +++ b/src/heka/test/lua/aim.lua @@ -6,8 +6,8 @@ require "string" -- Table tests local msgs = { - {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - {Timestamp = 1, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "ignore", Hostname = "spoof"}, + {Pid = 0, Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {Pid = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "ignore", Hostname = "spoof"}, } local err_msgs = { diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua index 4bd2b0e..b116249 100644 --- a/src/heka/test/lua/encode_message.lua +++ b/src/heka/test/lua/encode_message.lua @@ -8,15 +8,15 @@ require "table" local msgs = { { msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl\074\002\sh" + rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl@\000\074\002\sh" }, { msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h"}, - rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\001l\074\001\h" + rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\001l@\000\074\001\h" }, { msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, - rv = "\030\002\008\028\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl\074\002\sh", + rv = "\030\002\008\030\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl@\000\074\002\sh", framed = true, }, { @@ -28,7 +28,7 @@ local msgs = { strings = {"s1","s2","s3"}, bool = true, bools = {true,false,false}}}, - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0\82\10\10\4\98\111\111\108\16\4\64\1\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51" + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108@\0\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0\82\10\10\4\98\111\111\108\16\4\64\1\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51" }, { msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h", @@ -40,31 +40,31 @@ local msgs = { {name = "bool" ,value = true}, {name = "bools" ,value = {true,false,false}}} }, - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51\82\10\10\4\98\111\111\108\16\4\64\1\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0" + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108@\0\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51\82\10\10\4\98\111\111\108\16\4\64\1\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0" }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = "value", value_type = 0, representation = "widget"}}}, -- string - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\20\10\3\107\101\121\26\6\119\105\100\103\101\116\34\5\118\97\108\117\101", + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\20\10\3\107\101\121\26\6\119\105\100\103\101\116\34\5\118\97\108\117\101", }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = "value", value_type = 1, representation = "widget"}}}, -- bytes - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\22\10\3\107\101\121\16\1\26\6\119\105\100\103\101\116\42\5\118\97\108\117\101", + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\22\10\3\107\101\121\16\1\26\6\119\105\100\103\101\116\42\5\118\97\108\117\101", }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = 34, value_type = 2, representation = "widget"}}}, -- int - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\17\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\48\34", + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\17\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\48\34", }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = 34, value_type = 3, representation = "widget"}}}, -- double - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\24\10\3\107\101\121\16\3\26\6\119\105\100\103\101\116\57\0\0\0\0\0\0\65\64", + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\24\10\3\107\101\121\16\3\26\6\119\105\100\103\101\116\57\0\0\0\0\0\0\65\64", }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = true, value_type = 4, representation = "widget"}}}, -- bool - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\17\10\3\107\101\121\16\4\26\6\119\105\100\103\101\116\64\1", + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\17\10\3\107\101\121\16\4\26\6\119\105\100\103\101\116\64\1", }, { msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = {1,2,3}, value_type = 2, representation = "widget"}}}, -- int - rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108\74\2\115\104\82\20\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\50\3\1\2\3", + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\20\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\50\3\1\2\3", }, } @@ -79,7 +79,7 @@ for i, v in ipairs(msgs) do if v.rv ~= rv then --local et = {string.byte(v.rv, 1, -1)} local rt = {string.byte(rv, 1, -1)} - assert(v.rv == rv, string.format("test: %d\received: %s", i, table.concat(rt, " "))) + assert(v.rv == rv, string.format("test: %d received: %s", i, table.concat(rt, " "))) end end diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 2d07362..2e4ca5d 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -56,10 +56,10 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, }; struct im_result results[] = { - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03iim\x4a\x02hn", .pb_len = 29, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03iim\x4a\x02hn", .pb_len = 29, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03iim\x40\x00\x4a\x02hn", .pb_len = 31, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03iim\x40\x00\x4a\x02hn", .pb_len = 31, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x03iim\x4a\x02hn\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 165, .cp_numeric = NAN, .cp_string = "foo.log:123" }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x03iim\x40\x00\x4a\x02hn\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 167, .cp_numeric = NAN, .cp_string = "foo.log:123" }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, @@ -128,6 +128,7 @@ static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, static int aim(void *parent, const char *pb, size_t pb_len) { + static int offset = 28; // skip Uuid and Timestamp static int cnt = 0; struct im_result { const char *pb; @@ -137,12 +138,11 @@ static int aim(void *parent, const char *pb, size_t pb_len) }; struct im_result results[] = { - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x1f\x7d\x09\x9e\xf5\x9d\x40\x1d\xa8\xaf\x6a\xff\xc3\x21\xeb\x42\x10\x80\x88\xe4\xaa\xa0\xa9\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 88, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x5b\x7d\xee\xa0\x02\xbc\x45\xbb\xaf\xa9\xcc\x2c\xdd\x65\xde\x45\x10\x80\x88\xdc\xad\xcd\xbf\xbc\x95\x14\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 110, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03\x61\x69\x6d\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 34, .cp_numeric = NAN, .cp_string = NULL }, - }; + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x8e\xa8\xf3\xde\x88\xb5\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 44, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\xbf\x97\x9c\xcc\xbe\xc2\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x00\x00\x00\x00\x0a\x10\x00\x00", .pb_len = 44, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\xea\x95\xd4\xfc\x7c\x10\x40\x95\xa8\x17\xcb\x56\x26\x91\x8c\x47\x10\xba\xf6\xd5\xf9\xfd\xce\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 90, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\xfd\x49\x92\x77\x02\x37\x4b\xf0\xaf\x86\x6f\x9b\x80\x26\xf4\x35\x10\xaf\xec\x9e\xa4\xd8\xcf\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 112, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x7c\x32\xd6\x23\x98\xe8\x49\x9e\xa2\xe8\x0d\x78\x84\x8e\x75\xb2\x10\xf7\xf5\xdb\x89\x88\xe4\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 0, .cp_numeric = NAN, .cp_string = NULL }, }; // intentionally fail on size to to test the custom return value if (cnt >= (int)(sizeof results / sizeof results[0])) { fprintf(stderr, "tests and results are mis-matched\n"); @@ -160,25 +160,17 @@ static int aim(void *parent, const char *pb, size_t pb_len) return 99; } - if (cnt == 0) { - if (memcmp(pb, results[cnt].pb, pb_len)) { - fprintf(stderr, "test: %d\nexpected: ", cnt); - for (size_t i = 0; i < results[cnt].pb_len; ++i) { - fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); - } - fprintf(stderr, "\nreceived: "); - for (size_t i = 0; i < pb_len; ++i) { - fprintf(stderr, "\\x%02hhx", pb[i]); - } - fprintf(stderr, "\n"); - return 1; + if (memcmp(pb + offset, results[cnt].pb + offset, pb_len - offset)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = offset; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); } - } else { - lsb_heka_message m; - lsb_init_heka_message(&m, 2); - bool rv = lsb_decode_heka_message(&m, pb, pb_len, &logger); - lsb_free_heka_message(&m); - if (!rv) return 1; + fprintf(stderr, "\nreceived: "); + for (size_t i = offset; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; } cnt++; return LSB_HEKA_IM_SUCCESS; @@ -194,7 +186,7 @@ static int aim1(void *parent, const char *pb, size_t pb_len) fprintf(stderr, "pb null set\n"); return 1; } - size_t expected = 306; + size_t expected = 308; if (pb_len != expected) { fprintf(stderr, "pb_len expected: %" PRIuSIZE " received: %" PRIuSIZE "\n", expected, pb_len); @@ -601,7 +593,7 @@ static char* test_im_input() &logger, iim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); - mu_assert(332 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(338 == stats.im_bytes, "received %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_input failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -616,7 +608,7 @@ static char* test_im_analysis() &logger, aim); lsb_heka_stats stats = lsb_heka_get_stats(hsb); mu_assert(4 == stats.im_cnt, "expected %llu", stats.im_cnt); - mu_assert(266 == stats.im_bytes, "expected %llu", stats.im_bytes); + mu_assert(290 == stats.im_bytes, "expected %llu", stats.im_bytes); mu_assert(hsb, "lsb_heka_create_analysis failed"); e = lsb_heka_destroy_sandbox(hsb); return NULL; @@ -630,11 +622,12 @@ static char* test_encode_message() "path = [[" TEST_LUA_PATH "]]\n" "cpath = [[" TEST_LUA_CPATH "]]\n" "Hostname = 'sh'\n" - "Logger = 'sl'\n", + "Logger = 'sl'\n" + "Pid = 0\n", &logger, ucp); mu_assert(hsb, "lsb_heka_create_output failed"); lsb_heka_stats stats = lsb_heka_get_stats(hsb); - mu_assert(162 == stats.out_max, "received %llu", stats.out_max); + mu_assert(164 == stats.out_max, "received %llu", stats.out_max); e = lsb_heka_destroy_sandbox(hsb); return NULL; } From 489c5119d51f3821c0a84344f56574f2620daa13 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 8 Jun 2018 07:57:15 -0700 Subject: [PATCH 220/235] Add lua_tabletype function to improve serialization - LUA_TTEMPTY {} -- contains no array or hash parts - LUA_TTARRAY {nil} -- contains only an array part - LUA_TTHASH {foo = "bar"} -- contains only a hash part - LUA_TTMIXED {1, foo = "bar"} - contains both an array and hash --- CMakeLists.txt | 2 +- include/luasandbox/lua.h | 8 ++++++++ src/lua/lapi.c | 12 ++++++++++++ src/lua/ltable.c | 7 +++++++ src/lua/ltable.h | 1 + src/luasandbox_serialize.c | 9 +++++++-- src/test/lua/serialize.lua | 7 +++++++ src/test/output/serialize.lua51.data | 12 +++++++++++- 8 files changed, 54 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fe76408..e3efdaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.2.11 LANGUAGES C) +project(luasandbox VERSION 1.3.0 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/include/luasandbox/lua.h b/include/luasandbox/lua.h index 75760ef..232fca9 100644 --- a/include/luasandbox/lua.h +++ b/include/luasandbox/lua.h @@ -81,6 +81,13 @@ typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize); #define LUA_TUSERDATA 7 #define LUA_TTHREAD 8 +/* +** table types +*/ +#define LUA_TTEMPTY 0 +#define LUA_TTARRAY 1 +#define LUA_TTHASH 2 +#define LUA_TTMIXED 3 /* minimum Lua stack available to a C function */ @@ -132,6 +139,7 @@ LUA_API void (lua_xmove) (lua_State *from, lua_State *to, int n); ** access functions (stack -> C) */ +LUA_API int (lua_tabletype) (lua_State *L, int idx); LUA_API int (lua_isnumber) (lua_State *L, int idx); LUA_API int (lua_isstring) (lua_State *L, int idx); LUA_API int (lua_iscfunction) (lua_State *L, int idx); diff --git a/src/lua/lapi.c b/src/lua/lapi.c index 5d5145d..b8dfde1 100644 --- a/src/lua/lapi.c +++ b/src/lua/lapi.c @@ -257,6 +257,18 @@ LUA_API int lua_iscfunction (lua_State *L, int idx) { } +LUA_API int lua_tabletype (lua_State *L, int idx) { + StkId t; + int tt; + lua_lock(L); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + tt = luaH_type(hvalue(t)); + lua_unlock(L); + return tt; +} + + LUA_API int lua_isnumber (lua_State *L, int idx) { TValue n; const TValue *o = index2adr(L, idx); diff --git a/src/lua/ltable.c b/src/lua/ltable.c index 8859231..352856c 100644 --- a/src/lua/ltable.c +++ b/src/lua/ltable.c @@ -577,6 +577,13 @@ int luaH_getn (Table *t) { } +int luaH_type (Table *t) { + if (t->sizearray == 0) { + return t->node == dummynode ? LUA_TTEMPTY : LUA_TTHASH; + } + return t->node == dummynode ? LUA_TTARRAY : LUA_TTMIXED; +} + #if defined(LUA_DEBUG) diff --git a/src/lua/ltable.h b/src/lua/ltable.h index f5b9d5e..d896ae4 100644 --- a/src/lua/ltable.h +++ b/src/lua/ltable.h @@ -29,6 +29,7 @@ LUAI_FUNC void luaH_resizearray (lua_State *L, Table *t, int nasize); LUAI_FUNC void luaH_free (lua_State *L, Table *t); LUAI_FUNC int luaH_next (lua_State *L, Table *t, StkId key); LUAI_FUNC int luaH_getn (Table *t); +LUAI_FUNC int luaH_type (Table *t); #if defined(LUA_DEBUG) diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c index 2dc1d93..5297f86 100644 --- a/src/luasandbox_serialize.c +++ b/src/luasandbox_serialize.c @@ -311,8 +311,13 @@ serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) seen = add_table_ref(&data->tables, ptr, pos); if (seen != NULL) { data->keys.pos += 1; - fprintf(data->fh, "%s = {}\n", data->keys.buf + pos); - ret = serialize_table(lsb, data, pos); + if (lua_tabletype(lsb->lua, vindex) == LUA_TTARRAY + && lua_objlen(lsb->lua, vindex) == 0) { + fprintf(data->fh, "%s = {nil}\n", data->keys.buf + pos); + } else { + fprintf(data->fh, "%s = {}\n", data->keys.buf + pos); + ret = serialize_table(lsb, data, pos); + } } else { snprintf(lsb->error_message, LSB_ERROR_SIZE, "lsb_serialize preserve table out of memory"); diff --git a/src/test/lua/serialize.lua b/src/test/lua/serialize.lua index d856be2..cdb088c 100644 --- a/src/test/lua/serialize.lua +++ b/src/test/lua/serialize.lua @@ -42,6 +42,13 @@ data = ud.new("ud2") dataRef = data +empty_array = {nil} +empty_array1 = {} +empty_array1[1] = nil +empty_object = {} +empty_object1 = {[1] = nil} +array = {1, "two", 3, "four", 5} + function process_message () return 0 end diff --git a/src/test/output/serialize.lua51.data b/src/test/output/serialize.lua51.data index 36640f0..081ddc0 100644 --- a/src/test/output/serialize.lua51.data +++ b/src/test/output/serialize.lua51.data @@ -1,6 +1,14 @@ if _PRESERVATION_VERSION and _PRESERVATION_VERSION ~= 0 then return end _G["ninf"] = -1/0 +_G["empty_object1"] = {} if _G["dataRef"] == nil then _G["dataRef"] = ud.new('ud2') end +_G["array"] = {} +_G["array"][1] = 1 +_G["array"][2] = "two" +_G["array"][3] = 3 +_G["array"][4] = "four" +_G["array"][5] = 5 +_G["empty_object"] = {} _G["rate"] = 0.12345678 _G["kvp"] = {} _G["kvp"]["a"] = "foo" @@ -12,6 +20,7 @@ _G["kvp"]["r"][4] = 92.002 _G["kvp"]["r"][5] = 91.10001 _G["kvp"]["r"]["key"] = "val" _G["kvp"]["b"] = "bar" +_G["key with spaces"] = "kws" _G["uuids"] = {} _G["uuids"][1] = {} _G["uuids"][1]["type"] = "test" @@ -20,6 +29,7 @@ _G["uuids"][2] = {} _G["uuids"][2]["type"] = "test1" _G["uuids"][2]["uuid"] = "BD48B609-8922-4E59-A358-C242075CE089" _G["inf"] = 1/0 +_G["empty_array"] = {nil} _G["data"] = _G["dataRef"] _G["cycleb"] = {} _G["cycleb"]["a"] = {} @@ -48,7 +58,7 @@ _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["bbbbbbbbbbbbbbbbbbb"]["BD48B609-8922-4E5 _G["large_key"]["aaaaaaaaaaaaaaaaaaa"]["BD48B609-8922-4E59-A358-C242075CE081"] = 1 _G["count"] = 0 _G["boolean"] = true -_G["key with spaces"] = "kws" +_G["empty_array1"] = {nil} _G["nested"] = {} _G["nested"]["arg2"] = 2 _G["nested"]["nested"] = {} From 7eba6300a74de94d9a78df6b9a3772133f9d52bc Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 26 Jun 2018 13:56:41 -0700 Subject: [PATCH 221/235] Add API lsb_heka_get_state --- include/luasandbox/heka/sandbox.h | 13 ++++++++++++- src/heka/sandbox.c | 7 +++++++ src/heka/test/test_heka_sandbox.c | 6 ++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index 1c48bea..da0d6d7 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -12,6 +12,7 @@ #include #include +#include "../../luasandbox.h" #include "../error.h" #include "../util/heka_message.h" @@ -339,7 +340,7 @@ LSB_HEKA_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); LSB_HEKA_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); /** - * Queries the state of the sandbox. + * Convenience function to test if a sandbox is running. * * @param hsb Heka sandbox * @@ -347,6 +348,16 @@ LSB_HEKA_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); */ LSB_HEKA_EXPORT bool lsb_heka_is_running(lsb_heka_sandbox *hsb); +/** + * Queries the state of the sandbox. + * + * @param hsb + * + * @return lsb_state + * + */ +LSB_HEKA_EXPORT lsb_state lsb_heka_get_state(lsb_heka_sandbox *hsb); + /** * Retrieve the currently active sandbox message. This call returns a handle to * internal data and is not thread safe. diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 69ad06a..55504ae 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -1089,6 +1089,13 @@ bool lsb_heka_is_running(lsb_heka_sandbox *hsb) } +lsb_state lsb_heka_get_state(lsb_heka_sandbox *hsb) +{ + if (!hsb) return LSB_UNKNOWN; + return lsb_get_state(hsb->lsb); +} + + const lsb_heka_message* lsb_heka_get_message(lsb_heka_sandbox *hsb) { if (!hsb) return NULL; diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 2e4ca5d..4ed1680 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -260,6 +260,8 @@ static char* test_api_assertion() lsb_heka_stats stats = lsb_heka_get_stats(NULL); mu_assert(0 == stats.mem_cur, "received: %llu", stats.mem_cur); mu_assert(!lsb_heka_is_running(NULL), "running is true"); + int state = lsb_heka_get_state(NULL); + mu_assert(LSB_UNKNOWN == state, "received: %d", state); return NULL; } @@ -353,6 +355,8 @@ static char* test_timer_event() mu_assert(0 == stats.te_avg, "received %g", stats.te_avg); mu_assert(0 == stats.te_sd, "received %g", stats.te_sd); mu_assert(true == lsb_heka_is_running(hsb), "not running"); + int state = lsb_heka_get_state(hsb); + mu_assert(LSB_RUNNING == state, "received: %d", state); mu_assert(0 == lsb_heka_timer_event(hsb, 0, false), "err: %s", lsb_heka_get_error(hsb)); @@ -361,6 +365,8 @@ static char* test_timer_event() mu_assert(1 == lsb_heka_timer_event(hsb, 2, false), "err: %s", lsb_heka_get_error(hsb)); mu_assert(false == lsb_heka_is_running(hsb), "not running"); + state = lsb_heka_get_state(hsb); + mu_assert(LSB_TERMINATED == state, "received: %d", state); stats = lsb_heka_get_stats(hsb); mu_assert(0 == stats.im_cnt, "received %llu", stats.im_cnt); From 20a27f42da94bbbd2ff477b8f717015a46212879 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 10 Aug 2018 20:24:48 -0700 Subject: [PATCH 222/235] Fix the potential integer overflow when parsing mal-formed protobuf data --- CMakeLists.txt | 2 +- src/cli/lsb_heka_cat.c | 2 +- src/heka/message.c | 10 +++++----- src/util/heka_message.c | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e3efdaf..d885362 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.3.0 LANGUAGES C) +project(luasandbox VERSION 1.3.1 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 81eb328..5f6eb86 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -51,7 +51,7 @@ read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) long long vi; p = lsb_pb_read_varint(p, e, &vi); - if (!p || vi < 0 || p + vi > e) { + if (!p || vi < 0 || vi > e - p) { return NULL; } s->s = p; diff --git a/src/heka/message.c b/src/heka/message.c index fef1a17..38fd99f 100644 --- a/src/heka/message.c +++ b/src/heka/message.c @@ -65,7 +65,7 @@ static const char* read_string(lua_State *lua, long long len = 0; p = lsb_pb_read_varint(p, e, &len); - if (!p || len < 0 || p + len > e) { + if (!p || len < 0 || len > e - p) { return NULL; } lua_pushlstring(lua, p, (size_t)len); @@ -104,7 +104,7 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) long long len = 0; p = lsb_pb_read_varint(p, e, &len); - if (!p || len < 0 || p + len > e) { + if (!p || len < 0 || len > e - p) { return NULL; } e = p + len; // only process to the end of the current field record @@ -154,7 +154,7 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) break; case 2: p = lsb_pb_read_varint(p, e, &len); - if (!p || len < 0 || p + len > e) { + if (!p || len < 0 || len > e - p) { p = NULL; break; } @@ -188,7 +188,7 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) break; case 2: p = lsb_pb_read_varint(p, e, &len); - if (!p || len < 0 || p + len > e || len % sizeof(double) != 0) { + if (!p || len < 0 || len > e - p || len % sizeof(double) != 0) { p = NULL; break; } @@ -218,7 +218,7 @@ static const char* process_fields(lua_State *lua, const char *p, const char *e) break; case 2: p = lsb_pb_read_varint(p, e, &len); - if (!p || len < 0 || p + len > e) { + if (!p || len < 0 || len > e - p) { p = NULL; break; } diff --git a/src/util/heka_message.c b/src/util/heka_message.c index fdae5ff..467a7e8 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -54,7 +54,7 @@ read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) long long vi; p = lsb_pb_read_varint(p, e, &vi); - if (!p || vi < 0 || p + vi > e) { + if (!p || vi < 0 || vi > e - p) { return NULL; } s->s = p; @@ -135,7 +135,7 @@ process_fields(lsb_heka_field *f, const char *p, const char *e) long long vi = 0; p = lsb_pb_read_varint(p, e, &vi); - if (!p || vi < 0 || p + vi > e) { + if (!p || vi < 0 || vi > e - p) { return NULL; } e = p + vi; // only process to the end of the current field record @@ -188,7 +188,7 @@ process_fields(lsb_heka_field *f, const char *p, const char *e) } if (wiretype == 2) { p = lsb_pb_read_varint(p, e, &vi); - if (!p || vi < 0 || p + vi > e) { + if (!p || vi < 0 || vi > e - p) { p = NULL; break; } From 0fe4c6ed8482cacc04f7aeebf0db0c320aafc5f2 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 10 Aug 2018 20:28:22 -0700 Subject: [PATCH 223/235] Add NULL pointer checks to the string_matcher API functions --- src/util/string_matcher.c | 2 ++ src/util/test/test_string_matcher.c | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/util/string_matcher.c b/src/util/string_matcher.c index e4fccec..69d3230 100644 --- a/src/util/string_matcher.c +++ b/src/util/string_matcher.c @@ -281,6 +281,7 @@ static const char* lmemfind(const char *s1, size_t l1, bool lsb_string_match(const char *s, size_t len, const char *p) { + if (!s || !p) { return false; } MatchState ms; int anchor = (*p == '^') ? (p++, 1) : 0; const char *s1 = s; @@ -298,6 +299,7 @@ bool lsb_string_match(const char *s, size_t len, const char *p) bool lsb_string_find(const char *s, size_t ls, const char *p, size_t lp) { + if (!s || !p) { return false; } if (lmemfind(s, ls, p, lp)) { return true; } diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c index 50289b1..bf70f40 100644 --- a/src/util/test/test_string_matcher.c +++ b/src/util/test/test_string_matcher.c @@ -133,6 +133,10 @@ static char* test_find_failure() "find test: %d string: %s pattern: %s", i / 2, tests[i], tests[i + 1]); } + mu_assert(!lsb_string_find(NULL, 0, tests[0], strlen(tests[1])), + "find test: string: NULL pattern: %s", tests[1]); + mu_assert(!lsb_string_find(tests[0], strlen(tests[0]), NULL, 0), + "find test: string: %s pattern: NULL", tests[0]); return NULL; } @@ -155,6 +159,10 @@ static char* test_invalid() "invalid test: %d string: %s pattern: %s", i / 2, tests[i], tests[i + 1]); } + mu_assert(!lsb_string_match(NULL, 0, tests[1]), + "invalid test: string: NULL pattern: %s", tests[1]); + mu_assert(!lsb_string_match(tests[0], strlen(tests[0]), NULL), + "invalid test: string: %s pattern: NULL", tests[0]); return NULL; } From e85f64f21d48e311e3811b67b124fd636a926608 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 10 Aug 2018 20:53:01 -0700 Subject: [PATCH 224/235] Add NIL message matcher support for string header values --- docs/util/message_matcher.md | 2 +- src/util/heka_message_matcher.c | 51 +- src/util/heka_message_matcher_parser.c | 722 +++++++++++----------- src/util/heka_message_matcher_parser.leg | 31 +- src/util/test/test_heka_message_matcher.c | 91 ++- 5 files changed, 514 insertions(+), 383 deletions(-) diff --git a/docs/util/message_matcher.md b/docs/util/message_matcher.md index d5bab31..361c4a2 100644 --- a/docs/util/message_matcher.md +++ b/docs/util/message_matcher.md @@ -41,7 +41,7 @@ consume (see [Heka Message Structure](/heka/message.md)) ## Constants -* NIL used to test the existence (!=) or non-existence (==) of a field variable +* NIL used to test the existence (!=) or non-existence (==) of optional headers or field variables ## Message Variables diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c index c1c7125..381b81a 100644 --- a/src/util/heka_message_matcher.c +++ b/src/util/heka_message_matcher.c @@ -8,6 +8,7 @@ #include "heka_message_matcher_impl.h" +#include #include #include #include @@ -23,36 +24,30 @@ static bool string_test(match_node *mn, lsb_const_string *val) const char *mn_val = mn->data + mn->var_len; switch (mn->op) { case OP_EQ: - if (val->len != mn->val_len) return false; + if (val->len != mn->val_len || !val->s) return false; return strncmp(val->s, mn_val, val->len) == 0; case OP_NE: - if (val->len != mn->val_len) return true; + if (val->len != mn->val_len || !val->s) return true; return strncmp(val->s, mn_val, val->len) != 0; case OP_LT: { + if (!val->s) return true; int cmp = strncmp(val->s, mn_val, val->len); - if (cmp == 0) { - return val->len < mn->val_len; - } - return cmp < 0; + return cmp == 0 ? val->len < mn->val_len : cmp < 0; } case OP_LTE: - return strncmp(val->s, mn_val, val->len) <= 0; + return val->s ? strncmp(val->s, mn_val, val->len) <= 0 : true; case OP_GT: { + if (!val->s) return false; int cmp = strncmp(val->s, mn_val, val->len); - if (cmp == 0) { - return val->len > mn->val_len; - } - return cmp > 0; + return cmp == 0 ? val->len > mn->val_len : cmp > 0; } case OP_GTE: { + if (!val->s) return false; int cmp = strncmp(val->s, mn_val, val->len); - if (cmp == 0) { - return val->len >= mn->val_len; - } - return cmp > 0; + return cmp == 0 ? val->len >= mn->val_len : cmp > 0; } case OP_RE: if (mn->val_mod == PATTERN_MOD_ESC) { @@ -75,7 +70,7 @@ static bool string_test(match_node *mn, lsb_const_string *val) static bool numeric_test(match_node *mn, double val) { - double d; + double d = 0; memcpy(&d, mn->data + mn->var_len, sizeof(double)); switch (mn->op) { case OP_EQ: @@ -109,18 +104,42 @@ static bool eval_node(match_node *mn, lsb_heka_message *m) case LSB_PB_TIMESTAMP: return numeric_test(mn, (double)m->timestamp); case LSB_PB_TYPE: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->type.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } return string_test(mn, &m->type); case LSB_PB_LOGGER: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->logger.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } return string_test(mn, &m->logger); case LSB_PB_SEVERITY: return numeric_test(mn, m->severity); case LSB_PB_PAYLOAD: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->payload.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } return string_test(mn, &m->payload); case LSB_PB_ENV_VERSION: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->env_version.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } return string_test(mn, &m->env_version); case LSB_PB_PID: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->pid == INT_MIN; + return mn->op == OP_EQ ? is_nil : !is_nil; + } return numeric_test(mn, m->pid); case LSB_PB_HOSTNAME: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->hostname.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } return string_test(mn, &m->hostname); case LSB_PB_UUID: return string_test(mn, &m->uuid); diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index c7e8ce3..0b98f63 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -3,8 +3,8 @@ #include #include #include -#define YYRULECOUNT 55 -#line 1 "heka_message_matcher_parser.leg" +#define YYRULECOUNT 56 +#line 1 "../src/util/heka_message_matcher_parser.leg" /* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ /* vim: set ts=2 et sw=2 tw=80: */ @@ -545,50 +545,51 @@ YY_LOCAL(void) yySet(yycontext *yy, char *text, int count) { yy->__val[count]= #define YYACCEPT yyAccept(yy, yythunkpos0) -YY_RULE(int) yy_second_frac(yycontext *yy); /* 55 */ -YY_RULE(int) yy_second(yycontext *yy); /* 54 */ -YY_RULE(int) yy_minute(yycontext *yy); /* 53 */ -YY_RULE(int) yy_hour(yycontext *yy); /* 52 */ -YY_RULE(int) yy_timeoffset(yycontext *yy); /* 51 */ -YY_RULE(int) yy_partialtime(yycontext *yy); /* 50 */ -YY_RULE(int) yy_fulltime(yycontext *yy); /* 49 */ -YY_RULE(int) yy_day(yycontext *yy); /* 48 */ -YY_RULE(int) yy_month(yycontext *yy); /* 47 */ -YY_RULE(int) yy_year(yycontext *yy); /* 46 */ -YY_RULE(int) yy_fulldate(yycontext *yy); /* 45 */ -YY_RULE(int) yy_rfc3339(yycontext *yy); /* 44 */ -YY_RULE(int) yy_ts_quoted(yycontext *yy); /* 43 */ -YY_RULE(int) yy_zero_to_255(yycontext *yy); /* 42 */ -YY_RULE(int) yy_index(yycontext *yy); /* 41 */ -YY_RULE(int) yy_nil(yycontext *yy); /* 40 */ -YY_RULE(int) yy_fields(yycontext *yy); /* 39 */ -YY_RULE(int) yy_exponent(yycontext *yy); /* 38 */ -YY_RULE(int) yy_decimal(yycontext *yy); /* 37 */ -YY_RULE(int) yy_number(yycontext *yy); /* 36 */ -YY_RULE(int) yy_sign(yycontext *yy); /* 35 */ -YY_RULE(int) yy_numeric_value(yycontext *yy); /* 34 */ -YY_RULE(int) yy_numeric_headers(yycontext *yy); /* 33 */ -YY_RULE(int) yy_string_match_mod(yycontext *yy); /* 32 */ -YY_RULE(int) yy_string_match(yycontext *yy); /* 31 */ -YY_RULE(int) yy_string_value(yycontext *yy); /* 30 */ -YY_RULE(int) yy_string_headers(yycontext *yy); /* 29 */ -YY_RULE(int) yy_boolean(yycontext *yy); /* 28 */ -YY_RULE(int) yy_false(yycontext *yy); /* 27 */ -YY_RULE(int) yy_true(yycontext *yy); /* 26 */ -YY_RULE(int) yy_relational(yycontext *yy); /* 25 */ -YY_RULE(int) yy_op_lt(yycontext *yy); /* 24 */ -YY_RULE(int) yy_op_lte(yycontext *yy); /* 23 */ -YY_RULE(int) yy_op_gt(yycontext *yy); /* 22 */ -YY_RULE(int) yy_op_gte(yycontext *yy); /* 21 */ -YY_RULE(int) yy_op_sne(yycontext *yy); /* 20 */ -YY_RULE(int) yy_op_seq(yycontext *yy); /* 19 */ -YY_RULE(int) yy_op_ne(yycontext *yy); /* 18 */ -YY_RULE(int) yy_op_eq(yycontext *yy); /* 17 */ -YY_RULE(int) yy_boolean_test(yycontext *yy); /* 16 */ -YY_RULE(int) yy_ts_test(yycontext *yy); /* 15 */ -YY_RULE(int) yy_field_test(yycontext *yy); /* 14 */ -YY_RULE(int) yy_numeric_test(yycontext *yy); /* 13 */ -YY_RULE(int) yy_string_test(yycontext *yy); /* 12 */ +YY_RULE(int) yy_second_frac(yycontext *yy); /* 56 */ +YY_RULE(int) yy_second(yycontext *yy); /* 55 */ +YY_RULE(int) yy_minute(yycontext *yy); /* 54 */ +YY_RULE(int) yy_hour(yycontext *yy); /* 53 */ +YY_RULE(int) yy_timeoffset(yycontext *yy); /* 52 */ +YY_RULE(int) yy_partialtime(yycontext *yy); /* 51 */ +YY_RULE(int) yy_fulltime(yycontext *yy); /* 50 */ +YY_RULE(int) yy_day(yycontext *yy); /* 49 */ +YY_RULE(int) yy_month(yycontext *yy); /* 48 */ +YY_RULE(int) yy_year(yycontext *yy); /* 47 */ +YY_RULE(int) yy_fulldate(yycontext *yy); /* 46 */ +YY_RULE(int) yy_rfc3339(yycontext *yy); /* 45 */ +YY_RULE(int) yy_ts_quoted(yycontext *yy); /* 44 */ +YY_RULE(int) yy_zero_to_255(yycontext *yy); /* 43 */ +YY_RULE(int) yy_index(yycontext *yy); /* 42 */ +YY_RULE(int) yy_fields(yycontext *yy); /* 41 */ +YY_RULE(int) yy_exponent(yycontext *yy); /* 40 */ +YY_RULE(int) yy_decimal(yycontext *yy); /* 39 */ +YY_RULE(int) yy_number(yycontext *yy); /* 38 */ +YY_RULE(int) yy_sign(yycontext *yy); /* 37 */ +YY_RULE(int) yy_numeric_value(yycontext *yy); /* 36 */ +YY_RULE(int) yy_string_match_mod(yycontext *yy); /* 35 */ +YY_RULE(int) yy_nil(yycontext *yy); /* 34 */ +YY_RULE(int) yy_string_match(yycontext *yy); /* 33 */ +YY_RULE(int) yy_string_value(yycontext *yy); /* 32 */ +YY_RULE(int) yy_string_headers(yycontext *yy); /* 31 */ +YY_RULE(int) yy_boolean(yycontext *yy); /* 30 */ +YY_RULE(int) yy_false(yycontext *yy); /* 29 */ +YY_RULE(int) yy_true(yycontext *yy); /* 28 */ +YY_RULE(int) yy_relational(yycontext *yy); /* 27 */ +YY_RULE(int) yy_op_lt(yycontext *yy); /* 26 */ +YY_RULE(int) yy_op_lte(yycontext *yy); /* 25 */ +YY_RULE(int) yy_op_gt(yycontext *yy); /* 24 */ +YY_RULE(int) yy_op_gte(yycontext *yy); /* 23 */ +YY_RULE(int) yy_op_sne(yycontext *yy); /* 22 */ +YY_RULE(int) yy_op_seq(yycontext *yy); /* 21 */ +YY_RULE(int) yy_op_ne(yycontext *yy); /* 20 */ +YY_RULE(int) yy_op_eq(yycontext *yy); /* 19 */ +YY_RULE(int) yy_boolean_test(yycontext *yy); /* 18 */ +YY_RULE(int) yy_field_test(yycontext *yy); /* 17 */ +YY_RULE(int) yy_pid(yycontext *yy); /* 16 */ +YY_RULE(int) yy_severity(yycontext *yy); /* 15 */ +YY_RULE(int) yy_optional_string_headers(yycontext *yy); /* 14 */ +YY_RULE(int) yy_timestamp(yycontext *yy); /* 13 */ +YY_RULE(int) yy_uuid(yycontext *yy); /* 12 */ YY_RULE(int) yy_close(yycontext *yy); /* 11 */ YY_RULE(int) yy_open(yycontext *yy); /* 10 */ YY_RULE(int) yy_test(yycontext *yy); /* 9 */ @@ -608,8 +609,8 @@ YY_ACTION(void) yy_1_nil(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_nil\n")); { -#line 376 - yy->ctx.mn.val_type = TYPE_NIL;; +#line 377 + yy->ctx.mn.val_type = TYPE_NIL; } #undef yythunkpos #undef yypos @@ -622,7 +623,7 @@ YY_ACTION(void) yy_1_second_frac(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second_frac\n")); { -#line 374 +#line 375 yy->ctx.mn.val.d += strtod(yytext, NULL); } #undef yythunkpos @@ -636,7 +637,7 @@ YY_ACTION(void) yy_1_second(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second\n")); { -#line 373 +#line 374 __ = atoi(yytext); } #undef yythunkpos @@ -650,7 +651,7 @@ YY_ACTION(void) yy_1_minute(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_minute\n")); { -#line 369 +#line 370 __ = atoi(yytext); } #undef yythunkpos @@ -664,7 +665,7 @@ YY_ACTION(void) yy_1_hour(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_hour\n")); { -#line 368 +#line 369 __ = atoi(yytext); } #undef yythunkpos @@ -680,7 +681,7 @@ YY_ACTION(void) yy_2_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_timeoffset\n")); { -#line 364 +#line 365 update_offset(&yy->ctx, yytext[0], h, m); } #undef yythunkpos @@ -698,7 +699,7 @@ YY_ACTION(void) yy_1_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_timeoffset\n")); { -#line 363 +#line 364 update_offset(&yy->ctx, '+', 0, 0); } #undef yythunkpos @@ -717,7 +718,7 @@ YY_ACTION(void) yy_1_partialtime(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_partialtime\n")); { -#line 362 +#line 363 update_time(&yy->ctx, h, m, s); } #undef yythunkpos @@ -734,7 +735,7 @@ YY_ACTION(void) yy_1_day(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_day\n")); { -#line 358 +#line 359 __ = atoi(yytext); } #undef yythunkpos @@ -748,7 +749,7 @@ YY_ACTION(void) yy_1_month(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_month\n")); { -#line 353 +#line 354 __ = atoi(yytext); } #undef yythunkpos @@ -762,7 +763,7 @@ YY_ACTION(void) yy_1_year(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_year\n")); { -#line 350 +#line 351 __ = atoi(yytext); } #undef yythunkpos @@ -779,7 +780,7 @@ YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fulldate\n")); { -#line 349 +#line 350 update_date(&yy->ctx, y, m, d); } #undef yythunkpos @@ -789,14 +790,14 @@ YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) #undef m #undef y } -YY_ACTION(void) yy_1_ts_test(yycontext *yy, char *yytext, int yyleng) +YY_ACTION(void) yy_1_timestamp(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ #define yypos yy->__pos #define yythunkpos yy->__thunkpos - yyprintf((stderr, "do yy_1_ts_test\n")); + yyprintf((stderr, "do yy_1_timestamp\n")); { -#line 347 +#line 348 set_timestamp(&yy->ctx); } #undef yythunkpos @@ -810,7 +811,7 @@ YY_ACTION(void) yy_1_index(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_index\n")); { -#line 341 +#line 342 __ = atoi(yytext); } #undef yythunkpos @@ -826,7 +827,7 @@ YY_ACTION(void) yy_3_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_fields\n")); { -#line 340 +#line 341 yy->ctx.mn.ai = a; } #undef yythunkpos @@ -844,7 +845,7 @@ YY_ACTION(void) yy_2_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_fields\n")); { -#line 340 +#line 341 yy->ctx.mn.fi = f; } #undef yythunkpos @@ -862,7 +863,7 @@ YY_ACTION(void) yy_1_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fields\n")); { -#line 340 +#line 341 set_field(&yy->ctx, yytext); } #undef yythunkpos @@ -878,35 +879,35 @@ YY_ACTION(void) yy_1_numeric_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_numeric_value\n")); { -#line 330 +#line 331 set_numeric_value(&yy->ctx, yytext); } #undef yythunkpos #undef yypos #undef yy } -YY_ACTION(void) yy_2_numeric_headers(yycontext *yy, char *yytext, int yyleng) +YY_ACTION(void) yy_1_pid(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ #define yypos yy->__pos #define yythunkpos yy->__thunkpos - yyprintf((stderr, "do yy_2_numeric_headers\n")); + yyprintf((stderr, "do yy_1_pid\n")); { -#line 327 +#line 329 yy->ctx.mn.id = LSB_PB_PID; } #undef yythunkpos #undef yypos #undef yy } -YY_ACTION(void) yy_1_numeric_headers(yycontext *yy, char *yytext, int yyleng) +YY_ACTION(void) yy_1_severity(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ #define yypos yy->__pos #define yythunkpos yy->__thunkpos - yyprintf((stderr, "do yy_1_numeric_headers\n")); + yyprintf((stderr, "do yy_1_severity\n")); { -#line 326 +#line 328 yy->ctx.mn.id = LSB_PB_SEVERITY; } #undef yythunkpos @@ -920,7 +921,7 @@ YY_ACTION(void) yy_1_string_match_mod(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_match_mod\n")); { -#line 324 +#line 326 yy->ctx.mn.val_mod = PATTERN_MOD_ESC; } #undef yythunkpos @@ -934,7 +935,7 @@ YY_ACTION(void) yy_1_string_match(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_match\n")); { -#line 322 +#line 324 set_match_mod(&yy->ctx); } #undef yythunkpos @@ -948,27 +949,13 @@ YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_value\n")); { -#line 320 +#line 322 set_string_value(&yy->ctx, yytext); } #undef yythunkpos #undef yypos #undef yy } -YY_ACTION(void) yy_6_string_headers(yycontext *yy, char *yytext, int yyleng) -{ -#define __ yy->__ -#define yypos yy->__pos -#define yythunkpos yy->__thunkpos - yyprintf((stderr, "do yy_6_string_headers\n")); - { -#line 316 - yy->ctx.mn.id = LSB_PB_UUID; - } -#undef yythunkpos -#undef yypos -#undef yy -} YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ @@ -976,7 +963,7 @@ YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_5_string_headers\n")); { -#line 315 +#line 318 yy->ctx.mn.id = LSB_PB_PAYLOAD; } #undef yythunkpos @@ -990,7 +977,7 @@ YY_ACTION(void) yy_4_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_4_string_headers\n")); { -#line 314 +#line 317 yy->ctx.mn.id = LSB_PB_ENV_VERSION; } #undef yythunkpos @@ -1004,7 +991,7 @@ YY_ACTION(void) yy_3_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_string_headers\n")); { -#line 313 +#line 316 yy->ctx.mn.id = LSB_PB_HOSTNAME; } #undef yythunkpos @@ -1018,7 +1005,7 @@ YY_ACTION(void) yy_2_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_string_headers\n")); { -#line 312 +#line 315 yy->ctx.mn.id = LSB_PB_LOGGER; } #undef yythunkpos @@ -1032,13 +1019,27 @@ YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_headers\n")); { -#line 311 +#line 314 yy->ctx.mn.id = LSB_PB_TYPE; } #undef yythunkpos #undef yypos #undef yy } +YY_ACTION(void) yy_1_uuid(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_uuid\n")); + { +#line 313 + yy->ctx.mn.id = LSB_PB_UUID; + } +#undef yythunkpos +#undef yypos +#undef yy +} YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) { #define __ yy->__ @@ -1046,7 +1047,7 @@ YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_close\n")); { -#line 307 +#line 309 pop_to_paren(&yy->ctx); } #undef yythunkpos @@ -1060,7 +1061,7 @@ YY_ACTION(void) yy_1_open(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_open\n")); { -#line 306 +#line 308 push_op(&yy->ctx, OP_OPEN); } #undef yythunkpos @@ -1074,7 +1075,7 @@ YY_ACTION(void) yy_1_or(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_or\n")); { -#line 305 +#line 307 push_op(&yy->ctx, OP_OR); } #undef yythunkpos @@ -1088,7 +1089,7 @@ YY_ACTION(void) yy_1_and(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_and\n")); { -#line 304 +#line 306 push_op(&yy->ctx, OP_AND); } #undef yythunkpos @@ -1102,7 +1103,7 @@ YY_ACTION(void) yy_2_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean\n")); { -#line 302 +#line 304 yy->ctx.mn.val_type = TYPE_FALSE; } #undef yythunkpos @@ -1116,7 +1117,7 @@ YY_ACTION(void) yy_1_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean\n")); { -#line 301 +#line 303 yy->ctx.mn.val_type = TYPE_TRUE; } #undef yythunkpos @@ -1130,7 +1131,7 @@ YY_ACTION(void) yy_2_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean_test\n")); { -#line 300 +#line 302 yy->ctx.mn.op = OP_FALSE; } #undef yythunkpos @@ -1144,7 +1145,7 @@ YY_ACTION(void) yy_1_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean_test\n")); { -#line 299 +#line 301 yy->ctx.mn.op = OP_TRUE; } #undef yythunkpos @@ -1158,7 +1159,7 @@ YY_ACTION(void) yy_1_op_lt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lt\n")); { -#line 290 +#line 292 yy->ctx.mn.op = OP_LT; } #undef yythunkpos @@ -1172,7 +1173,7 @@ YY_ACTION(void) yy_1_op_lte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lte\n")); { -#line 289 +#line 291 yy->ctx.mn.op = OP_LTE; } #undef yythunkpos @@ -1186,7 +1187,7 @@ YY_ACTION(void) yy_1_op_gt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gt\n")); { -#line 288 +#line 290 yy->ctx.mn.op = OP_GT; } #undef yythunkpos @@ -1200,7 +1201,7 @@ YY_ACTION(void) yy_1_op_gte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gte\n")); { -#line 287 +#line 289 yy->ctx.mn.op = OP_GTE; } #undef yythunkpos @@ -1214,7 +1215,7 @@ YY_ACTION(void) yy_1_op_sne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_sne\n")); { -#line 286 +#line 288 yy->ctx.mn.op = OP_NRE; } #undef yythunkpos @@ -1228,7 +1229,7 @@ YY_ACTION(void) yy_1_op_seq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_seq\n")); { -#line 285 +#line 287 yy->ctx.mn.op = OP_RE; } #undef yythunkpos @@ -1242,7 +1243,7 @@ YY_ACTION(void) yy_1_op_ne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_ne\n")); { -#line 284 +#line 286 yy->ctx.mn.op = OP_NE; } #undef yythunkpos @@ -1256,7 +1257,7 @@ YY_ACTION(void) yy_1_op_eq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_eq\n")); { -#line 283 +#line 285 yy->ctx.mn.op = OP_EQ; } #undef yythunkpos @@ -1270,7 +1271,7 @@ YY_ACTION(void) yy_1_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_test\n")); { -#line 281 +#line 283 push_output(&yy->ctx, &yy->ctx.mn); } #undef yythunkpos @@ -1630,101 +1631,92 @@ if (!(YY_END)) goto l34; yyprintf((stderr, " fail %s @ %s\n", "index", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_nil(yycontext *yy) -{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "nil")); if (!yymatchString(yy, "NIL")) goto l35; yyDo(yy, yy_1_nil, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l35; - yyprintf((stderr, " ok %s @ %s\n", "nil", yy->__buf+yy->__pos)); - return 1; - l35:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "nil", yy->__buf+yy->__pos)); - return 0; -} YY_RULE(int) yy_fields(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 2, 0); - yyprintf((stderr, "%s\n", "fields")); if (!yymatchString(yy, "Fields[")) goto l36; yyText(yy, yy->__begin, yy->__end); { + yyprintf((stderr, "%s\n", "fields")); if (!yymatchString(yy, "Fields[")) goto l35; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l36; +if (!(YY_BEGIN)) goto l35; #undef yytext #undef yyleng } - l37:; - { int yypos38= yy->__pos, yythunkpos38= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\377\377\377\377\377\377\377\377\377\377\377\337\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377")) goto l38; goto l37; - l38:; yy->__pos= yypos38; yy->__thunkpos= yythunkpos38; + l36:; + { int yypos37= yy->__pos, yythunkpos37= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\377\377\377\377\377\377\377\377\377\377\377\337\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377")) goto l37; goto l36; + l37:; yy->__pos= yypos37; yy->__thunkpos= yythunkpos37; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l36; +if (!(YY_END)) goto l35; #undef yytext #undef yyleng - } if (!yymatchChar(yy, ']')) goto l36; yyDo(yy, yy_1_fields, yy->__begin, yy->__end); - { int yypos39= yy->__pos, yythunkpos39= yy->__thunkpos; if (!yy_index(yy)) goto l39; yyDo(yy, yySet, -2, 0); goto l40; - l39:; yy->__pos= yypos39; yy->__thunkpos= yythunkpos39; + } if (!yymatchChar(yy, ']')) goto l35; yyDo(yy, yy_1_fields, yy->__begin, yy->__end); + { int yypos38= yy->__pos, yythunkpos38= yy->__thunkpos; if (!yy_index(yy)) goto l38; yyDo(yy, yySet, -2, 0); goto l39; + l38:; yy->__pos= yypos38; yy->__thunkpos= yythunkpos38; } - l40:; yyDo(yy, yy_2_fields, yy->__begin, yy->__end); - { int yypos41= yy->__pos, yythunkpos41= yy->__thunkpos; if (!yy_index(yy)) goto l41; yyDo(yy, yySet, -1, 0); goto l42; - l41:; yy->__pos= yypos41; yy->__thunkpos= yythunkpos41; + l39:; yyDo(yy, yy_2_fields, yy->__begin, yy->__end); + { int yypos40= yy->__pos, yythunkpos40= yy->__thunkpos; if (!yy_index(yy)) goto l40; yyDo(yy, yySet, -1, 0); goto l41; + l40:; yy->__pos= yypos40; yy->__thunkpos= yythunkpos40; } - l42:; yyDo(yy, yy_3_fields, yy->__begin, yy->__end); + l41:; yyDo(yy, yy_3_fields, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "fields", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 2, 0); return 1; - l36:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l35:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "fields", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_exponent(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "exponent")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\000\000\040\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l43; - { int yypos44= yy->__pos, yythunkpos44= yy->__thunkpos; if (!yy_sign(yy)) goto l44; goto l45; - l44:; yy->__pos= yypos44; yy->__thunkpos= yythunkpos44; + yyprintf((stderr, "%s\n", "exponent")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\000\000\040\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l42; + { int yypos43= yy->__pos, yythunkpos43= yy->__thunkpos; if (!yy_sign(yy)) goto l43; goto l44; + l43:; yy->__pos= yypos43; yy->__thunkpos= yythunkpos43; } - l45:; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l43; - l46:; - { int yypos47= yy->__pos, yythunkpos47= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l47; goto l46; - l47:; yy->__pos= yypos47; yy->__thunkpos= yythunkpos47; + l44:; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l42; + l45:; + { int yypos46= yy->__pos, yythunkpos46= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l46; goto l45; + l46:; yy->__pos= yypos46; yy->__thunkpos= yythunkpos46; } yyprintf((stderr, " ok %s @ %s\n", "exponent", yy->__buf+yy->__pos)); return 1; - l43:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l42:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "exponent", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_decimal(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "decimal")); if (!yymatchChar(yy, '.')) goto l48; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l48; - l49:; - { int yypos50= yy->__pos, yythunkpos50= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l50; goto l49; - l50:; yy->__pos= yypos50; yy->__thunkpos= yythunkpos50; + yyprintf((stderr, "%s\n", "decimal")); if (!yymatchChar(yy, '.')) goto l47; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l47; + l48:; + { int yypos49= yy->__pos, yythunkpos49= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l49; goto l48; + l49:; yy->__pos= yypos49; yy->__thunkpos= yythunkpos49; } yyprintf((stderr, " ok %s @ %s\n", "decimal", yy->__buf+yy->__pos)); return 1; - l48:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l47:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "decimal", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_number(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "number")); - { int yypos52= yy->__pos, yythunkpos52= yy->__thunkpos; if (!yymatchChar(yy, '0')) goto l53; goto l52; - l53:; yy->__pos= yypos52; yy->__thunkpos= yythunkpos52; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l51; - l54:; - { int yypos55= yy->__pos, yythunkpos55= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l55; goto l54; - l55:; yy->__pos= yypos55; yy->__thunkpos= yythunkpos55; + { int yypos51= yy->__pos, yythunkpos51= yy->__thunkpos; if (!yymatchChar(yy, '0')) goto l52; goto l51; + l52:; yy->__pos= yypos51; yy->__thunkpos= yythunkpos51; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l50; + l53:; + { int yypos54= yy->__pos, yythunkpos54= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l54; goto l53; + l54:; yy->__pos= yypos54; yy->__thunkpos= yythunkpos54; } } - l52:; + l51:; yyprintf((stderr, " ok %s @ %s\n", "number", yy->__buf+yy->__pos)); return 1; - l51:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l50:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "number", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_sign(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "sign")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\050\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l56; + yyprintf((stderr, "%s\n", "sign")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\050\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l55; yyprintf((stderr, " ok %s @ %s\n", "sign", yy->__buf+yy->__pos)); return 1; - l56:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l55:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "sign", yy->__buf+yy->__pos)); return 0; } @@ -1733,480 +1725,512 @@ YY_RULE(int) yy_numeric_value(yycontext *yy) yyprintf((stderr, "%s\n", "numeric_value")); yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l57; +if (!(YY_BEGIN)) goto l56; #undef yytext #undef yyleng } - { int yypos58= yy->__pos, yythunkpos58= yy->__thunkpos; if (!yy_sign(yy)) goto l58; goto l59; - l58:; yy->__pos= yypos58; yy->__thunkpos= yythunkpos58; + { int yypos57= yy->__pos, yythunkpos57= yy->__thunkpos; if (!yy_sign(yy)) goto l57; goto l58; + l57:; yy->__pos= yypos57; yy->__thunkpos= yythunkpos57; } - l59:; if (!yy_number(yy)) goto l57; - { int yypos60= yy->__pos, yythunkpos60= yy->__thunkpos; if (!yy_decimal(yy)) goto l60; goto l61; - l60:; yy->__pos= yypos60; yy->__thunkpos= yythunkpos60; + l58:; if (!yy_number(yy)) goto l56; + { int yypos59= yy->__pos, yythunkpos59= yy->__thunkpos; if (!yy_decimal(yy)) goto l59; goto l60; + l59:; yy->__pos= yypos59; yy->__thunkpos= yythunkpos59; } - l61:; - { int yypos62= yy->__pos, yythunkpos62= yy->__thunkpos; if (!yy_exponent(yy)) goto l62; goto l63; - l62:; yy->__pos= yypos62; yy->__thunkpos= yythunkpos62; + l60:; + { int yypos61= yy->__pos, yythunkpos61= yy->__thunkpos; if (!yy_exponent(yy)) goto l61; goto l62; + l61:; yy->__pos= yypos61; yy->__thunkpos= yythunkpos61; } - l63:; yyText(yy, yy->__begin, yy->__end); { + l62:; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l57; +if (!(YY_END)) goto l56; #undef yytext #undef yyleng } yyDo(yy, yy_1_numeric_value, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "numeric_value", yy->__buf+yy->__pos)); return 1; - l57:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l56:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "numeric_value", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_numeric_headers(yycontext *yy) +YY_RULE(int) yy_string_match_mod(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "numeric_headers")); - { int yypos65= yy->__pos, yythunkpos65= yy->__thunkpos; if (!yymatchString(yy, "Severity")) goto l66; yyDo(yy, yy_1_numeric_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l66; goto l65; - l66:; yy->__pos= yypos65; yy->__thunkpos= yythunkpos65; if (!yymatchString(yy, "Pid")) goto l64; yyDo(yy, yy_2_numeric_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l64; - } - l65:; - yyprintf((stderr, " ok %s @ %s\n", "numeric_headers", yy->__buf+yy->__pos)); + yyprintf((stderr, "%s\n", "string_match_mod")); if (!yymatchChar(yy, '%')) goto l63; yyDo(yy, yy_1_string_match_mod, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); return 1; - l64:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "numeric_headers", yy->__buf+yy->__pos)); + l63:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_string_match_mod(yycontext *yy) +YY_RULE(int) yy_nil(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "string_match_mod")); if (!yymatchChar(yy, '%')) goto l67; yyDo(yy, yy_1_string_match_mod, yy->__begin, yy->__end); - yyprintf((stderr, " ok %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); + yyprintf((stderr, "%s\n", "nil")); if (!yymatchString(yy, "NIL")) goto l64; yyDo(yy, yy_1_nil, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l64; + yyprintf((stderr, " ok %s @ %s\n", "nil", yy->__buf+yy->__pos)); return 1; - l67:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); + l64:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "nil", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_match(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_match")); - { int yypos69= yy->__pos, yythunkpos69= yy->__thunkpos; if (!yy_op_seq(yy)) goto l70; goto l69; - l70:; yy->__pos= yypos69; yy->__thunkpos= yythunkpos69; if (!yy_op_sne(yy)) goto l68; + { int yypos66= yy->__pos, yythunkpos66= yy->__thunkpos; if (!yy_op_seq(yy)) goto l67; goto l66; + l67:; yy->__pos= yypos66; yy->__thunkpos= yythunkpos66; if (!yy_op_sne(yy)) goto l65; } - l69:; if (!yy_sp(yy)) goto l68; if (!yy_string_value(yy)) goto l68; - { int yypos71= yy->__pos, yythunkpos71= yy->__thunkpos; if (!yy_string_match_mod(yy)) goto l71; goto l72; - l71:; yy->__pos= yypos71; yy->__thunkpos= yythunkpos71; + l66:; if (!yy_sp(yy)) goto l65; if (!yy_string_value(yy)) goto l65; + { int yypos68= yy->__pos, yythunkpos68= yy->__thunkpos; if (!yy_string_match_mod(yy)) goto l68; goto l69; + l68:; yy->__pos= yypos68; yy->__thunkpos= yythunkpos68; } - l72:; yyDo(yy, yy_1_string_match, yy->__begin, yy->__end); + l69:; yyDo(yy, yy_1_string_match, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "string_match", yy->__buf+yy->__pos)); return 1; - l68:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l65:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_match", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_value(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_value")); - { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l75; yyText(yy, yy->__begin, yy->__end); { + { int yypos71= yy->__pos, yythunkpos71= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l72; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l75; +if (!(YY_BEGIN)) goto l72; #undef yytext #undef yyleng } - l76:; - { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; - { int yypos78= yy->__pos, yythunkpos78= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l79; goto l78; - l79:; yy->__pos= yypos78; yy->__thunkpos= yythunkpos78; - { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l80; goto l77; - l80:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; - } if (!yymatchDot(yy)) goto l77; - } - l78:; goto l76; + l73:; + { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; + { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l76; goto l75; + l76:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; + { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l77; goto l74; l77:; yy->__pos= yypos77; yy->__thunkpos= yythunkpos77; + } if (!yymatchDot(yy)) goto l74; + } + l75:; goto l73; + l74:; yy->__pos= yypos74; yy->__thunkpos= yythunkpos74; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l75; +if (!(YY_END)) goto l72; #undef yytext #undef yyleng - } if (!yymatchChar(yy, '"')) goto l75; goto l74; - l75:; yy->__pos= yypos74; yy->__thunkpos= yythunkpos74; if (!yymatchChar(yy, '\'')) goto l73; yyText(yy, yy->__begin, yy->__end); { + } if (!yymatchChar(yy, '"')) goto l72; goto l71; + l72:; yy->__pos= yypos71; yy->__thunkpos= yythunkpos71; if (!yymatchChar(yy, '\'')) goto l70; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_BEGIN)) goto l73; +if (!(YY_BEGIN)) goto l70; #undef yytext #undef yyleng } - l81:; - { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; - { int yypos83= yy->__pos, yythunkpos83= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l84; goto l83; - l84:; yy->__pos= yypos83; yy->__thunkpos= yythunkpos83; - { int yypos85= yy->__pos, yythunkpos85= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l85; goto l82; - l85:; yy->__pos= yypos85; yy->__thunkpos= yythunkpos85; - } if (!yymatchDot(yy)) goto l82; - } - l83:; goto l81; + l78:; + { int yypos79= yy->__pos, yythunkpos79= yy->__thunkpos; + { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l81; goto l80; + l81:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; + { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l82; goto l79; l82:; yy->__pos= yypos82; yy->__thunkpos= yythunkpos82; + } if (!yymatchDot(yy)) goto l79; + } + l80:; goto l78; + l79:; yy->__pos= yypos79; yy->__thunkpos= yythunkpos79; } yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(YY_END)) goto l73; +if (!(YY_END)) goto l70; #undef yytext #undef yyleng - } if (!yymatchChar(yy, '\'')) goto l73; + } if (!yymatchChar(yy, '\'')) goto l70; } - l74:; yyText(yy, yy->__begin, yy->__end); { + l71:; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(check_string_len(yytext))) goto l73; +if (!(check_string_len(yytext))) goto l70; #undef yytext #undef yyleng } yyDo(yy, yy_1_string_value, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "string_value", yy->__buf+yy->__pos)); return 1; - l73:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l70:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_value", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_string_headers(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "string_headers")); - { int yypos87= yy->__pos, yythunkpos87= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l88; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l88; goto l87; - l88:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Logger")) goto l89; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l89; goto l87; - l89:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Hostname")) goto l90; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l90; goto l87; - l90:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "EnvVersion")) goto l91; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l91; goto l87; - l91:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Payload")) goto l92; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l92; goto l87; - l92:; yy->__pos= yypos87; yy->__thunkpos= yythunkpos87; if (!yymatchString(yy, "Uuid")) goto l86; yyDo(yy, yy_6_string_headers, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l86; - } - l87:; + { int yypos84= yy->__pos, yythunkpos84= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l85; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); goto l84; + l85:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Logger")) goto l86; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); goto l84; + l86:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Hostname")) goto l87; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); goto l84; + l87:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "EnvVersion")) goto l88; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); goto l84; + l88:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Payload")) goto l83; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); + } + l84:; yyprintf((stderr, " ok %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); return 1; - l86:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l83:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_boolean(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "boolean")); - { int yypos94= yy->__pos, yythunkpos94= yy->__thunkpos; if (!yy_true(yy)) goto l95; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l94; - l95:; yy->__pos= yypos94; yy->__thunkpos= yythunkpos94; if (!yy_false(yy)) goto l93; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); + { int yypos90= yy->__pos, yythunkpos90= yy->__thunkpos; if (!yy_true(yy)) goto l91; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l90; + l91:; yy->__pos= yypos90; yy->__thunkpos= yythunkpos90; if (!yy_false(yy)) goto l89; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); } - l94:; + l90:; yyprintf((stderr, " ok %s @ %s\n", "boolean", yy->__buf+yy->__pos)); return 1; - l93:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l89:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "boolean", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_false(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l96; if (!yy_sp(yy)) goto l96; + yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l92; if (!yy_sp(yy)) goto l92; yyprintf((stderr, " ok %s @ %s\n", "false", yy->__buf+yy->__pos)); return 1; - l96:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l92:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "false", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_true(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l97; if (!yy_sp(yy)) goto l97; + yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l93; if (!yy_sp(yy)) goto l93; yyprintf((stderr, " ok %s @ %s\n", "true", yy->__buf+yy->__pos)); return 1; - l97:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l93:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "true", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_relational(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "relational")); - { int yypos99= yy->__pos, yythunkpos99= yy->__thunkpos; if (!yy_op_eq(yy)) goto l100; goto l99; - l100:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_ne(yy)) goto l101; goto l99; - l101:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_gte(yy)) goto l102; goto l99; - l102:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_gt(yy)) goto l103; goto l99; - l103:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_lte(yy)) goto l104; goto l99; - l104:; yy->__pos= yypos99; yy->__thunkpos= yythunkpos99; if (!yy_op_lt(yy)) goto l98; - } - l99:; + { int yypos95= yy->__pos, yythunkpos95= yy->__thunkpos; if (!yy_op_eq(yy)) goto l96; goto l95; + l96:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_ne(yy)) goto l97; goto l95; + l97:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_gte(yy)) goto l98; goto l95; + l98:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_gt(yy)) goto l99; goto l95; + l99:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_lte(yy)) goto l100; goto l95; + l100:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_lt(yy)) goto l94; + } + l95:; yyprintf((stderr, " ok %s @ %s\n", "relational", yy->__buf+yy->__pos)); return 1; - l98:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l94:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "relational", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_lt(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l101; if (!yy_sp(yy)) goto l101; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); return 1; - l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l101:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_lte(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l102; if (!yy_sp(yy)) goto l102; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); return 1; - l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l102:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_gt(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l103; if (!yy_sp(yy)) goto l103; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); return 1; - l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l103:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_gte(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l104; if (!yy_sp(yy)) goto l104; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); return 1; - l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l104:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_sne(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l109; if (!yy_sp(yy)) goto l109; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); return 1; - l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_seq(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l110; if (!yy_sp(yy)) goto l110; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); return 1; - l110:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_ne(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l111; if (!yy_sp(yy)) goto l111; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); return 1; - l111:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_op_eq(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l112; if (!yy_sp(yy)) goto l112; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); return 1; - l112:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_boolean_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "boolean_test")); - { int yypos114= yy->__pos, yythunkpos114= yy->__thunkpos; if (!yy_true(yy)) goto l115; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l114; - l115:; yy->__pos= yypos114; yy->__thunkpos= yythunkpos114; if (!yy_false(yy)) goto l113; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); + { int yypos110= yy->__pos, yythunkpos110= yy->__thunkpos; if (!yy_true(yy)) goto l111; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l110; + l111:; yy->__pos= yypos110; yy->__thunkpos= yythunkpos110; if (!yy_false(yy)) goto l109; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); } - l114:; + l110:; yyprintf((stderr, " ok %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); return 1; - l113:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_ts_test(yycontext *yy) +YY_RULE(int) yy_field_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "ts_test")); if (!yymatchString(yy, "Timestamp")) goto l116; if (!yy_sp(yy)) goto l116; if (!yy_relational(yy)) goto l116; if (!yy_sp(yy)) goto l116; - { int yypos117= yy->__pos, yythunkpos117= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l118; goto l117; - l118:; yy->__pos= yypos117; yy->__thunkpos= yythunkpos117; if (!yy_ts_quoted(yy)) goto l116; + yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l112; if (!yy_sp(yy)) goto l112; + { int yypos113= yy->__pos, yythunkpos113= yy->__thunkpos; if (!yy_relational(yy)) goto l114; if (!yy_sp(yy)) goto l114; + { int yypos115= yy->__pos, yythunkpos115= yy->__thunkpos; if (!yy_string_value(yy)) goto l116; goto l115; + l116:; yy->__pos= yypos115; yy->__thunkpos= yythunkpos115; if (!yy_numeric_value(yy)) goto l114; } - l117:; yyDo(yy, yy_1_ts_test, yy->__begin, yy->__end); - yyprintf((stderr, " ok %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); + l115:; goto l113; + l114:; yy->__pos= yypos113; yy->__thunkpos= yythunkpos113; if (!yy_string_match(yy)) goto l117; goto l113; + l117:; yy->__pos= yypos113; yy->__thunkpos= yythunkpos113; + { int yypos118= yy->__pos, yythunkpos118= yy->__thunkpos; if (!yy_op_eq(yy)) goto l119; goto l118; + l119:; yy->__pos= yypos118; yy->__thunkpos= yythunkpos118; if (!yy_op_ne(yy)) goto l112; + } + l118:; if (!yy_sp(yy)) goto l112; + { int yypos120= yy->__pos, yythunkpos120= yy->__thunkpos; if (!yy_boolean(yy)) goto l121; goto l120; + l121:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; if (!yy_nil(yy)) goto l112; + } + l120:; + } + l113:; + yyprintf((stderr, " ok %s @ %s\n", "field_test", yy->__buf+yy->__pos)); return 1; - l116:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "ts_test", yy->__buf+yy->__pos)); + l112:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "field_test", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_field_test(yycontext *yy) +YY_RULE(int) yy_pid(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l119; if (!yy_sp(yy)) goto l119; - { int yypos120= yy->__pos, yythunkpos120= yy->__thunkpos; if (!yy_relational(yy)) goto l121; if (!yy_sp(yy)) goto l121; - { int yypos122= yy->__pos, yythunkpos122= yy->__thunkpos; if (!yy_string_value(yy)) goto l123; goto l122; - l123:; yy->__pos= yypos122; yy->__thunkpos= yythunkpos122; if (!yy_numeric_value(yy)) goto l121; - } - l122:; goto l120; - l121:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; if (!yy_string_match(yy)) goto l124; goto l120; - l124:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; + yyprintf((stderr, "%s\n", "pid")); if (!yymatchString(yy, "Pid")) goto l122; yyDo(yy, yy_1_pid, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l122; + { int yypos123= yy->__pos, yythunkpos123= yy->__thunkpos; if (!yy_relational(yy)) goto l124; if (!yy_sp(yy)) goto l124; if (!yy_numeric_value(yy)) goto l124; goto l123; + l124:; yy->__pos= yypos123; yy->__thunkpos= yythunkpos123; { int yypos125= yy->__pos, yythunkpos125= yy->__thunkpos; if (!yy_op_eq(yy)) goto l126; goto l125; - l126:; yy->__pos= yypos125; yy->__thunkpos= yythunkpos125; if (!yy_op_ne(yy)) goto l119; + l126:; yy->__pos= yypos125; yy->__thunkpos= yythunkpos125; if (!yy_op_ne(yy)) goto l122; } - l125:; if (!yy_sp(yy)) goto l119; - { int yypos127= yy->__pos, yythunkpos127= yy->__thunkpos; if (!yy_boolean(yy)) goto l128; goto l127; - l128:; yy->__pos= yypos127; yy->__thunkpos= yythunkpos127; if (!yy_nil(yy)) goto l119; + l125:; if (!yy_sp(yy)) goto l122; if (!yy_nil(yy)) goto l122; } - l127:; + l123:; + yyprintf((stderr, " ok %s @ %s\n", "pid", yy->__buf+yy->__pos)); + return 1; + l122:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "pid", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_severity(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "severity")); if (!yymatchString(yy, "Severity")) goto l127; yyDo(yy, yy_1_severity, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l127; if (!yy_relational(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_numeric_value(yy)) goto l127; + yyprintf((stderr, " ok %s @ %s\n", "severity", yy->__buf+yy->__pos)); + return 1; + l127:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "severity", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_optional_string_headers(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "optional_string_headers")); if (!yy_string_headers(yy)) goto l128; if (!yy_sp(yy)) goto l128; + { int yypos129= yy->__pos, yythunkpos129= yy->__thunkpos; if (!yy_relational(yy)) goto l130; if (!yy_sp(yy)) goto l130; if (!yy_string_value(yy)) goto l130; goto l129; + l130:; yy->__pos= yypos129; yy->__thunkpos= yythunkpos129; if (!yy_string_match(yy)) goto l131; goto l129; + l131:; yy->__pos= yypos129; yy->__thunkpos= yythunkpos129; + { int yypos132= yy->__pos, yythunkpos132= yy->__thunkpos; if (!yy_op_eq(yy)) goto l133; goto l132; + l133:; yy->__pos= yypos132; yy->__thunkpos= yythunkpos132; if (!yy_op_ne(yy)) goto l128; } - l120:; - yyprintf((stderr, " ok %s @ %s\n", "field_test", yy->__buf+yy->__pos)); + l132:; if (!yy_sp(yy)) goto l128; if (!yy_nil(yy)) goto l128; + } + l129:; + yyprintf((stderr, " ok %s @ %s\n", "optional_string_headers", yy->__buf+yy->__pos)); return 1; - l119:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "field_test", yy->__buf+yy->__pos)); + l128:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "optional_string_headers", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_numeric_test(yycontext *yy) +YY_RULE(int) yy_timestamp(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "numeric_test")); if (!yy_numeric_headers(yy)) goto l129; if (!yy_sp(yy)) goto l129; if (!yy_relational(yy)) goto l129; if (!yy_sp(yy)) goto l129; if (!yy_numeric_value(yy)) goto l129; - yyprintf((stderr, " ok %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); + yyprintf((stderr, "%s\n", "timestamp")); if (!yymatchString(yy, "Timestamp")) goto l134; if (!yy_sp(yy)) goto l134; if (!yy_relational(yy)) goto l134; if (!yy_sp(yy)) goto l134; + { int yypos135= yy->__pos, yythunkpos135= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l136; goto l135; + l136:; yy->__pos= yypos135; yy->__thunkpos= yythunkpos135; if (!yy_ts_quoted(yy)) goto l134; + } + l135:; yyDo(yy, yy_1_timestamp, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "timestamp", yy->__buf+yy->__pos)); return 1; - l129:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "numeric_test", yy->__buf+yy->__pos)); + l134:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "timestamp", yy->__buf+yy->__pos)); return 0; } -YY_RULE(int) yy_string_test(yycontext *yy) +YY_RULE(int) yy_uuid(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "string_test")); if (!yy_string_headers(yy)) goto l130; if (!yy_sp(yy)) goto l130; - { int yypos131= yy->__pos, yythunkpos131= yy->__thunkpos; if (!yy_relational(yy)) goto l132; if (!yy_sp(yy)) goto l132; if (!yy_string_value(yy)) goto l132; goto l131; - l132:; yy->__pos= yypos131; yy->__thunkpos= yythunkpos131; if (!yy_string_match(yy)) goto l130; + yyprintf((stderr, "%s\n", "uuid")); if (!yymatchString(yy, "Uuid")) goto l137; yyDo(yy, yy_1_uuid, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l137; + { int yypos138= yy->__pos, yythunkpos138= yy->__thunkpos; if (!yy_relational(yy)) goto l139; if (!yy_sp(yy)) goto l139; if (!yy_string_value(yy)) goto l139; goto l138; + l139:; yy->__pos= yypos138; yy->__thunkpos= yythunkpos138; if (!yy_string_match(yy)) goto l137; } - l131:; - yyprintf((stderr, " ok %s @ %s\n", "string_test", yy->__buf+yy->__pos)); + l138:; + yyprintf((stderr, " ok %s @ %s\n", "uuid", yy->__buf+yy->__pos)); return 1; - l130:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; - yyprintf((stderr, " fail %s @ %s\n", "string_test", yy->__buf+yy->__pos)); + l137:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "uuid", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_close(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "close")); if (!yymatchChar(yy, ')')) goto l133; yyDo(yy, yy_1_close, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l133; + yyprintf((stderr, "%s\n", "close")); if (!yymatchChar(yy, ')')) goto l140; yyDo(yy, yy_1_close, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l140; yyprintf((stderr, " ok %s @ %s\n", "close", yy->__buf+yy->__pos)); return 1; - l133:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l140:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "close", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_open(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "open")); if (!yymatchChar(yy, '(')) goto l134; yyDo(yy, yy_1_open, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l134; + yyprintf((stderr, "%s\n", "open")); if (!yymatchChar(yy, '(')) goto l141; yyDo(yy, yy_1_open, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l141; yyprintf((stderr, " ok %s @ %s\n", "open", yy->__buf+yy->__pos)); return 1; - l134:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l141:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "open", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_test(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "test")); - { int yypos136= yy->__pos, yythunkpos136= yy->__thunkpos; if (!yy_string_test(yy)) goto l137; goto l136; - l137:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_numeric_test(yy)) goto l138; goto l136; - l138:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_field_test(yy)) goto l139; goto l136; - l139:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_ts_test(yy)) goto l140; goto l136; - l140:; yy->__pos= yypos136; yy->__thunkpos= yythunkpos136; if (!yy_boolean_test(yy)) goto l135; - } - l136:; yyDo(yy, yy_1_test, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l135; + { int yypos143= yy->__pos, yythunkpos143= yy->__thunkpos; if (!yy_uuid(yy)) goto l144; goto l143; + l144:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_timestamp(yy)) goto l145; goto l143; + l145:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_optional_string_headers(yy)) goto l146; goto l143; + l146:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_severity(yy)) goto l147; goto l143; + l147:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_pid(yy)) goto l148; goto l143; + l148:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_field_test(yy)) goto l149; goto l143; + l149:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_boolean_test(yy)) goto l142; + } + l143:; yyDo(yy, yy_1_test, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l142; yyprintf((stderr, " ok %s @ %s\n", "test", yy->__buf+yy->__pos)); return 1; - l135:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l142:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "test", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_and(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "and")); if (!yymatchString(yy, "&&")) goto l141; yyText(yy, yy->__begin, yy->__end); { + yyprintf((stderr, "%s\n", "and")); if (!yymatchString(yy, "&&")) goto l150; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(cond_cnt(&yy->ctx))) goto l141; +if (!(cond_cnt(&yy->ctx))) goto l150; #undef yytext #undef yyleng - } yyDo(yy, yy_1_and, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l141; + } yyDo(yy, yy_1_and, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l150; yyprintf((stderr, " ok %s @ %s\n", "and", yy->__buf+yy->__pos)); return 1; - l141:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l150:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "and", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_expr(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "expr")); - { int yypos143= yy->__pos, yythunkpos143= yy->__thunkpos; if (!yy_test(yy)) goto l144; goto l143; - l144:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_open(yy)) goto l142; if (!yy_ored(yy)) goto l142; if (!yy_close(yy)) goto l142; + { int yypos152= yy->__pos, yythunkpos152= yy->__thunkpos; if (!yy_test(yy)) goto l153; goto l152; + l153:; yy->__pos= yypos152; yy->__thunkpos= yythunkpos152; if (!yy_open(yy)) goto l151; if (!yy_ored(yy)) goto l151; if (!yy_close(yy)) goto l151; } - l143:; + l152:; yyprintf((stderr, " ok %s @ %s\n", "expr", yy->__buf+yy->__pos)); return 1; - l142:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l151:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "expr", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_or(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "or")); if (!yymatchString(yy, "||")) goto l145; yyText(yy, yy->__begin, yy->__end); { + yyprintf((stderr, "%s\n", "or")); if (!yymatchString(yy, "||")) goto l154; yyText(yy, yy->__begin, yy->__end); { #define yytext yy->__text #define yyleng yy->__textlen -if (!(cond_cnt(&yy->ctx))) goto l145; +if (!(cond_cnt(&yy->ctx))) goto l154; #undef yytext #undef yyleng - } yyDo(yy, yy_1_or, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l145; + } yyDo(yy, yy_1_or, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l154; yyprintf((stderr, " ok %s @ %s\n", "or", yy->__buf+yy->__pos)); return 1; - l145:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l154:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "or", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_anded(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "anded")); if (!yy_expr(yy)) goto l146; - l147:; - { int yypos148= yy->__pos, yythunkpos148= yy->__thunkpos; if (!yy_and(yy)) goto l148; if (!yy_expr(yy)) goto l148; goto l147; - l148:; yy->__pos= yypos148; yy->__thunkpos= yythunkpos148; - } if (!yy_sp(yy)) goto l146; + yyprintf((stderr, "%s\n", "anded")); if (!yy_expr(yy)) goto l155; + l156:; + { int yypos157= yy->__pos, yythunkpos157= yy->__thunkpos; if (!yy_and(yy)) goto l157; if (!yy_expr(yy)) goto l157; goto l156; + l157:; yy->__pos= yypos157; yy->__thunkpos= yythunkpos157; + } if (!yy_sp(yy)) goto l155; yyprintf((stderr, " ok %s @ %s\n", "anded", yy->__buf+yy->__pos)); return 1; - l146:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l155:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "anded", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_eol(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyprintf((stderr, "%s\n", "eol")); - { int yypos150= yy->__pos, yythunkpos150= yy->__thunkpos; if (!yymatchDot(yy)) goto l150; goto l149; - l150:; yy->__pos= yypos150; yy->__thunkpos= yythunkpos150; + { int yypos159= yy->__pos, yythunkpos159= yy->__thunkpos; if (!yymatchDot(yy)) goto l159; goto l158; + l159:; yy->__pos= yypos159; yy->__thunkpos= yythunkpos159; } yyprintf((stderr, " ok %s @ %s\n", "eol", yy->__buf+yy->__pos)); return 1; - l149:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l158:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "eol", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_ored(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "ored")); if (!yy_anded(yy)) goto l151; - l152:; - { int yypos153= yy->__pos, yythunkpos153= yy->__thunkpos; if (!yy_or(yy)) goto l153; if (!yy_anded(yy)) goto l153; goto l152; - l153:; yy->__pos= yypos153; yy->__thunkpos= yythunkpos153; - } if (!yy_sp(yy)) goto l151; + yyprintf((stderr, "%s\n", "ored")); if (!yy_anded(yy)) goto l160; + l161:; + { int yypos162= yy->__pos, yythunkpos162= yy->__thunkpos; if (!yy_or(yy)) goto l162; if (!yy_anded(yy)) goto l162; goto l161; + l162:; yy->__pos= yypos162; yy->__thunkpos= yythunkpos162; + } if (!yy_sp(yy)) goto l160; yyprintf((stderr, " ok %s @ %s\n", "ored", yy->__buf+yy->__pos)); return 1; - l151:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l160:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "ored", yy->__buf+yy->__pos)); return 0; } YY_RULE(int) yy_sp(yycontext *yy) { yyprintf((stderr, "%s\n", "sp")); - l155:; - { int yypos156= yy->__pos, yythunkpos156= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\002\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l156; goto l155; - l156:; yy->__pos= yypos156; yy->__thunkpos= yythunkpos156; + l164:; + { int yypos165= yy->__pos, yythunkpos165= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\002\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l165; goto l164; + l165:; yy->__pos= yypos165; yy->__thunkpos= yythunkpos165; } yyprintf((stderr, " ok %s @ %s\n", "sp", yy->__buf+yy->__pos)); return 1; } YY_RULE(int) yy_match(yycontext *yy) { int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; - yyprintf((stderr, "%s\n", "match")); if (!yy_sp(yy)) goto l157; if (!yy_ored(yy)) goto l157; if (!yy_eol(yy)) goto l157; yyDo(yy, yy_1_match, yy->__begin, yy->__end); + yyprintf((stderr, "%s\n", "match")); if (!yy_sp(yy)) goto l166; if (!yy_ored(yy)) goto l166; if (!yy_eol(yy)) goto l166; yyDo(yy, yy_1_match, yy->__begin, yy->__end); yyprintf((stderr, " ok %s @ %s\n", "match", yy->__buf+yy->__pos)); return 1; - l157:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + l166:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; yyprintf((stderr, " fail %s @ %s\n", "match", yy->__buf+yy->__pos)); return 0; } @@ -2259,7 +2283,7 @@ YY_PARSE(yycontext *) YYRELEASE(yycontext *yyctx) } #endif -#line 382 "heka_message_matcher_parser.leg" +#line 383 "../src/util/heka_message_matcher_parser.leg" static match_node* copy_node(unsigned char parent, match_node *mn, diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index be76063..a612c2a 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -273,10 +273,12 @@ match = sp ored eol {pop_all_ops(&yy->ctx)} ored = anded (or anded)* sp anded = expr (and expr)* sp expr = test | open ored close -test = ( string_test - | numeric_test +test = ( uuid + | timestamp + | optional_string_headers + | severity + | pid | field_test - | ts_test | boolean_test ) {push_output(&yy->ctx, &yy->ctx.mn)} sp @@ -306,14 +308,14 @@ or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp open = '(' {push_op(&yy->ctx, OP_OPEN)} sp close = ')' {pop_to_paren(&yy->ctx)} sp -string_test = string_headers sp (relational sp string_value | string_match) +optional_string_headers = string_headers sp (relational sp string_value | string_match | (op_eq | op_ne) sp nil) -string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} sp - | "Logger" {yy->ctx.mn.id = LSB_PB_LOGGER} sp - | "Hostname" {yy->ctx.mn.id = LSB_PB_HOSTNAME} sp - | "EnvVersion" {yy->ctx.mn.id = LSB_PB_ENV_VERSION} sp - | "Payload" {yy->ctx.mn.id = LSB_PB_PAYLOAD} sp - | "Uuid" {yy->ctx.mn.id = LSB_PB_UUID} sp +uuid = "Uuid" {yy->ctx.mn.id = LSB_PB_UUID} sp (relational sp string_value | string_match) +string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} + | "Logger" {yy->ctx.mn.id = LSB_PB_LOGGER} + | "Hostname" {yy->ctx.mn.id = LSB_PB_HOSTNAME} + | "EnvVersion" {yy->ctx.mn.id = LSB_PB_ENV_VERSION} + | "Payload" {yy->ctx.mn.id = LSB_PB_PAYLOAD} string_value = ( '"' < ('\\\"' | (!'"' .))* > '"' | "'" < ("\\\'" | (!"'" .))* > "'" @@ -323,10 +325,9 @@ string_match = (op_seq | op_sne) sp string_value string_match_mod? {set_match_mo string_match_mod = "%" {yy->ctx.mn.val_mod = PATTERN_MOD_ESC} -numeric_headers = "Severity" {yy->ctx.mn.id = LSB_PB_SEVERITY} sp - | "Pid" {yy->ctx.mn.id = LSB_PB_PID} sp +severity = "Severity" {yy->ctx.mn.id = LSB_PB_SEVERITY} sp relational sp numeric_value +pid = "Pid" {yy->ctx.mn.id = LSB_PB_PID} sp (relational sp numeric_value | (op_eq | op_ne) sp nil) -numeric_test = numeric_headers sp relational sp numeric_value numeric_value = < sign? number decimal? exponent? > {set_numeric_value(&yy->ctx, yytext)} sign = [-+] number = "0" @@ -344,7 +345,7 @@ zero_to_255 = "2" [0-5] [0-5] | [1-9] [0-9] | [0-9] -ts_test = ("Timestamp" sp relational sp (numeric_value | ts_quoted)) {set_timestamp(&yy->ctx)} +timestamp = ("Timestamp" sp relational sp (numeric_value | ts_quoted)) {set_timestamp(&yy->ctx)} ts_quoted = '"' rfc3339 '"' | "'" rfc3339 "'" fulldate = (y:year "-" m:month "-" d:day) {update_date(&yy->ctx, y, m, d)} year = < [0-9] [0-9] [0-9] [0-9] > {$$ = atoi(yytext)} @@ -373,7 +374,7 @@ second = ( ) {$$ = atoi(yytext)} second_frac = < decimal > {yy->ctx.mn.val.d += strtod(yytext, NULL)} -nil = "NIL" {yy->ctx.mn.val_type = TYPE_NIL;} sp +nil = "NIL" {yy->ctx.mn.val_type = TYPE_NIL} sp true = "TRUE" sp false = "FALSE" sp sp = [ \t]* diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 86b6c5d..4fff3fb 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -21,6 +21,9 @@ char pb[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x90\x01Test Payload with a longer string to attempt to create a difference in pattern match time versus the string literal match time for a unique-item\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; size_t pblen = sizeof(pb); +char pbmin[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\x00"; +size_t pbminlen = sizeof(pbmin); + static char* test_stub() { @@ -74,6 +77,7 @@ static char* test_true_matcher() , "Hostname != ''" , "Logger == 'GoSpec'" , "Pid != 0" + , "Pid != NIL" , "Severity != 5" , "Severity < 7" , "Severity <= 7" @@ -125,6 +129,13 @@ static char* test_true_matcher() , T128 , "Payload =~ 'unique-item'%" , "Fields[Timestamp] =~ ' -0700'%" + , "Type != NIL" + , "Hostname != NIL" + , "EnvVersion != NIL" + , "Pid != NIL" + , "Logger != NIL" + , "Payload != NIL" + , "Pid == 32157" , NULL }; lsb_heka_message m; @@ -141,6 +152,71 @@ static char* test_true_matcher() } +static char* test_nil_header_true_matcher() +{ + char *tests[] = { + "Type == NIL" + , "Type != ''" + , "Type < ''" + , "Type <= ''" + , "Type !~ '.'" + , "Type !~ 'foo'" + , "Hostname == NIL" + , "EnvVersion == NIL" + , "Pid == NIL" + , "Logger == NIL" + , "Payload == NIL" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 1); + mu_assert(lsb_decode_heka_message(&m, pbmin, pbminlen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m), "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_nil_header_false_matcher() +{ + char *tests[] = { + "Type != NIL" + , "Type == '%b()'" + , "Type == ''" + , "Type > ''" + , "Type >= ''" + , "Type =~ '.'" + , "Type =~ 'foo'" + , "Hostname != NIL" + , "Hostname == ''" + , "EnvVersion != NIL" + , "EnvVersion == ''" + , "Pid != NIL" + , "Logger != NIL" + , "Logger == ''" + , "Payload != NIL" + , "Payload == ''" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 1); + mu_assert(lsb_decode_heka_message(&m, pbmin, pbminlen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m) == false, "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_free_heka_message(&m); + return NULL; +} + + static char* test_false_matcher() { char *tests[] = { @@ -187,6 +263,12 @@ static char* test_false_matcher() , "Type == '" S255 "'" , "Payload =~ 'not.found'%" , "Fields[foo][1] =~ 'not.found'%" + , "Type == NIL" + , "Hostname == NIL" + , "EnvVersion == NIL" + , "Pid == NIL" + , "Logger == NIL" + , "Payload == NIL" , NULL }; lsb_heka_message m; @@ -225,8 +307,11 @@ static char* test_malformed_matcher() , "Type != 'test\"" // mis matched quote types , "Pid =~ 6" // incorrect type for the operator , "NIL" // invalid use of constant - , "Type == NIL" // existence check only works on fields - , "Fields[test] > NIL" // existence check only works with equals and not equals + , "Type > NIL" // existence check only works with equals and not equals + , "Pid > NIL" + , "Fields[test] > NIL" + , "Uuid == NIL" // required header, cannot be nil + , "Severity == NIL" // defaulted header, cannot be nil , "TRUE FALSE" // missing operator , "Timestamp == '20150411T173026'" // non rfc3339 timestamp , T128 " && Type =~ '.'" // too many tests @@ -352,7 +437,9 @@ static char* all_tests() mu_run_test(test_stub); mu_run_test(test_api_assertion); mu_run_test(test_true_matcher); + mu_run_test(test_nil_header_true_matcher); mu_run_test(test_false_matcher); + mu_run_test(test_nil_header_false_matcher); mu_run_test(test_malformed_matcher); mu_run_test(benchmark_match_hs); From 0a1ac914665ecb53068b14a8f49645112880a1f0 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 7 Sep 2018 09:57:55 -0700 Subject: [PATCH 225/235] Issue 229 - Remove old test artifact --- src/test/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 6c922c4..439ed9e 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -12,7 +12,6 @@ add_executable(test_generic_sandbox test_generic_sandbox.c) target_link_libraries(test_generic_sandbox luasandboxtest) set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/test") -add_test(NAME test_move_luasandbox_lua_modules COMMAND cmake -E copy_directory ${CMAKE_SOURCE_DIR}/modules ${CMAKE_CURRENT_BINARY_DIR}/modules) add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) add_test(NAME test_generic_sandbox COMMAND test_generic_sandbox) From 513940503aafc937d19868bb5e889bb0020cd26b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sun, 9 Sep 2018 07:13:18 -0700 Subject: [PATCH 226/235] Issue 230 - Initialize fields as needed --- CMakeLists.txt | 2 +- src/util/heka_message.c | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d885362..c5c7791 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.3.1 LANGUAGES C) +project(luasandbox VERSION 1.3.2 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/util/heka_message.c b/src/util/heka_message.c index 467a7e8..6fc8321 100644 --- a/src/util/heka_message.c +++ b/src/util/heka_message.c @@ -134,6 +134,7 @@ process_fields(lsb_heka_field *f, const char *p, const char *e) int wiretype = 0; long long vi = 0; + memset(f, 0, sizeof(lsb_heka_field)); p = lsb_pb_read_varint(p, e, &vi); if (!p || vi < 0 || vi > e - p) { return NULL; @@ -291,8 +292,8 @@ bool lsb_decode_heka_message(lsb_heka_message *m, } return false; } + // the new memory will be initialized as needed Issue #231 m->fields = tmp; - memset(&m->fields[m->fields_len], 0, step * sizeof(lsb_heka_field)); } cp = process_fields(&m->fields[m->fields_len], cp, ep); ++m->fields_len; @@ -457,7 +458,8 @@ void lsb_clear_heka_message(lsb_heka_message *m) lsb_init_const_string(&m->env_version); lsb_init_const_string(&m->hostname); - if (m->fields) memset(m->fields, 0, m->fields_size * sizeof(lsb_heka_field)); + // The fields will be cleared as they are built out anything beyond fields_len + // should be considered uninitialized Issue #231. m->timestamp = 0; m->severity = 7; m->pid = INT_MIN; @@ -469,10 +471,10 @@ void lsb_free_heka_message(lsb_heka_message *m) { if (!m) return; + lsb_clear_heka_message(m); free(m->fields); m->fields = NULL; m->fields_size = 0; - lsb_clear_heka_message(m); } From e5e923525cea9b9c21c212eb1422c1a92ce8d4ba Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 7 Nov 2018 13:00:59 -0800 Subject: [PATCH 227/235] Add sandbox documentation for _PRESERVATION_VERSION --- docs/sandbox.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/sandbox.md b/docs/sandbox.md index ab87a0d..d32839f 100644 --- a/docs/sandbox.md +++ b/docs/sandbox.md @@ -113,6 +113,24 @@ NULs terminate each argument. **Note:** To extend the function set exposed to Lua see lsb_add_function() +## Special Lua Global Variables + +- **_PRESERVATION_VERSION** (number) This variable is examined during state +restoration; if the previous version does not match the current version the +restoration is aborted and the sandbox starts cleanly. The version should be +incremented any time an incompatible change is made to the global data schema. + +When versioning is required, the use of a configuration variable name +`preservation_version` with the following syntax is recommended. +```lua +-- initial (allows the user to bump the version due to a cfg change) +_PRESERVATION_VERSION = read_config("preservation_version") or 0 + +-- if the plugin code alters the global data schema in an incompatible way this +-- ensures the state is properly cleared when there is a user provided version. +_PRESERVATION_VERSION = (read_config("preservation_version") or 0) + 1 +``` + ## How to interact with the sandbox (creating an API) The best place to start is with some examples: From a0728c0ca4705cd9243a187b9ae5ec9f1dfd1f71 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Tue, 20 Nov 2018 10:50:25 -0800 Subject: [PATCH 228/235] Update doxygen configuration --- cmake/doxygen.cmake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmake/doxygen.cmake b/cmake/doxygen.cmake index 90749a2..280fec8 100644 --- a/cmake/doxygen.cmake +++ b/cmake/doxygen.cmake @@ -4,6 +4,7 @@ find_package(Doxygen QUIET) find_program(LUA_EXE lua QUIET) +find_program(PANDOC_EXE pandoc QUIET) find_program(gitbook gitbook QUIET) if(DOXYGEN_FOUND AND LUA_EXE AND PANDOC_EXE) set(DOXYCONF_IN ${CMAKE_SOURCE_DIR}/doxygen.in.conf) @@ -14,12 +15,12 @@ if(DOXYGEN_FOUND AND LUA_EXE AND PANDOC_EXE) file(WRITE ${DOXYCONF_OUT} " PROJECT_NAME = \"${PROJECT_NAME}\" PROJECT_BRIEF = \"${CPACK_PACKAGE_DESCRIPTION_SUMMARY}\" -OUTPUT_DIRECTORY = \"${CMAKE_SOURCE_DIR}/gh-pages\" +OUTPUT_DIRECTORY = \"${CMAKE_SOURCE_DIR}/gh-pages\" HTML_OUTPUT = doxygen GENERATE_LATEX = NO GENERATE_TODOLIST = YES FULL_PATH_NAMES = YES -STRIP_FROPATH = \"${CMAKE_SOURCE_DIR}\" +STRIP_FROM_PATH = \"${CMAKE_SOURCE_DIR}\" SOURCE_BROWSER = YES TAB_SIZE = 4 EXTRACT_ALL = YES From 989f0ba6a6bbdefeeeddd51964efee36e2e3f7a5 Mon Sep 17 00:00:00 2001 From: Mozilla-GitHub-Standards <48073334+Mozilla-GitHub-Standards@users.noreply.github.com> Date: Wed, 27 Mar 2019 19:25:30 -0700 Subject: [PATCH 229/235] Add Mozilla Code of Conduct file Fixes #233. _(Message COC002)_ --- CODE_OF_CONDUCT.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 CODE_OF_CONDUCT.md diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..498baa3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,15 @@ +# Community Participation Guidelines + +This repository is governed by Mozilla's code of conduct and etiquette guidelines. +For more details, please read the +[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). + +## How to Report +For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. + + From 55ca254db047d233c76c162538cdca690c4ba78a Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Thu, 28 Mar 2019 09:07:40 -0700 Subject: [PATCH 230/235] Remove debian 7 from the travis build --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1f1a0dd..c5cadb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,6 @@ env: matrix: - DOCKER_IMAGE=centos:6 CMAKE_URL=auto - DOCKER_IMAGE=centos:7 CMAKE_URL=auto - - DOCKER_IMAGE=debian:7 CMAKE_URL=auto - DOCKER_IMAGE=debian:8 CMAKE_URL=auto - DOCKER_IMAGE=debian:8 CC=clang CMAKE_URL=auto - DOCKER_IMAGE=debian:stretch From b1e26a3d4016ab44911aedbb1242c2516ee0b93b Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Fri, 10 May 2019 18:10:20 -0700 Subject: [PATCH 231/235] Issue 235 - Allow the backslash to be escaped in the message matcher An escaped backslash is necessary if a matcher string ends in backslash otherwise it will simply be passed through as is. However, this change breaks existing matcher strings with two consecutive backslashes, they now have to be double escaped. e.g. "\\\\foo\\bar" would become "\\\\\\\\foo\\bar" The second backslash does not need escaping since "\b" is not special in a matcher string. --- CMakeLists.txt | 2 +- src/util/heka_message_matcher_parser.c | 106 +++++++++++----------- src/util/heka_message_matcher_parser.leg | 14 +-- src/util/test/test_heka_message_matcher.c | 4 + 4 files changed, 67 insertions(+), 59 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c5c7791..be150c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.3.2 LANGUAGES C) +project(luasandbox VERSION 1.3.3 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index 0b98f63..1b0275c 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -231,7 +231,8 @@ static void set_string_value(context *ctx, char *s) ctx->mn.val_type = TYPE_STRING; int i, j; for (i = 0, j = 0; s[i]; ++i, ++j) { - if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { ++i; } s[j] = s[i]; @@ -261,7 +262,8 @@ static bool check_string_len(char *s) { int i, j; for (i = 0, j = 0; s[i]; ++i, ++j) { - if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { ++i; } } @@ -609,7 +611,7 @@ YY_ACTION(void) yy_1_nil(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_nil\n")); { -#line 377 +#line 379 yy->ctx.mn.val_type = TYPE_NIL; } #undef yythunkpos @@ -623,7 +625,7 @@ YY_ACTION(void) yy_1_second_frac(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second_frac\n")); { -#line 375 +#line 377 yy->ctx.mn.val.d += strtod(yytext, NULL); } #undef yythunkpos @@ -637,7 +639,7 @@ YY_ACTION(void) yy_1_second(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_second\n")); { -#line 374 +#line 376 __ = atoi(yytext); } #undef yythunkpos @@ -651,7 +653,7 @@ YY_ACTION(void) yy_1_minute(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_minute\n")); { -#line 370 +#line 372 __ = atoi(yytext); } #undef yythunkpos @@ -665,7 +667,7 @@ YY_ACTION(void) yy_1_hour(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_hour\n")); { -#line 369 +#line 371 __ = atoi(yytext); } #undef yythunkpos @@ -681,7 +683,7 @@ YY_ACTION(void) yy_2_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_timeoffset\n")); { -#line 365 +#line 367 update_offset(&yy->ctx, yytext[0], h, m); } #undef yythunkpos @@ -699,7 +701,7 @@ YY_ACTION(void) yy_1_timeoffset(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_timeoffset\n")); { -#line 364 +#line 366 update_offset(&yy->ctx, '+', 0, 0); } #undef yythunkpos @@ -718,7 +720,7 @@ YY_ACTION(void) yy_1_partialtime(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_partialtime\n")); { -#line 363 +#line 365 update_time(&yy->ctx, h, m, s); } #undef yythunkpos @@ -735,7 +737,7 @@ YY_ACTION(void) yy_1_day(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_day\n")); { -#line 359 +#line 361 __ = atoi(yytext); } #undef yythunkpos @@ -749,7 +751,7 @@ YY_ACTION(void) yy_1_month(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_month\n")); { -#line 354 +#line 356 __ = atoi(yytext); } #undef yythunkpos @@ -763,7 +765,7 @@ YY_ACTION(void) yy_1_year(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_year\n")); { -#line 351 +#line 353 __ = atoi(yytext); } #undef yythunkpos @@ -780,7 +782,7 @@ YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fulldate\n")); { -#line 350 +#line 352 update_date(&yy->ctx, y, m, d); } #undef yythunkpos @@ -797,7 +799,7 @@ YY_ACTION(void) yy_1_timestamp(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_timestamp\n")); { -#line 348 +#line 350 set_timestamp(&yy->ctx); } #undef yythunkpos @@ -811,7 +813,7 @@ YY_ACTION(void) yy_1_index(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_index\n")); { -#line 342 +#line 344 __ = atoi(yytext); } #undef yythunkpos @@ -827,7 +829,7 @@ YY_ACTION(void) yy_3_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_fields\n")); { -#line 341 +#line 343 yy->ctx.mn.ai = a; } #undef yythunkpos @@ -845,7 +847,7 @@ YY_ACTION(void) yy_2_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_fields\n")); { -#line 341 +#line 343 yy->ctx.mn.fi = f; } #undef yythunkpos @@ -863,7 +865,7 @@ YY_ACTION(void) yy_1_fields(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_fields\n")); { -#line 341 +#line 343 set_field(&yy->ctx, yytext); } #undef yythunkpos @@ -879,7 +881,7 @@ YY_ACTION(void) yy_1_numeric_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_numeric_value\n")); { -#line 331 +#line 333 set_numeric_value(&yy->ctx, yytext); } #undef yythunkpos @@ -893,7 +895,7 @@ YY_ACTION(void) yy_1_pid(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_pid\n")); { -#line 329 +#line 331 yy->ctx.mn.id = LSB_PB_PID; } #undef yythunkpos @@ -907,7 +909,7 @@ YY_ACTION(void) yy_1_severity(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_severity\n")); { -#line 328 +#line 330 yy->ctx.mn.id = LSB_PB_SEVERITY; } #undef yythunkpos @@ -921,7 +923,7 @@ YY_ACTION(void) yy_1_string_match_mod(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_match_mod\n")); { -#line 326 +#line 328 yy->ctx.mn.val_mod = PATTERN_MOD_ESC; } #undef yythunkpos @@ -935,7 +937,7 @@ YY_ACTION(void) yy_1_string_match(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_match\n")); { -#line 324 +#line 326 set_match_mod(&yy->ctx); } #undef yythunkpos @@ -949,7 +951,7 @@ YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_value\n")); { -#line 322 +#line 324 set_string_value(&yy->ctx, yytext); } #undef yythunkpos @@ -963,7 +965,7 @@ YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_5_string_headers\n")); { -#line 318 +#line 320 yy->ctx.mn.id = LSB_PB_PAYLOAD; } #undef yythunkpos @@ -977,7 +979,7 @@ YY_ACTION(void) yy_4_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_4_string_headers\n")); { -#line 317 +#line 319 yy->ctx.mn.id = LSB_PB_ENV_VERSION; } #undef yythunkpos @@ -991,7 +993,7 @@ YY_ACTION(void) yy_3_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_3_string_headers\n")); { -#line 316 +#line 318 yy->ctx.mn.id = LSB_PB_HOSTNAME; } #undef yythunkpos @@ -1005,7 +1007,7 @@ YY_ACTION(void) yy_2_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_string_headers\n")); { -#line 315 +#line 317 yy->ctx.mn.id = LSB_PB_LOGGER; } #undef yythunkpos @@ -1019,7 +1021,7 @@ YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_string_headers\n")); { -#line 314 +#line 316 yy->ctx.mn.id = LSB_PB_TYPE; } #undef yythunkpos @@ -1033,7 +1035,7 @@ YY_ACTION(void) yy_1_uuid(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_uuid\n")); { -#line 313 +#line 315 yy->ctx.mn.id = LSB_PB_UUID; } #undef yythunkpos @@ -1047,7 +1049,7 @@ YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_close\n")); { -#line 309 +#line 311 pop_to_paren(&yy->ctx); } #undef yythunkpos @@ -1061,7 +1063,7 @@ YY_ACTION(void) yy_1_open(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_open\n")); { -#line 308 +#line 310 push_op(&yy->ctx, OP_OPEN); } #undef yythunkpos @@ -1075,7 +1077,7 @@ YY_ACTION(void) yy_1_or(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_or\n")); { -#line 307 +#line 309 push_op(&yy->ctx, OP_OR); } #undef yythunkpos @@ -1089,7 +1091,7 @@ YY_ACTION(void) yy_1_and(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_and\n")); { -#line 306 +#line 308 push_op(&yy->ctx, OP_AND); } #undef yythunkpos @@ -1103,7 +1105,7 @@ YY_ACTION(void) yy_2_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean\n")); { -#line 304 +#line 306 yy->ctx.mn.val_type = TYPE_FALSE; } #undef yythunkpos @@ -1117,7 +1119,7 @@ YY_ACTION(void) yy_1_boolean(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean\n")); { -#line 303 +#line 305 yy->ctx.mn.val_type = TYPE_TRUE; } #undef yythunkpos @@ -1131,7 +1133,7 @@ YY_ACTION(void) yy_2_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_2_boolean_test\n")); { -#line 302 +#line 304 yy->ctx.mn.op = OP_FALSE; } #undef yythunkpos @@ -1145,7 +1147,7 @@ YY_ACTION(void) yy_1_boolean_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_boolean_test\n")); { -#line 301 +#line 303 yy->ctx.mn.op = OP_TRUE; } #undef yythunkpos @@ -1159,7 +1161,7 @@ YY_ACTION(void) yy_1_op_lt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lt\n")); { -#line 292 +#line 294 yy->ctx.mn.op = OP_LT; } #undef yythunkpos @@ -1173,7 +1175,7 @@ YY_ACTION(void) yy_1_op_lte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_lte\n")); { -#line 291 +#line 293 yy->ctx.mn.op = OP_LTE; } #undef yythunkpos @@ -1187,7 +1189,7 @@ YY_ACTION(void) yy_1_op_gt(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gt\n")); { -#line 290 +#line 292 yy->ctx.mn.op = OP_GT; } #undef yythunkpos @@ -1201,7 +1203,7 @@ YY_ACTION(void) yy_1_op_gte(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_gte\n")); { -#line 289 +#line 291 yy->ctx.mn.op = OP_GTE; } #undef yythunkpos @@ -1215,7 +1217,7 @@ YY_ACTION(void) yy_1_op_sne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_sne\n")); { -#line 288 +#line 290 yy->ctx.mn.op = OP_NRE; } #undef yythunkpos @@ -1229,7 +1231,7 @@ YY_ACTION(void) yy_1_op_seq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_seq\n")); { -#line 287 +#line 289 yy->ctx.mn.op = OP_RE; } #undef yythunkpos @@ -1243,7 +1245,7 @@ YY_ACTION(void) yy_1_op_ne(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_ne\n")); { -#line 286 +#line 288 yy->ctx.mn.op = OP_NE; } #undef yythunkpos @@ -1257,7 +1259,7 @@ YY_ACTION(void) yy_1_op_eq(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_op_eq\n")); { -#line 285 +#line 287 yy->ctx.mn.op = OP_EQ; } #undef yythunkpos @@ -1271,7 +1273,7 @@ YY_ACTION(void) yy_1_test(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_test\n")); { -#line 283 +#line 285 push_output(&yy->ctx, &yy->ctx.mn); } #undef yythunkpos @@ -1285,7 +1287,7 @@ YY_ACTION(void) yy_1_match(yycontext *yy, char *yytext, int yyleng) #define yythunkpos yy->__thunkpos yyprintf((stderr, "do yy_1_match\n")); { -#line 272 +#line 274 pop_all_ops(&yy->ctx); } #undef yythunkpos @@ -1800,7 +1802,7 @@ if (!(YY_BEGIN)) goto l72; } l73:; { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; - { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; if (!yymatchString(yy, "\\\"")) goto l76; goto l75; + { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; if (!yymatchChar(yy, '\\')) goto l76; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\004\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l76; goto l75; l76:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l77; goto l74; l77:; yy->__pos= yypos77; yy->__thunkpos= yythunkpos77; @@ -1824,7 +1826,7 @@ if (!(YY_BEGIN)) goto l70; } l78:; { int yypos79= yy->__pos, yythunkpos79= yy->__thunkpos; - { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchString(yy, "\\\'")) goto l81; goto l80; + { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchChar(yy, '\\')) goto l81; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\200\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l81; goto l80; l81:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l82; goto l79; l82:; yy->__pos= yypos82; yy->__thunkpos= yythunkpos82; @@ -2283,7 +2285,7 @@ YY_PARSE(yycontext *) YYRELEASE(yycontext *yyctx) } #endif -#line 383 "../src/util/heka_message_matcher_parser.leg" +#line 385 "../src/util/heka_message_matcher_parser.leg" static match_node* copy_node(unsigned char parent, match_node *mn, diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index a612c2a..68309f8 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -224,7 +224,8 @@ static void set_string_value(context *ctx, char *s) ctx->mn.val_type = TYPE_STRING; int i, j; for (i = 0, j = 0; s[i]; ++i, ++j) { - if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { ++i; } s[j] = s[i]; @@ -254,7 +255,8 @@ static bool check_string_len(char *s) { int i, j; for (i = 0, j = 0; s[i]; ++i, ++j) { - if (s[i] == '\\' && (s[i + 1] == '"' || s[i + 1] == '\'')) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { ++i; } } @@ -305,8 +307,8 @@ boolean = true {yy->ctx.mn.val_type = TYPE_TRUE} and = "&&" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_AND)} sp or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp -open = '(' {push_op(&yy->ctx, OP_OPEN)} sp -close = ')' {pop_to_paren(&yy->ctx)} sp +open = "(" {push_op(&yy->ctx, OP_OPEN)} sp +close = ")" {pop_to_paren(&yy->ctx)} sp optional_string_headers = string_headers sp (relational sp string_value | string_match | (op_eq | op_ne) sp nil) @@ -317,8 +319,8 @@ string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} | "EnvVersion" {yy->ctx.mn.id = LSB_PB_ENV_VERSION} | "Payload" {yy->ctx.mn.id = LSB_PB_PAYLOAD} -string_value = ( '"' < ('\\\"' | (!'"' .))* > '"' - | "'" < ("\\\'" | (!"'" .))* > "'" +string_value = ( '"' < (("\\" ["\\]) | (!'"' .))* > '"' + | "'" < (("\\" ['\\]) | (!"'" .))* > "'" ) &{check_string_len(yytext)} {set_string_value(&yy->ctx, yytext)} string_match = (op_seq | op_sne) sp string_value string_match_mod? {set_match_mod(&yy->ctx)} diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 4fff3fb..5df35b0 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -136,6 +136,8 @@ static char* test_true_matcher() , "Logger != NIL" , "Payload != NIL" , "Pid == 32157" + , "Uuid < '\\\\'" + , "Uuid < \"\\\\\"" , NULL }; lsb_heka_message m; @@ -269,6 +271,7 @@ static char* test_false_matcher() , "Pid == NIL" , "Logger == NIL" , "Payload == NIL" + , "Uuid > '\\\\'" , NULL }; lsb_heka_message m; @@ -319,6 +322,7 @@ static char* test_malformed_matcher() , "Fields[test][256] == 1" // field index out of bounds , "Fields[test][0][256] == 1" // array index out of bounds , "Payload =~ 'foo'i" // invalid string match pattern modifier + , "Uuid < '\\'" // unescaped backslash leaving an open string '\' , NULL }; lsb_heka_message m; From 2824589f9ec1d8c9ec1a7dd831fc52f0629a1b55 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Sat, 1 Jun 2019 10:49:03 -0700 Subject: [PATCH 232/235] Fixes for BSD --- .travis.yml | 2 -- cmake/mozsvc.cmake | 2 +- src/cli/lsb_heka_cat.c | 6 +++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index c5cadb3..b48dd37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,13 +5,11 @@ services: env: matrix: - - DOCKER_IMAGE=centos:6 CMAKE_URL=auto - DOCKER_IMAGE=centos:7 CMAKE_URL=auto - DOCKER_IMAGE=debian:8 CMAKE_URL=auto - DOCKER_IMAGE=debian:8 CC=clang CMAKE_URL=auto - DOCKER_IMAGE=debian:stretch - DOCKER_IMAGE=fedora:latest - - DOCKER_IMAGE=ubuntu:12.04 CMAKE_URL=auto - DOCKER_IMAGE=ubuntu:latest CMAKE_URL=auto # LTS - DOCKER_IMAGE=ubuntu:devel diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 379ce7c..2f62115 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -24,7 +24,7 @@ if(MSVC) else() # Predefined Macros: clang|gcc -dM -E -x c /dev/null # Compiler options: http://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html#Invoking-GCC - set(CMAKE_C_FLAGS "-std=gnu99 -pedantic $ENV{CFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -fvisibility=hidden") + set(CMAKE_C_FLAGS "-std=gnu11 -pedantic $ENV{CFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -fvisibility=hidden") set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic $ENV{CXXFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") set(CMAKE_C_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c index 5f6eb86..0743ca0 100644 --- a/src/cli/lsb_heka_cat.c +++ b/src/cli/lsb_heka_cat.c @@ -223,11 +223,15 @@ static size_t read_file(FILE *fh, lsb_input_buffer *ib) log_cb(NULL, NULL, 0, "buffer reallocation failed"); exit(EXIT_FAILURE); } + size_t cnt = ib->size - ib->readpos; size_t nread = fread(ib->buf + ib->readpos, 1, - ib->size - ib->readpos, + cnt, fh); ib->readpos += nread; + if (cnt != nread) { + clearerr(fh); + } return nread; } From 3d15a102ae92a6e7a262431e317da4030b2e6190 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Wed, 31 Jul 2019 08:55:16 -0700 Subject: [PATCH 233/235] Issue 238 - Allow multi line message matchers --- CMakeLists.txt | 2 +- src/util/heka_message_matcher_parser.c | 2 +- src/util/heka_message_matcher_parser.leg | 2 +- src/util/test/test_heka_message_matcher.c | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index be150c5..bd208f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.3.3 LANGUAGES C) +project(luasandbox VERSION 1.3.4 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c index 1b0275c..d9d54ef 100644 --- a/src/util/heka_message_matcher_parser.c +++ b/src/util/heka_message_matcher_parser.c @@ -2221,7 +2221,7 @@ YY_RULE(int) yy_sp(yycontext *yy) { yyprintf((stderr, "%s\n", "sp")); l164:; - { int yypos165= yy->__pos, yythunkpos165= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\002\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l165; goto l164; + { int yypos165= yy->__pos, yythunkpos165= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\046\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l165; goto l164; l165:; yy->__pos= yypos165; yy->__thunkpos= yythunkpos165; } yyprintf((stderr, " ok %s @ %s\n", "sp", yy->__buf+yy->__pos)); diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg index 68309f8..0c46d3c 100644 --- a/src/util/heka_message_matcher_parser.leg +++ b/src/util/heka_message_matcher_parser.leg @@ -379,7 +379,7 @@ second_frac = < decimal > {yy->ctx.mn.val.d += strtod(yytext, NULL)} nil = "NIL" {yy->ctx.mn.val_type = TYPE_NIL} sp true = "TRUE" sp false = "FALSE" sp -sp = [ \t]* +sp = [ \r\n\t]* eol = !. %% diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c index 5df35b0..899039d 100644 --- a/src/util/test/test_heka_message_matcher.c +++ b/src/util/test/test_heka_message_matcher.c @@ -138,6 +138,7 @@ static char* test_true_matcher() , "Pid == 32157" , "Uuid < '\\\\'" , "Uuid < \"\\\\\"" + , "(Severity == 7 || Logger == 'GoSpec') \r\n\t&& Type == 'TEST'" , NULL }; lsb_heka_message m; From 0de27ea5329b010c9e12af523c9e032b39372958 Mon Sep 17 00:00:00 2001 From: Mike Trinkala Date: Mon, 30 Dec 2019 12:42:47 -0800 Subject: [PATCH 234/235] Add inject_message to the Heka output plugin API Allows for better error logging, alerting see https://github.com/mozilla-services/lua_sandbox/issues/240 --- CMakeLists.txt | 2 +- docs/heka/output.md | 22 +++++++--- include/luasandbox/heka/sandbox.h | 24 ++++++++++ src/heka/sandbox.c | 27 ++++++++++-- src/heka/sandbox_impl.h | 6 +-- src/heka/test/lua/oim.lua | 25 +++++++++++ src/heka/test/test_heka_sandbox.c | 73 ++++++++++++++++++++++++++++--- 7 files changed, 159 insertions(+), 20 deletions(-) create mode 100644 src/heka/test/lua/oim.lua diff --git a/CMakeLists.txt b/CMakeLists.txt index bd208f3..84825bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. cmake_minimum_required(VERSION 3.6 FATAL_ERROR) -project(luasandbox VERSION 1.3.4 LANGUAGES C) +project(luasandbox VERSION 1.4.0 LANGUAGES C) set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) diff --git a/docs/heka/output.md b/docs/heka/output.md index dfb5b9c..c58a20e 100644 --- a/docs/heka/output.md +++ b/docs/heka/output.md @@ -78,11 +78,11 @@ Converts a Heka protobuf encoded message string into a Lua table. See ### encode_message Returns a Heka protocol buffer message using the contents of the specified Lua -table. `Logger` and `Hostname` are restricted header values. An override -configuration option is provided `restricted_headers`; when true the headers are -always set to the configuration values; when false (default) the headers are set -to the values provide in the message table, if no value is provided it defaults -to the appropriate configuration value. +table. `Timestamp`, `Logger`, `Hostname` and `Pid` are restricted header values. +An override configuration option is provided `restricted_headers`; when true the +headers are always set to the configuration values; when false (default) the +headers are set to the values provided in the message table, if no value is +provided it defaults to the appropriate configuration value. Note: this operation uses the internal output buffer so it is goverened by the `output_limit` configuration setting. @@ -95,6 +95,18 @@ Note: this operation uses the internal output buffer so it is goverened by the * heka_pb (string) - Heka protobuf binary string, framed as specified or an error is thrown +### inject_message + +Creates a new Heka protocol buffer message, in the input queue, using the +contents of the specified Lua table. The `restricted_headers` configuration +defaults to false (see encode_message above for a full description). + +*Arguments* +* msg ([Heka message table](message.md)) + +*Return* +* none (throws an error if the table does not match the Heka message schema) + ### create_message_matcher Returns a Heka protocol buffer message matcher; used to dynamically filter diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h index da0d6d7..b524f52 100644 --- a/include/luasandbox/heka/sandbox.h +++ b/include/luasandbox/heka/sandbox.h @@ -228,6 +228,30 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_logger *logger, lsb_heka_update_checkpoint ucp); +/** + * Create a sandbox supporting the Heka Output Plugin API with + * inject_message support + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified path to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param ucp checkpoint_updated callback when using batch or async output + * @param im inject_message callback + * + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_HEKA_EXPORT +lsb_heka_sandbox* lsb_heka_create_output_im(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_update_checkpoint ucp, + lsb_heka_im_analysis im); /** * Host access to the output sandbox process_message API * diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c index 55504ae..17cd176 100644 --- a/src/heka/sandbox.c +++ b/src/heka/sandbox.c @@ -344,10 +344,10 @@ static int update_checkpoint(lua_State *lua) // fall thru case 1: luaL_checktype(lua, 1, LUA_TLIGHTUSERDATA); - result = hsb->cb.ucp(hsb->parent, lua_touserdata(lua, 1)); + result = hsb->ucp(hsb->parent, lua_touserdata(lua, 1)); break; case 0: // batch case - result = hsb->cb.ucp(hsb->parent, NULL); + result = hsb->ucp(hsb->parent, NULL); break; default: return luaL_error(lua, "%s() invalid number of args: %d", __func__, n); @@ -876,6 +876,19 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, const char *lsb_cfg, lsb_logger *logger, lsb_heka_update_checkpoint ucp) +{ + return lsb_heka_create_output_im(parent, lua_file, state_file, lsb_cfg, + logger, ucp, NULL); +} + + +lsb_heka_sandbox* lsb_heka_create_output_im(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_update_checkpoint ucp, + lsb_heka_im_analysis im) { if (!lua_file) { if (logger && logger->cb) { @@ -892,7 +905,6 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, return NULL; } - lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); if (!hsb) { if (logger && logger->cb) { @@ -904,7 +916,8 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, hsb->type = 'o'; hsb->parent = parent; hsb->msg = NULL; - hsb->cb.ucp = ucp; + hsb->ucp = ucp; + hsb->cb.aim = im; hsb->name = NULL; hsb->hostname = NULL; @@ -922,6 +935,11 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); lsb_add_function(hsb->lsb, update_checkpoint, LSB_HEKA_UPDATE_CHECKPOINT); lsb_add_function(hsb->lsb, mm_create, "create_message_matcher"); + if (im) { + lsb_add_function(hsb->lsb, inject_message_analysis, "inject_message"); + // inject_payload is intentionally excluded from output plugins + // you can construct whatever you need with inject_message + } // start io.write override with zero copy functionality lua_getfield(lua, LUA_REGISTRYINDEX, "_PRELOADED"); @@ -950,6 +968,7 @@ lsb_heka_sandbox* lsb_heka_create_output(void *parent, } + void lsb_heka_stop_sandbox_clean(lsb_heka_sandbox *hsb) { lsb_stop_sandbox_clean(hsb->lsb); diff --git a/src/heka/sandbox_impl.h b/src/heka/sandbox_impl.h index c15a740..977516c 100644 --- a/src/heka/sandbox_impl.h +++ b/src/heka/sandbox_impl.h @@ -33,14 +33,14 @@ struct lsb_heka_sandbox { char *name; char *hostname; union { - lsb_heka_im_input iim; - lsb_heka_im_analysis aim; - lsb_heka_update_checkpoint ucp; + lsb_heka_im_input iim; // used in input plugins only + lsb_heka_im_analysis aim; // used in analysis and output plugins } cb; struct heka_stats stats; char type; bool restricted_headers; int pid; + lsb_heka_update_checkpoint ucp; // used in output plugins only }; #endif diff --git a/src/heka/test/lua/oim.lua b/src/heka/test/lua/oim.lua new file mode 100644 index 0000000..68435a9 --- /dev/null +++ b/src/heka/test/lua/oim.lua @@ -0,0 +1,25 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +-- Table tests +local msgs = { + {Timestamp = 1, Uuid = "\001\002\003\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {Pid = 2, Timestamp = 3, Uuid = "\004\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "ignore", Hostname = "spoof"}, +} + +local err_msgs = { + {err = "bad argument #1 to '?' (table expected, got nil)"}, +} + +for i, v in ipairs(msgs) do + inject_message(v) +end + +for i, v in ipairs(err_msgs) do + local ok, err = pcall(inject_message, v.msg) + if ok then error(string.format("test: %d should have failed", i)) end + assert(v.err == err, string.format("test: %d expected: %s received: %s", i, v.err, err)) +end diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c index 4ed1680..a69f363 100644 --- a/src/heka/test/test_heka_sandbox.c +++ b/src/heka/test/test_heka_sandbox.c @@ -133,16 +133,14 @@ static int aim(void *parent, const char *pb, size_t pb_len) struct im_result { const char *pb; size_t pb_len; - double cp_numeric; - const char *cp_string; }; struct im_result results[] = { - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x8e\xa8\xf3\xde\x88\xb5\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 44, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\xbf\x97\x9c\xcc\xbe\xc2\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x00\x00\x00\x00\x0a\x10\x00\x00", .pb_len = 44, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\xea\x95\xd4\xfc\x7c\x10\x40\x95\xa8\x17\xcb\x56\x26\x91\x8c\x47\x10\xba\xf6\xd5\xf9\xfd\xce\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 90, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\xfd\x49\x92\x77\x02\x37\x4b\xf0\xaf\x86\x6f\x9b\x80\x26\xf4\x35\x10\xaf\xec\x9e\xa4\xd8\xcf\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 112, .cp_numeric = NAN, .cp_string = NULL }, - { .pb = "\x0a\x10\x7c\x32\xd6\x23\x98\xe8\x49\x9e\xa2\xe8\x0d\x78\x84\x8e\x75\xb2\x10\xf7\xf5\xdb\x89\x88\xe4\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 0, .cp_numeric = NAN, .cp_string = NULL }, }; // intentionally fail on size to to test the custom return value + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x8e\xa8\xf3\xde\x88\xb5\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 44}, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\xbf\x97\x9c\xcc\xbe\xc2\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x00\x00\x00\x00\x0a\x10\x00\x00", .pb_len = 44}, + { .pb = "\x0a\x10\xea\x95\xd4\xfc\x7c\x10\x40\x95\xa8\x17\xcb\x56\x26\x91\x8c\x47\x10\xba\xf6\xd5\xf9\xfd\xce\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 90}, + { .pb = "\x0a\x10\xfd\x49\x92\x77\x02\x37\x4b\xf0\xaf\x86\x6f\x9b\x80\x26\xf4\x35\x10\xaf\xec\x9e\xa4\xd8\xcf\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 112}, + { .pb = "\x0a\x10\x7c\x32\xd6\x23\x98\xe8\x49\x9e\xa2\xe8\x0d\x78\x84\x8e\x75\xb2\x10\xf7\xf5\xdb\x89\x88\xe4\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 0}, }; // intentionally fail on size to to test the custom return value if (cnt >= (int)(sizeof results / sizeof results[0])) { fprintf(stderr, "tests and results are mis-matched\n"); @@ -225,6 +223,51 @@ static int ucp(void *parent, void *sequence_id) } +static int oim(void *parent, const char *pb, size_t pb_len) +{ + static int cnt = 0; + struct im_result { + const char *pb; + size_t pb_len; + }; + + struct im_result results[] = { + { .pb = "\x0a\x10\x01\x02\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03\x6f\x69\x6d\x40\x00\x4a\x04\x74\x65\x73\x74", .pb_len = 33 }, + { .pb = "\x0a\x10\x04\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x06\x69\x67\x6e\x6f\x72\x65\x40\x02\x4a\x05\x73\x70\x6f\x6f\x66", .pb_len = 37 }, }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return LSB_HEKA_IM_LIMIT; + } + + if (parent) { + fprintf(stderr, "test: %d parent set\n", cnt); + } + + if (pb_len != results[cnt].pb_len) { + fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" + PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); + cnt++; + return 99; + } + + if (memcmp(pb, results[cnt].pb, pb_len)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = 0; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); + } + fprintf(stderr, "\nreceived: "); + for (size_t i = 0; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; + } + cnt++; + return LSB_HEKA_IM_SUCCESS; +} + + static char* test_api_assertion() { lsb_heka_message m; @@ -621,6 +664,21 @@ static char* test_im_analysis() } +static char* test_im_output() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output_im(NULL, "lua/oim.lua", NULL, + "Hostname = 'test';Logger = 'oim'", + &logger, ucp, oim); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(2 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(70 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(hsb, "lsb_heka_create_input failed"); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + static char* test_encode_message() { lsb_heka_sandbox *hsb; @@ -808,6 +866,7 @@ static char* all_tests() mu_run_test(test_pm_output); mu_run_test(test_im_input); mu_run_test(test_im_analysis); + mu_run_test(test_im_output); mu_run_test(test_encode_message); mu_run_test(test_decode_message); mu_run_test(test_read_message); From 610855db25becb8f31dfd849873072c424e2b10c Mon Sep 17 00:00:00 2001 From: nikolas Date: Thu, 2 Jul 2020 17:10:59 -0400 Subject: [PATCH 235/235] Fix spelling error: decople -> decouple (#242) * Fix spelling error: decople -> decouple * Fix spelling: retrive -> retrieve --- docs/heka/index.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/heka/index.md b/docs/heka/index.md index 188d272..1fcd83d 100644 --- a/docs/heka/index.md +++ b/docs/heka/index.md @@ -5,7 +5,7 @@ This document describes the 1.0 release of the Heka sandbox API built on the [Generic Sandbox Interface](/sandbox.md). The 1.0 release is a refined implementation of its predecessor which was developed in -[Heka](https://github.com/mozilla-services/heka). The goal is to decople it from +[Heka](https://github.com/mozilla-services/heka). The goal is to decouple it from Go and make it easily embeddable in any language. The Go version of Heka has been deprecated and replaced by [Hindsight](https://github.com/mozilla-services/hindsight). @@ -35,8 +35,8 @@ support skipping, retrying, batching, and asynchronous output. 1. `read_message` * returns `nil` for optional header fields if they don't exist instead of an empty string or zero - * added a `framed` parameter to retrive the raw message with stream framing - * added a `size` parameter to retrive size of the raw message without having + * added a `framed` parameter to retrieve the raw message with stream framing + * added a `size` parameter to retrieve size of the raw message without having to copy it down 1. `timer_event` has a second parameter `shutdown` that is set to true when the sandbox is exiting.