diff --git a/.gitignore b/.gitignore index 8a24132..201bee5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ *.preserve release/ +src/test/modules/ +Testing +*~ +gh-pages/ diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..b48dd37 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,17 @@ +language: generic +sudo: required +services: + - docker + +env: + matrix: + - DOCKER_IMAGE=centos:7 CMAKE_URL=auto + - DOCKER_IMAGE=debian:8 CMAKE_URL=auto + - DOCKER_IMAGE=debian:8 CC=clang CMAKE_URL=auto + - DOCKER_IMAGE=debian:stretch + - DOCKER_IMAGE=fedora:latest + - DOCKER_IMAGE=ubuntu:latest CMAKE_URL=auto # LTS + - DOCKER_IMAGE=ubuntu:devel + +script: + - ./build/run.sh build diff --git a/CMakeLists.txt b/CMakeLists.txt index 00303f1..84825bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,25 +2,53 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -project(lua_sandbox C) +cmake_minimum_required(VERSION 3.6 FATAL_ERROR) +project(luasandbox VERSION 1.4.0 LANGUAGES C) + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Generic Lua sandbox for dynamic data analysis") -set(CPACK_PACKAGE_VERSION_MAJOR 0) -set(CPACK_PACKAGE_VERSION_MINOR 7) -set(CPACK_PACKAGE_VERSION_PATCH 1) +set(CPACK_PACKAGE_VERSION_MAJOR ${PROJECT_VERSION_MAJOR}) +set(CPACK_PACKAGE_VERSION_MINOR ${PROJECT_VERSION_MINOR}) +set(CPACK_PACKAGE_VERSION_PATCH ${PROJECT_VERSION_PATCH}) +set(CPACK_PACKAGE_CONTACT "Mike Trinkala ") +set(CPACK_DEBIAN_FILE_NAME "DEB-DEFAULT") +set(CPACK_RPM_FILE_NAME "RPM-DEFAULT") +set(CPACK_RPM_PACKAGE_LICENSE "MPLv2.0") -option(LUA_JIT "Enable LuaJIT" off) -if(LUA_JIT) - add_definitions(-DLUA_JIT) +include(GNUInstallDirs) +if(WIN32) + set(INSTALL_CMAKE_DIR cmake) + set(CMAKE_INSTALL_LIBDIR ${CMAKE_INSTALL_BINDIR}) +else() + set(INSTALL_CMAKE_DIR "${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/cmake") endif() -find_package(Git REQUIRED) - set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(mozsvc) -include(externals) -include_directories("${CMAKE_SOURCE_DIR}/include") -install(DIRECTORY "${CMAKE_SOURCE_DIR}/modules/" DESTINATION modules) +include(CheckFunctionExists) +check_function_exists(clock_gettime HAVE_CLOCK_GETTIME) +if(HAVE_CLOCK_GETTIME) + add_definitions(-DHAVE_CLOCK_GETTIME) +endif() + +include(CMakePackageConfigHelpers) +configure_package_config_file(cmake/${PROJECT_NAME}Config.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + INSTALL_DESTINATION ${INSTALL_CMAKE_DIR} + PATH_VARS CMAKE_INSTALL_INCLUDEDIR CMAKE_INSTALL_LIBDIR) +write_basic_package_version_file(${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + COMPATIBILITY SameMajorVersion) +install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}Config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}ConfigVersion.cmake + DESTINATION ${INSTALL_CMAKE_DIR}) + +find_library(LIBM_LIBRARY m) + +include_directories("${CMAKE_SOURCE_DIR}/include" "${CMAKE_SOURCE_DIR}/include/luasandbox" ) +install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +install(FILES "${CMAKE_SOURCE_DIR}/README.md" DESTINATION ${CMAKE_INSTALL_DOCDIR}) add_subdirectory(src) +add_dependencies(luasandbox luasandboxutil) +add_dependencies(luasandboxtest luasandbox) +add_dependencies(luasandboxheka luasandboxtest) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..498baa3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,15 @@ +# Community Participation Guidelines + +This repository is governed by Mozilla's code of conduct and etiquette guidelines. +For more details, please read the +[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). + +## How to Report +For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. + + diff --git a/README.md b/README.md index 92047a7..729e91d 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,84 @@ -Prerequisites {#mainpage} -==== -* C compiler (GCC 4.7+, Visual Studio 2013 (LuaJIT), MinGW (Lua 5.1)) -* CMake (2.8.7+) - http://cmake.org/cmake/resources/software.html +# Lua Sandbox Library + +## Overview + +Sandboxes provide a dynamic and isolated execution environment +for data parsing, transformation, and analysis. They allow access to data +without jeopardizing the integrity or performance of the processing +infrastructure. This broadens the audience that the data can be +exposed to and facilitates new uses of the data (i.e. debugging, monitoring, +dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, +etc.) + +The Lua sandbox is a library allowing customized control over the Lua execution +environment including functionality like global data preservation/restoration on +shutdown/startup, output collection in textual or binary formats and an array of +parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) + +These libraries and utilities have been mostly extracted from +[Hindsight](https://github.com/mozilla-services/hindsight). The goal was to +decouple the Heka/Hindsight functionality from any particular infrastructure and +make it embeddable into any tool or language. + +### Features + +- small - memory requirements are as little as 8 KiB for a basic sandbox +- fast - microsecond execution times +- stateful - ability to resume where it left off after a restart/reboot +- isolated - failures are contained and malfunctioning sandboxes are terminated. + Containment is defined in terms of restriction to the operating system, + file system, libraries, memory use, Lua instruction use, and output size. + +[Full Documentation](http://mozilla-services.github.io/lua_sandbox) + +## Installation + +### Prerequisites +* C compiler (GCC 4.7+, Visual Studio 2013) +* CMake (3.0+) - http://cmake.org/cmake/resources/software.html * Git http://git-scm.com/download -Optional (used for documentation) ----- +#### Optional (used for documentation) * Graphviz (2.28.0) - http://graphviz.org/Download..php -* Doxygen (1.8+)- http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc +* Doxygen (1.8.11+) - http://www.stack.nl/~dimitri/doxygen/download.html#latestsrc +* gitbook (2.3) - https://www.gitbook.com/ +* lua (5.1) - https://www.lua.org/download.html -lua_sandbox - UNIX Build Instructions ----- +### CMake Build Instructions git clone https://github.com/mozilla-services/lua_sandbox.git - cd lua_sandbox + cd lua_sandbox mkdir release cd release + + # UNIX cmake -DCMAKE_BUILD_TYPE=release .. make - ctest - -lua_sandbox - Windows Build Instructions ----- - - # in a VS2013 command prompt window - git clone https://github.com/mozilla-services/lua_sandbox.git - cd lua_sandbox - mkdir release - cd release + # Windows Visual Studio 2013 cmake -DCMAKE_BUILD_TYPE=release -G "NMake Makefiles" .. nmake - # To run the tests you must install - cmake -DCMAKE_INSTALL_PREFIX="" .. - nmake install DESTDIR=test - cd ..\src\test - ..\..\release\test\lib\test_lua_sandbox.exe + ctest + cpack -G TGZ # (DEB|RPM|ZIP) + +## Releases + +* The master branch is the current release and is considered stable at all + times. +* New versions can be released as frequently as every two weeks (our sprint + cycle). The only exception would be for a high priority patch. +* All active work is flagged with the sprint milestone and tracked in the + project dashboard. +* New releases occur the day after the sprint finishes. + * The version in the dev branch is updated + * The changes are merged into master + * A new tag is created + +## Contributions +* All pull requests must be made against the dev branch, direct commits to + master are not permitted. +* All non trivial contributions should start with an issue being filed (if it is + a new feature please propose your design/approach before doing any work as not + all feature requests are accepted). diff --git a/build/functions.sh b/build/functions.sh new file mode 100755 index 0000000..4ea418a --- /dev/null +++ b/build/functions.sh @@ -0,0 +1,199 @@ +#!/bin/sh + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Author: Mathieu Parent + +# ================================================================= +# Show usage +usage() { + echo "Build, test and make packages (optionally using Docker)" >&2 + echo " $0 build" >&2 + echo "" >&2 + echo "Options are passed using environment variables:" >&2 + echo " - DOCKER_IMAGE: Run the build in a container (default: )" >&2 + echo " - CPACK_GENERATOR: DEB, RPM, TGZ, ... (default guessed)" >&2 + echo " - CMAKE_URL: If set, download cmake from it (default: , auto also supported)" >&2 + echo " - CMAKE_SHA256: Used to checksum CMAKE_URL (default: )" >&2 + echo " - CC: C compiler (default: gcc)" >&2 + echo " - CXX: C++ compiler (default: g++)" >&2 +} + + +# ================================================================= +# Run command within container +docker_run() { + DOCKER_IMAGE="${DOCKER_IMAGE:-debian:jessie}" + DOCKER_VOLUME="$(dirname "$PWD")" + ( set -x + docker pull "$DOCKER_IMAGE" + ret=0 + docker run \ + --volume "${DOCKER_VOLUME}:${DOCKER_VOLUME}" \ + --workdir "$PWD" \ + --rm=true --tty=true \ + --env "CPACK_GENERATOR=${CPACK_GENERATOR}" \ + --env "CMAKE_URL=${CMAKE_URL}" \ + --env "CMAKE_SHA256=${CMAKE_SHA256}" \ + --env "CC=${CC}" \ + --env "CXX=${CXX}" \ + --env "DISTRO=${DOCKER_IMAGE}" \ + "$DOCKER_IMAGE" \ + $@ + ) +} + +# ================================================================= +# Guess environment variables +setup_env() { + if [ "$(id -u)" = 0 ]; then + as_root="" + else + as_root="sudo" + fi + if [ -z "${CPACK_GENERATOR}" ]; then + if [ -x "`command -v apt-get 2>/dev/null`" ]; then + CPACK_GENERATOR=DEB + elif [ -x "`command -v rpm 2>/dev/null`" ]; then + CPACK_GENERATOR=RPM + else + CPACK_GENERATOR=TGZ + fi + fi + CC="${CC:-gcc}" + CXX="${CXX:-g++}" +} + +# ================================================================= +# Install cmake from URL +install_cmake() { + if echo "$PATH" | grep -qF "/cmake-dist/bin:" ; then + # Already done + return + fi + if [ "$CMAKE_URL" = "auto" ]; then + CMAKE_URL="https://cmake.org/files/v3.7/cmake-3.7.1-Linux-x86_64.tar.gz" + CMAKE_SHA256=7b4b7a1d9f314f45722899c0521c261e4bfab4a6b532609e37fef391da6bade2 + elif [ -z "$CMAKE_URL" ]; then + echo "Missing env CMAKE_URL" + exit 1 + elif [ -z "$CMAKE_SHA256" ]; then + echo "Missing env CMAKE_SHA256" + exit 1 + fi + ( set -x + wget -O cmake.tar.gz "${CMAKE_URL}" + echo "${CMAKE_SHA256} cmake.tar.gz" > cmake-SHA256.txt + sha256sum --check cmake-SHA256.txt || { + echo 'Checksum error' + exit 1 + } + mkdir cmake-dist + cd cmake-dist + tar xzf "../cmake.tar.gz" --strip-components=1 + ) + PATH="$PWD/cmake-dist/bin:${PATH}" + echo "+PATH=${PATH}" +} + +# ================================================================= +# Install packages from distribution +# The following substitutions are done: +# - cmake is replaced by downloaded, if CMAKE_URL is set +# - rpm-build is ignored on DEB distributions +# - ca-certificates is ignored on RPM distributions +install_packages() { + local packages + packages="$@" + do_install_cmake=no + if [ -n "$CMAKE_URL" ] && echo " $packages " | grep -qF " cmake "; then + packages="$(echo " $packages " | sed "s/ cmake / wget ca-certificates /")" + do_install_cmake=yes + fi + packages="$(echo " $packages " | sed -e "s/ c-compiler / $CC /" -e "s/ c++-compiler / $CXX /")" + if [ "$CPACK_GENERATOR" = "DEB" ]; then + packages="$(echo "$packages" | sed -e "s/ rpm-build / /" -e "s/ clang++ / clang /")" + ( set -x; + $as_root apt-get update + $as_root apt-get install -y $packages + ) + elif [ "$CPACK_GENERATOR" = "RPM" ]; then + packages="$(echo "$packages" | sed -e "s/ ca-certificates / /" -e "s/ g++ / gcc-c++ /" -e "s/ clang++ / clang /")" + ( set -x; + $as_root yum install -y $packages + ) + else + echo "Unhandled CPACK_GENERATOR: $CPACK_GENERATOR" >&2 + exit 1 + fi + if [ "$do_install_cmake" = "yes" ]; then + install_cmake + fi +} + +# ================================================================= +# Install all packages from directory +install_packages_from_dir() { + local dir + dir="$1" + if [ ! -d "$dir" ]; then + echo "Not a directory: $dir" + exit 1 + fi + if [ "$CPACK_GENERATOR" = "DEB" ]; then + (set -x; $as_root dpkg -i "$dir"/*.deb) + elif [ "$CPACK_GENERATOR" = "RPM" ]; then + (set -x; $as_root rpm -i "$dir"/*.rpm) + else + echo "Unhandled CPACK_GENERATOR: $CPACK_GENERATOR" >&2 + exit 1 + fi +} + +# ================================================================= +# Build +build() { + ( set -x + + rm -rf ./release + # From README.md: + mkdir release + cd release + + cmake -DCMAKE_BUILD_TYPE=release .. + make + + ctest -V + cpack -G "${CPACK_GENERATOR}" + ) +} + +# ================================================================= +# Main +main() { + if [ -n "$DOCKER_IMAGE" ]; then + docker_run "$0" build + else + setup_env + install_packages c-compiler cmake make rpm-build + if [ "$(basename "$PWD")" != "lua_sandbox" ]; then + local old_dir + local lsb_dir + old_dir="$PWD" + lsb_dir="$(dirname "$old_dir")/lua_sandbox" + echo "+cd $lsb_dir" + cd "$lsb_dir" + build + install_packages_from_dir ./release + echo "+cd $old_dir" + cd "$old_dir" + fi + if [ -n "$build_function" ]; then + "$build_function" + else + build + fi + fi +} diff --git a/build/run.sh b/build/run.sh new file mode 100755 index 0000000..71b6d04 --- /dev/null +++ b/build/run.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +# +# Author: Mathieu Parent + +set -e + +. "$(dirname $0)/functions.sh" + +if [ "$1" != "build" -o $# -ge 2 ]; then + usage + exit 1 +fi + +build_function= +main diff --git a/cmake/doxygen.cmake b/cmake/doxygen.cmake index ca21c84..280fec8 100644 --- a/cmake/doxygen.cmake +++ b/cmake/doxygen.cmake @@ -3,36 +3,42 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. find_package(Doxygen QUIET) -if(DOXYGEN_FOUND) +find_program(LUA_EXE lua QUIET) +find_program(PANDOC_EXE pandoc QUIET) +find_program(gitbook gitbook QUIET) +if(DOXYGEN_FOUND AND LUA_EXE AND PANDOC_EXE) set(DOXYCONF_IN ${CMAKE_SOURCE_DIR}/doxygen.in.conf) set(DOXYCONF_OUT ${CMAKE_BINARY_DIR}/doxygen.conf) if(EXISTS ${DOXYCONF_IN}) configure_file(${DOXYCONF_IN} ${DOXYCONF_OUT}) else() file(WRITE ${DOXYCONF_OUT} " -PROJECT_NAME = \"${PROJECT_NAME}\" -PROJECT_BRIEF = \"${CPACK_PACKAGE_DESCRIPTION_SUMMARY}\" -OUTPUT_DIRECTORY = docs -GENERATE_LATEX = NO -GENERATE_TODOLIST = YES -FULL_PATH_NAMES = YES -STRIP_FROPATH = \"${CMAKE_SOURCE_DIR}\" -SOURCE_BROWSER = YES -TAB_SIZE = 4 -EXTRACT_ALL = YES -JAVADOC_AUTOBRIEF = YES -RECURSIVE = YES -INPUT = \"${CMAKE_SOURCE_DIR}\" -EXCLUDE_PATTERNS = \"${CMAKE_SOURCE_DIR}/.*\" \"${CMAKE_SOURCE_DIR}/debug*\" \"${CMAKE_SOURCE_DIR}/release*\" -EXAMPLE_PATH = ${EXAMPLE_PATHS} -IMAGE_PATH = ${IMAGE_PATHS} -BUILTIN_STL_SUPPORT = YES -STRIP_CODE_COMMENTS = NO -SHOW_DIRECTORIES = YES -PROJECT_NUMBER = ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") +PROJECT_NAME = \"${PROJECT_NAME}\" +PROJECT_BRIEF = \"${CPACK_PACKAGE_DESCRIPTION_SUMMARY}\" +OUTPUT_DIRECTORY = \"${CMAKE_SOURCE_DIR}/gh-pages\" +HTML_OUTPUT = doxygen +GENERATE_LATEX = NO +GENERATE_TODOLIST = YES +FULL_PATH_NAMES = YES +STRIP_FROM_PATH = \"${CMAKE_SOURCE_DIR}\" +SOURCE_BROWSER = YES +TAB_SIZE = 4 +EXTRACT_ALL = YES +JAVADOC_AUTOBRIEF = YES +RECURSIVE = YES +INPUT = \"${CMAKE_SOURCE_DIR}/include\" \"${CMAKE_SOURCE_DIR}/README.md\" +USE_MDFILE_AS_MAINPAGE = \"${CMAKE_SOURCE_DIR}/README.md\" +EXAMPLE_PATH = ${EXAMPLE_PATHS} +IMAGE_PATH = ${IMAGE_PATHS} +BUILTIN_STL_SUPPORT = YES +STRIP_CODE_COMMENTS = NO +SHOW_DIRECTORIES = YES +PROJECT_NUMBER = ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") endif() - add_custom_target(docs ${DOXYGEN_EXECUTABLE} ${DOXYCONF_OUT}) + add_custom_target(docs ${DOXYGEN_EXECUTABLE} ${DOXYCONF_OUT} + COMMAND lua gen_gh_pages.lua "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}" + "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) else() - message("Doxygen was not found, the documentation pages will not be generated") + message("The optional documentation tools were not found; the doc target has not been created") endif() diff --git a/cmake/externals.cmake b/cmake/externals.cmake deleted file mode 100644 index e44c4e2..0000000 --- a/cmake/externals.cmake +++ /dev/null @@ -1,88 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -include(ExternalProject) - -get_filename_component(GIT_PATH ${GIT_EXECUTABLE} PATH) -find_program(PATCH_EXECUTABLE patch HINTS "${GIT_PATH}" "${GIT_PATH}/../bin") -if (NOT PATCH_EXECUTABLE) - message(FATAL_ERROR "patch not found") -endif() - -set(EP_BASE "${CMAKE_BINARY_DIR}/ep_base") -set_property(DIRECTORY PROPERTY EP_BASE ${EP_BASE}) -set(CJSON_PATCH_FILE "lua-cjson-2_1_0.patch") -set(SANDBOX_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${EP_BASE} -DEP_BASE=${EP_BASE} -DADDRESS_MODEL=${ADDRESS_MODEL} --no-warn-unused-cli) -set(LUA_INCLUDE_DIR "${EP_BASE}/include") - -if (LUA_JIT) - set(LUA_PROJECT "luajit-2_0_2") - if(MSVC) - externalproject_add( - ${LUA_PROJECT} - BUILD_IN_SOURCE 1 - URL http://luajit.org/download/LuaJIT-2.0.2.tar.gz - URL_MD5 112dfb82548b03377fbefbba2e0e3a5b - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/luajit-2_0_2.patch - CONFIGURE_COMMAND "" - BUILD_COMMAND cmake -E chdir src msvcbuild.bat - INSTALL_COMMAND cmake -E copy src/lua.dll ${EP_BASE}/lib/lua.dll - COMMAND cmake -E copy src/lua.lib ${EP_BASE}/lib/lua.lib - COMMAND cmake -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" - COMMAND cmake -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" - COMMAND cmake -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" - COMMAND cmake -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" - COMMAND cmake -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" - ) - elseif(UNIX) - externalproject_add( - ${LUA_PROJECT} - BUILD_IN_SOURCE 1 - URL http://luajit.org/download/LuaJIT-2.0.2.tar.gz - URL_MD5 112dfb82548b03377fbefbba2e0e3a5b - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/luajit-2_0_2.patch - CONFIGURE_COMMAND "" - BUILD_COMMAND make - INSTALL_COMMAND cmake -E copy src/libluajit.a ${EP_BASE}/lib/liblua.a - COMMAND cmake -E copy src/lauxlib.h "${LUA_INCLUDE_DIR}/lauxlib.h" - COMMAND cmake -E copy src/luaconf.h "${LUA_INCLUDE_DIR}/luaconf.h" - COMMAND cmake -E copy src/lua.h "${LUA_INCLUDE_DIR}/lua.h" - COMMAND cmake -E copy src/luajit.h "${LUA_INCLUDE_DIR}/luajit.h" - COMMAND cmake -E copy src/lualib.h "${LUA_INCLUDE_DIR}/lualib.h" - ) - else() - message(FATAL_ERROR "Cannot use LuaJIT with ${CMAKE_GENERATOR}") - endif() -else() - set(LUA_PROJECT "lua-5_1_5") - externalproject_add( - ${LUA_PROJECT} - URL http://www.lua.org/ftp/lua-5.1.5.tar.gz - URL_MD5 2e115fe26e435e33b0d5c022e4490567 - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/lua-5_1_5.patch - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} - ) -endif() -include_directories(${LUA_INCLUDE_DIR}) - -externalproject_add( - lpeg-0_12 - URL http://www.inf.puc-rio.br/~roberto/lpeg/lpeg-0.12.tar.gz - URL_MD5 4abb3c28cd8b6565c6a65e88f06c9162 - PATCH_COMMAND ${PATCH_EXECUTABLE} -p1 < ${CMAKE_CURRENT_LIST_DIR}/lpeg-0_12.patch - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} -) -add_dependencies(lpeg-0_12 ${LUA_PROJECT}) - -externalproject_add( - lua-cjson-2_1_0 - GIT_REPOSITORY https://github.com/trink/lua-cjson.git - GIT_TAG be85986dc481ad2a0d3abc06ac57e1d1241d9d4c - CMAKE_ARGS ${SANDBOX_CMAKE_ARGS} - INSTALL_DIR ${EP_BASE} -) -add_dependencies(lua-cjson-2_1_0 ${LUA_PROJECT}) - diff --git a/cmake/lpeg-0_12.patch b/cmake/lpeg-0_12.patch deleted file mode 100644 index d6c15e0..0000000 --- a/cmake/lpeg-0_12.patch +++ /dev/null @@ -1,53 +0,0 @@ -diff -Naur lpeg-0.12.orig/lptree.c lpeg-0_12/lptree.c ---- lpeg-0.12.orig/lptree.c Fri Apr 12 09:31:19 2013 -+++ lpeg-0_12/lptree.c Fri Aug 2 10:15:43 2013 -@@ -1217,8 +1217,7 @@ - }; - - --int luaopen_lpeg (lua_State *L); --int luaopen_lpeg (lua_State *L) { -+LUALIB_API int luaopen_lpeg (lua_State *L) { - luaL_newmetatable(L, PATTERN_T); - lua_pushnumber(L, MAXBACK); /* initialize maximum backtracking */ - lua_setfield(L, LUA_REGISTRYINDEX, MAXSTACKIDX); -diff -Naur lpeg-0.12.orig/CMakeLists.txt lpeg-0.12/CMakeLists.txt ---- lpeg-0.12.orig/CMakeLists.txt Wed Dec 31 16:00:00 1969 -+++ lpeg-0.12.orig/CMakeLists.txt Fri Aug 2 09:26:42 2013 -@@ -0,0 +1,36 @@ -+# This Source Code Form is subject to the terms of the Mozilla Public -+# License, v. 2.0. If a copy of the MPL was not distributed with this -+# file, You can obtain one at http://mozilla.org/MPL/2.0/. -+ -+cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -+project(lpeg C) -+ -+set(LPEG_SOURCE -+lpcap.c -+lpcode.c -+lptree.c -+lpvm.c -+) -+ -+include_directories("${EP_BASE}/include") -+if (WIN32) -+ add_library(lpeg SHARED ${LPEG_SOURCE}) -+ add_definitions(-DLUA_BUILD_AS_DLL -DLUA_CORE) -+ if(MSVC) -+ target_link_libraries(lpeg "${EP_BASE}/lib/lua.lib") -+ else(MINGW) -+ set_target_properties(lpeg PROPERTIES LINK_FLAGS -s) -+ if (ADDRESS_MODEL EQUAL 32) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") -+ set_target_properties(lpeg PROPERTIES LINK_FLAGS "-s -m32") -+ endif() -+ target_link_libraries(lpeg "${EP_BASE}/lib/liblua.dll") -+ else() -+ message(FATAL_ERROR "Only MSVC and MinGW compilers are supported on Windows") -+ endif() -+else() -+ add_library(lpeg STATIC ${LPEG_SOURCE}) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -Wall -fPIC") -+endif() -+ -+install(TARGETS lpeg DESTINATION lib) diff --git a/cmake/lua-5_1_5.patch b/cmake/lua-5_1_5.patch deleted file mode 100644 index abc703b..0000000 --- a/cmake/lua-5_1_5.patch +++ /dev/null @@ -1,136 +0,0 @@ -diff -Naur lua-5.1.5/CMakeLists.txt Source/lua-5_1_5/CMakeLists.txt ---- lua-5.1.5/CMakeLists.txt Wed Dec 31 16:00:00 1969 -+++ Source/lua-5_1_5/CMakeLists.txt Wed May 21 13:16:57 2014 -@@ -0,0 +1,62 @@ -+# This Source Code Form is subject to the terms of the Mozilla Public -+# License, v. 2.0. If a copy of the MPL was not distributed with this -+# file, You can obtain one at http://mozilla.org/MPL/2.0/. -+ -+cmake_minimum_required(VERSION 2.8 FATAL_ERROR) -+project(lua C) -+ -+set(LUA_SOURCE -+src/lapi.c -+src/lauxlib.c -+src/lbaselib.c -+src/lcode.c -+src/ldblib.c -+src/ldebug.c -+src/ldo.c -+src/ldump.c -+src/lfunc.c -+src/lgc.c -+src/linit.c -+src/liolib.c -+src/llex.c -+src/lmathlib.c -+src/lmem.c -+src/loadlib.c -+src/lobject.c -+src/lopcodes.c -+src/loslib.c -+src/lparser.c -+src/lstate.c -+src/lstring.c -+src/lstrlib.c -+src/ltable.c -+src/ltablib.c -+src/ltm.c -+src/lundump.c -+src/lvm.c -+src/lzio.c -+) -+ -+set(CMAKE_INSTALL_PREFIX ${EP_BASE}) -+if (WIN32) -+ add_library(lua SHARED ${LUA_SOURCE}) -+ add_definitions(-DLUA_BUILD_AS_DLL) -+ if (MINGW) -+ set_target_properties(lua PROPERTIES LINK_FLAGS -s) -+ if (ADDRESS_MODEL EQUAL 32) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") -+ set_target_properties(lua PROPERTIES LINK_FLAGS "-s -m32") -+ endif() -+ endif() -+elseif(APPLE) -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") -+ add_library(lua STATIC ${LUA_SOURCE}) -+ add_definitions(-DLUA_USE_LINUX) -+ target_link_libraries(lua -lreadline) -+else() -+ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") -+ add_library(lua STATIC ${LUA_SOURCE}) -+endif() -+ -+install(TARGETS lua DESTINATION lib) -+install(FILES src/lua.h src/luaconf.h src/lualib.h src/lauxlib.h DESTINATION include) -diff -Naur lua-5.1.5/src/ldebug.c Source/lua-5_1_5/src/ldebug.c ---- lua-5.1.5/src/ldebug.c Thu May 8 09:56:26 2008 -+++ Source/lua-5_1_5/src/ldebug.c Wed May 21 13:16:58 2014 -@@ -80,6 +80,10 @@ - return L->basehookcount; - } - -+LUA_API int lua_gethookcountremaining (lua_State *L) { -+ return L->hookcount; -+} -+ - - LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar) { - int status; -diff -Naur lua-5.1.5/src/loslib.c Source/lua-5_1_5/src/loslib.c ---- lua-5.1.5/src/loslib.c Fri Jan 18 08:38:18 2008 -+++ Source/lua-5_1_5/src/loslib.c Wed May 21 13:16:58 2014 -@@ -154,9 +154,40 @@ - if (*s != '%' || *(s + 1) == '\0') /* no conversion specifier? */ - luaL_addchar(&b, *s); - else { -+ cc[1] = *(++s); -+#ifdef _WIN32 -+ int valid = 0; -+ switch (*s) { -+ case 'a': -+ case 'A': -+ case 'b': -+ case 'B': -+ case 'c': -+ case 'd': -+ case 'H': -+ case 'I': -+ case 'j': -+ case 'm': -+ case 'M': -+ case 'p': -+ case 'S': -+ case 'U': -+ case 'w': -+ case 'W': -+ case 'x': -+ case 'X': -+ case 'y': -+ case 'Y': -+ case 'Z': -+ case 'z': -+ case '%': -+ valid = 1; -+ break; -+ } -+ if (!valid) continue; -+#endif - size_t reslen; - char buff[200]; /* should be big enough for any conversion result */ -- cc[1] = *(++s); - reslen = strftime(buff, sizeof(buff), cc, stm); - luaL_addlstring(&b, buff, reslen); - } -diff -Naur lua-5.1.5/src/lua.h Source/lua-5_1_5/src/lua.h ---- lua-5.1.5/src/lua.h Fri Jan 13 12:36:20 2012 -+++ Source/lua-5_1_5/src/lua.h Wed May 21 13:16:58 2014 -@@ -341,6 +341,7 @@ - LUA_API lua_Hook lua_gethook (lua_State *L); - LUA_API int lua_gethookmask (lua_State *L); - LUA_API int lua_gethookcount (lua_State *L); -+LUA_API int lua_gethookcountremaining (lua_State *L); - - - struct lua_Debug { diff --git a/cmake/luajit-2_0_2.patch b/cmake/luajit-2_0_2.patch deleted file mode 100644 index 2c3ad0d..0000000 --- a/cmake/luajit-2_0_2.patch +++ /dev/null @@ -1,140 +0,0 @@ -diff -Naur LuaJIT-2.0.2/src/lj_api.c luajit-2_0_2/src/lj_api.c ---- LuaJIT-2.0.2/src/lj_api.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_api.c 2013-12-03 21:17:05.042524855 +0000 -@@ -1178,6 +1178,18 @@ - res = (int)(g->gc.stepmul); - g->gc.stepmul = (MSize)data; - break; -+ case LUA_GCSETMEMLIMIT: -+ res = data; -+ lj_gc_fullgc(L); -+ g->gc.maxmem = g->gc.total; -+ g->gc.memlimit = (MSize)data; -+ break; -+ case LUA_GCMAXCOUNT: -+ res = (int)(g->gc.maxmem >> 10); -+ break; -+ case LUA_GCMAXCOUNTB: -+ res = (int)(g->gc.maxmem & 0x3ff); -+ break; - default: - res = -1; /* Invalid option. */ - } -diff -Naur LuaJIT-2.0.2/src/lj_dispatch.c luajit-2_0_2/src/lj_dispatch.c ---- LuaJIT-2.0.2/src/lj_dispatch.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_dispatch.c 2013-12-03 21:17:05.042524855 +0000 -@@ -339,6 +339,11 @@ - return (int)G(L)->hookcstart; - } - -+LUA_API int lua_gethookcountremaining(lua_State *L) -+{ -+ return (int)G(L)->hookcount; -+} -+ - /* Call a hook. */ - static void callhook(lua_State *L, int event, BCLine line) - { -diff -Naur LuaJIT-2.0.2/src/lj_gc.c luajit-2_0_2/src/lj_gc.c ---- LuaJIT-2.0.2/src/lj_gc.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_gc.c 2013-12-03 21:17:05.042524855 +0000 -@@ -800,24 +800,37 @@ - { - global_State *g = G(L); - lua_assert((osz == 0) == (p == NULL)); -- p = g->allocf(g->allocd, p, osz, nsz); -+ if (g->gc.memlimit == 0 || (g->gc.total - osz) + nsz < g->gc.memlimit) { -+ p = g->allocf(g->allocd, p, osz, nsz); -+ } else { -+ p = NULL; -+ } - if (p == NULL && nsz > 0) - lj_err_mem(L); - lua_assert((nsz == 0) == (p == NULL)); - lua_assert(checkptr32(p)); - g->gc.total = (g->gc.total - osz) + nsz; -- return p; -+ if (g->gc.total > g->gc.maxmem) { -+ g->gc.maxmem = g->gc.total; -+ } -+ return p; - } - - /* Allocate new GC object and link it to the root set. */ - void * LJ_FASTCALL lj_mem_newgco(lua_State *L, MSize size) - { - global_State *g = G(L); -- GCobj *o = (GCobj *)g->allocf(g->allocd, NULL, 0, size); -+ GCobj *o = NULL; -+ if (g->gc.memlimit == 0 || g->gc.total + size < g->gc.memlimit) { -+ o = (GCobj *)g->allocf(g->allocd, NULL, 0, size); -+ } - if (o == NULL) - lj_err_mem(L); - lua_assert(checkptr32(o)); - g->gc.total += size; -+ if (g->gc.total > g->gc.maxmem) { -+ g->gc.maxmem = g->gc.total; -+ } - setgcrefr(o->gch.nextgc, g->gc.root); - setgcref(g->gc.root, o); - newwhite(g, o); -diff -Naur LuaJIT-2.0.2/src/lj_obj.h luajit-2_0_2/src/lj_obj.h ---- LuaJIT-2.0.2/src/lj_obj.h 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_obj.h 2013-12-03 21:17:05.042524855 +0000 -@@ -490,6 +490,8 @@ - - typedef struct GCState { - MSize total; /* Memory currently allocated. */ -+ MSize maxmem; /* Maximum Memory that was allocated */ -+ MSize memlimit; /* Maximum Memory limit 0 = unlimited */ - MSize threshold; /* Memory threshold. */ - uint8_t currentwhite; /* Current white color. */ - uint8_t state; /* GC state. */ -diff -Naur LuaJIT-2.0.2/src/lj_state.c luajit-2_0_2/src/lj_state.c ---- LuaJIT-2.0.2/src/lj_state.c 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lj_state.c 2013-12-03 21:17:05.042524855 +0000 -@@ -208,6 +208,8 @@ - setgcref(g->gc.root, obj2gco(L)); - setmref(g->gc.sweep, &g->gc.root); - g->gc.total = sizeof(GG_State); -+ g->gc.maxmem = g->gc.total; -+ g->gc.memlimit = 0; - g->gc.pause = LUAI_GCPAUSE; - g->gc.stepmul = LUAI_GCMUL; - lj_dispatch_init((GG_State *)L); -diff -Naur LuaJIT-2.0.2/src/lua.h luajit-2_0_2/src/lua.h ---- LuaJIT-2.0.2/src/lua.h 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/lua.h 2013-12-03 21:17:05.042524855 +0000 -@@ -226,6 +226,9 @@ - #define LUA_GCSTEP 5 - #define LUA_GCSETPAUSE 6 - #define LUA_GCSETSTEPMUL 7 -+#define LUA_GCSETMEMLIMIT 8 -+#define LUA_GCMAXCOUNT 9 -+#define LUA_GCMAXCOUNTB 10 - - LUA_API int (lua_gc) (lua_State *L, int what, int data); - -@@ -340,6 +343,7 @@ - LUA_API lua_Hook lua_gethook (lua_State *L); - LUA_API int lua_gethookmask (lua_State *L); - LUA_API int lua_gethookcount (lua_State *L); -+LUA_API int lua_gethookcountremaining (lua_State *L); - - /* From Lua 5.2. */ - LUA_API void *lua_upvalueid (lua_State *L, int idx, int n); -diff -Naur LuaJIT-2.0.2/src/msvcbuild.bat luajit-2_0_2/src/msvcbuild.bat ---- LuaJIT-2.0.2/src/msvcbuild.bat 2013-06-03 19:00:00.000000000 +0000 -+++ luajit-2_0_2/src/msvcbuild.bat 2013-12-03 21:24:09.434507458 +0000 -@@ -20,8 +20,8 @@ - @set LJLIB=lib /nologo - @set DASMDIR=..\dynasm - @set DASM=%DASMDIR%\dynasm.lua --@set LJDLLNAME=lua51.dll --@set LJLIBNAME=lua51.lib -+@set LJDLLNAME=lua.dll -+@set LJLIBNAME=lua.lib - @set ALL_LIB=lib_base.c lib_math.c lib_bit.c lib_string.c lib_table.c lib_io.c lib_os.c lib_package.c lib_debug.c lib_jit.c lib_ffi.c - - %LJCOMPILE% host\minilua.c diff --git a/cmake/luasandboxConfig.cmake.in b/cmake/luasandboxConfig.cmake.in new file mode 100644 index 0000000..f002aa1 --- /dev/null +++ b/cmake/luasandboxConfig.cmake.in @@ -0,0 +1,13 @@ +set(LUASANDBOX_VERSION x.y.z) + +@PACKAGE_INIT@ + +set_and_check(LUASANDBOX_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_INCLUDEDIR@") +set_and_check(LUASANDBOX_LIB_DIR "@PACKAGE_CMAKE_INSTALL_LIBDIR@") + +FIND_LIBRARY(LUASANDBOX_UTIL_LIBRARY NAMES luasandboxutil PATHS ${LUASANDBOX_LIB_DIR}) +FIND_LIBRARY(LUASANDBOX_LIBRARY NAMES luasandbox PATHS ${LUASANDBOX_LIB_DIR}) +FIND_LIBRARY(LUASANDBOX_HEKA_LIBRARY NAMES luasandboxheka PATHS ${LUASANDBOX_LIB_DIR}) +FIND_LIBRARY(LUASANDBOX_TEST_LIBRARY NAMES luasandboxtest PATHS ${LUASANDBOX_LIB_DIR}) + +set(LUASANDBOX_LIBRARIES ${LUASANDBOX_HEKA_LIBRARY} ${LUASANDBOX_LIBRARY} ${LUASANDBOX_UTIL_LIBRARY}) diff --git a/cmake/mozsvc.cmake b/cmake/mozsvc.cmake index 6025911..2f62115 100644 --- a/cmake/mozsvc.cmake +++ b/cmake/mozsvc.cmake @@ -10,7 +10,7 @@ if(MSVC) set(CMAKE_C_FLAGS "/W3 /WX") # enable C++ exception handling - set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /EHsc") + set(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} /EHs") # debug multi threaded dll runtime, complete debugging info, runtime error checking set(CMAKE_C_FLAGS_DEBUG "/MDd /Zi /RTC1") @@ -20,15 +20,12 @@ if(MSVC) set(CMAKE_C_FLAGS_RELEASE "/MD /O2 /Ob2 /DNDEBUG") set(CMAKE_CXX_FLAGS_RELEASE ${CMAKE_C_FLAGS_RELEASE}) - set(CPACK_GENERATOR "NSIS") + set(CPACK_GENERATOR "ZIP") else() # Predefined Macros: clang|gcc -dM -E -x c /dev/null # Compiler options: http://gcc.gnu.org/onlinedocs/gcc/Invoking-GCC.html#Invoking-GCC - set(CMAKE_C_FLAGS "-std=gnu99 -pedantic -Werror -Wall -Wextra") - if (NOT WIN32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fPIC") - endif() - set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic -Werror -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") + set(CMAKE_C_FLAGS "-std=gnu11 -pedantic $ENV{CFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -fvisibility=hidden") + set(CMAKE_CXX_FLAGS "-std=c++11 -pedantic $ENV{CXXFLAGS} $ENV{CPPFLAGS} -Wall -Wextra -fPIC -isystem /usr/local/include -isystem /opt/local/include") set(CMAKE_C_FLAGS_DEBUG "-g") set(CMAKE_CXX_FLAGS_DEBUG ${CMAKE_C_FLAGS_DEBUG}) @@ -47,6 +44,12 @@ endif() set(CPACK_PACKAGE_VENDOR "Mozilla Services") set(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_SOURCE_DIR}/LICENSE.txt") +if(CMAKE_BUILD_TYPE MATCHES "^[Dd][Ee][Bb][Uu][Gg]$") + set(CPACK_STRIP_FILES FALSE) +else() + set(CPACK_STRIP_FILES TRUE) +endif() + include(CPack) include(CTest) include(doxygen) diff --git a/covfn.txt b/covfn.txt new file mode 100644 index 0000000..5f7c3a2 --- /dev/null +++ b/covfn.txt @@ -0,0 +1,158 @@ +Function Source Line FnCov C/D Coverage +------------------------------------------------------------------------------------------------ -------------------------------- ----- --------------------- +add_table_ref(table_ref_array*,const void*,size_t) .../luasandbox_serialize.c 171 1 / 1 1 / 4 = 25% +lsb_create(void*,const char*,const char*,lsb_logger*) ../src/luasandbox.c 428 1 / 1 16 / 42 = 38% +restore_global_data(lsb_lua_sandbox*) .../luasandbox_serialize.c 488 1 / 1 7 / 18 = 38% +lsb_heka_create_input(void*,const char*,const char*,const char*,lsb_logger*,lsb_heka_im_input) ../src/heka/sandbox.c 407 1 / 1 14 / 34 = 41% +lsb_heka_create_analysis(void*,const char*,cons...,const char*,lsb_logger*,lsb_heka_im_analysis) ../src/heka/sandbox.c 598 1 / 1 14 / 34 = 41% +lsb_heka_create_output(void*,const char*,const ... char*,lsb_logger*,lsb_heka_update_checkpoint) ../src/heka/sandbox.c 689 1 / 1 14 / 34 = 41% +serialize_data(lsb_lua_sandbox*,int,lsb_output_buffer*) .../luasandbox_serialize.c 230 1 / 1 9 / 20 = 45% +heka_create_stream_reader(lua_State*) ...rc/heka/stream_reader.c 174 1 / 1 5 / 10 = 50% +lsb_get_output(lsb_lua_sandbox*,size_t*) ../src/luasandbox_output.c 110 1 / 1 2 / 4 = 50% +instruction_manager(lua_State*,lua_Debug*) ../src/luasandbox.c 119 1 / 1 1 / 2 = 50% +read_config(lua_State*) ../src/luasandbox.c 201 1 / 1 1 / 2 = 50% +set_random_seed() ../src/luasandbox.c 400 1 / 1 7 / 12 = 58% +lsb_read_file(const char*) ../src/util/util.c 49 1 / 1 6 / 10 = 60% +heka_encode_message_table(lsb_lua_sandbox*,int) ../src/heka/message.c 874 1 / 1 16 / 26 = 61% +min_expand(MatchState*,const char*,const char*,const char*) ...c/util/string_matcher.c 171 1 / 1 5 / 8 = 62% +encode_field_value(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 399 1 / 1 70 / 110 = 63% +preserve_global_data(lsb_lua_sandbox*) .../luasandbox_serialize.c 368 1 / 1 27 / 42 = 64% +lsb_init_heka_message(lsb_heka_message*,int) ../src/util/heka_message.c 429 1 / 1 4 / 6 = 66% +process_fields(lua_State*,const char*,const char*) ../src/heka/message.c 89 1 / 1 55 / 82 = 67% +encode_field_array(lsb_lua_sandbox*,lsb_output_buffer*,int,const char*,int) ../src/heka/message.c 326 1 / 1 19 / 28 = 67% +lsb_heka_pm_input(lsb_heka_sandbox*,double,const char*,_Bool) ../src/heka/sandbox.c 564 1 / 1 11 / 16 = 68% +set_restrictions(lua_State*,lsb_heka_sandbox*) ../src/heka/sandbox.c 308 1 / 1 20 / 29 = 68% +read_string(lua_State*,int,const char*,const char*) ../src/heka/message.c 48 1 / 1 7 / 10 = 70% +lsb_init(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 528 1 / 1 27 / 38 = 71% +check_int(lua_State*,int,const char*,int) ../src/luasandbox.c 254 1 / 1 5 / 7 = 71% +lsb_outputfd(lsb_output_buffer*,double) ...rc/util/output_buffer.c 177 1 / 1 29 / 40 = 72% +encode_fields(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 614 1 / 1 27 / 36 = 75% +serialize_kvp(lsb_lua_sandbox*,serialization_data*,size_t) .../luasandbox_serialize.c 288 1 / 1 18 / 24 = 75% +decode_header(char*,size_t,size_t) ../src/util/heka_message.c 25 1 / 1 12 / 16 = 75% +read_message(lua_State*) ../src/heka/sandbox.c 41 1 / 1 6 / 8 = 75% +mm_eval(lua_State*) ../src/heka/sandbox.c 74 1 / 1 6 / 8 = 75% +hsr_decode_message(lua_State*) ...rc/heka/stream_reader.c 27 1 / 1 6 / 8 = 75% +process_varint(lua_State*,const char*,int,int,const char*,const char*) ../src/heka/message.c 68 1 / 1 3 / 4 = 75% +encode_int(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 283 1 / 1 3 / 4 = 75% +output(lua_State*) ../src/luasandbox.c 127 1 / 1 3 / 4 = 75% +lsb_stop_sandbox(lsb_lua_sandbox*) ../src/luasandbox.c 621 1 / 1 3 / 4 = 75% +lsb_clear_heka_message(lsb_heka_message*) ../src/util/heka_message.c 443 1 / 1 3 / 4 = 75% +lsb_outputc(lsb_output_buffer*,char) ...rc/util/output_buffer.c 78 1 / 1 3 / 4 = 75% +lsb_outputs(lsb_output_buffer*,const char*,size_t) ...rc/util/output_buffer.c 146 1 / 1 3 / 4 = 75% +lsb_set_tz(const char*) ../src/util/util.c 143 1 / 1 3 / 4 = 75% +lsb_decode_heka_message(lsb_heka_message*,const char*,size_t,lsb_logger*) ../src/util/heka_message.c 207 1 / 1 59 / 77 = 76% +process_message(lsb_heka_sandbox*,lsb_heka_message*,lua_State*,int,_Bool) ../src/heka/sandbox.c 478 1 / 1 33 / 43 = 76% +lsb_pcall_setup(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 721 1 / 1 14 / 18 = 77% +read_string(int,const char*,const char*,lsb_const_string*) ../src/util/heka_message.c 45 1 / 1 8 / 10 = 80% +read_string_value(const char*,const char*,int,lsb_read_value*) ../src/util/heka_message.c 63 1 / 1 8 / 10 = 80% +read_integer_value(const char*,const char*,int,lsb_read_value*) ../src/util/heka_message.c 84 1 / 1 8 / 10 = 80% +check_string(lua_State*,int,const char*,const char*) ../src/luasandbox.c 232 1 / 1 4 / 5 = 80% +matchbalance(MatchState*,const char*,const char*) ...c/util/string_matcher.c 137 1 / 1 13 / 16 = 81% +lsb_outputf(lsb_output_buffer*,const char*,...) ...rc/util/output_buffer.c 92 1 / 1 23 / 28 = 82% +heka_read_message(lua_State*,lsb_heka_message*) ../src/heka/message.c 925 1 / 1 48 / 58 = 82% +heka_decode_message(lua_State*) ../src/heka/message.c 689 1 / 1 54 / 65 = 83% +hsr_find_message(lua_State*) ...rc/heka/stream_reader.c 56 1 / 1 25 / 30 = 83% +lsb_heka_timer_event(lsb_heka_sandbox*,time_t,_Bool) ../src/heka/sandbox.c 811 1 / 1 15 / 18 = 83% +set_missing_headers(lua_State*,int,lsb_heka_sandbox*) ../src/heka/message.c 28 1 / 1 10 / 12 = 83% +lsb_heka_pm_analysis(lsb_heka_sandbox*,lsb_heka_message*,_Bool) ../src/heka/sandbox.c 670 1 / 1 10 / 12 = 83% +ignore_value_type(lsb_lua_sandbox*,serialization_data*,int,lua_CFunction*) .../luasandbox_serialize.c 115 1 / 1 10 / 12 = 83% +inject_message_analysis(lua_State*) ../src/heka/sandbox.c 191 1 / 1 5 / 6 = 83% +lsb_destroy(lsb_lua_sandbox*) ../src/luasandbox.c 634 1 / 1 5 / 6 = 83% +lsb_pb_output_varint(char*,unsigned long long) ../src/util/protobuf.c 56 1 / 1 5 / 6 = 83% +lsb_output(lsb_lua_sandbox*,int,int,int) ../src/luasandbox_output.c 42 1 / 1 27 / 32 = 84% +heka_encode_message(lua_State*) ../src/heka/message.c 816 1 / 1 11 / 13 = 84% +match_class(int,int) ...c/util/string_matcher.c 68 1 / 1 11 / 13 = 84% +match(MatchState*,const char*,const char*) ...c/util/string_matcher.c 183 1 / 1 39 / 46 = 84% +inject_message_input(lua_State*) ../src/heka/sandbox.c 112 1 / 1 18 / 21 = 85% +lsb_heka_pm_output(lsb_heka_sandbox*,lsb_heka_message*,void*,_Bool) ../src/heka/sandbox.c 785 1 / 1 12 / 14 = 85% +lsb_expand_input_buffer(lsb_input_buffer*,size_t) ../src/util/input_buffer.c 49 1 / 1 12 / 14 = 85% +numeric_test(match_node*,double) .../heka_message_matcher.c 67 1 / 1 6 / 7 = 85% +lsb_find_heka_message(lsb_heka_message*,lsb_input_buffer*,_Bool,size_t*,lsb_logger*) ../src/util/heka_message.c 331 1 / 1 31 / 36 = 86% +output_print(lua_State*) ../src/luasandbox.c 143 1 / 1 19 / 22 = 86% +load_sandbox_config(const char*,lsb_logger*) ../src/luasandbox.c 285 1 / 1 19 / 22 = 86% +matchbracketclass(int,const char*,const char*) ...c/util/string_matcher.c 99 1 / 1 19 / 22 = 86% +update_checkpoint(lua_State*) ../src/heka/sandbox.c 275 1 / 1 7 / 8 = 87% +lsb_init_input_buffer(lsb_input_buffer*,size_t) ../src/util/input_buffer.c 20 1 / 1 7 / 8 = 87% +lsb_outputd(lsb_output_buffer*,double) ...rc/util/output_buffer.c 160 1 / 1 7 / 8 = 87% +process_fields(lsb_heka_field*,const char*,const char*) ../src/util/heka_message.c 127 1 / 1 50 / 57 = 87% +copy_table(lua_State*,lua_State*,lsb_logger*) ../src/luasandbox.c 327 1 / 1 22 / 25 = 88% +lsb_expand_output_buffer(lsb_output_buffer*,size_t) ...rc/util/output_buffer.c 52 1 / 1 16 / 18 = 88% +inject_payload(lua_State*) ../src/heka/sandbox.c 216 1 / 1 18 / 20 = 90% +lsb_add_function(lsb_lua_sandbox*,lua_CFunction,const char*) ../src/luasandbox.c 710 1 / 1 9 / 10 = 90% +lsb_init_output_buffer(lsb_output_buffer*,size_t) ...rc/util/output_buffer.c 27 1 / 1 9 / 10 = 90% +eval_node(match_node*,lsb_heka_message*) .../heka_message_matcher.c 89 1 / 1 31 / 34 = 91% +eval_tree(match_node*,match_node*,lsb_heka_message*) .../heka_message_matcher.c 151 1 / 1 22 / 24 = 91% +memory_manager(void*,void*,size_t,size_t) ../src/luasandbox.c 83 1 / 1 11 / 12 = 91% +lsb_read_heka_field(const lsb_heka_message*,lsb_const_string*,int,int,lsb_read_value*) ../src/util/heka_message.c 474 1 / 1 24 / 26 = 92% +classend(const char*) ...c/util/string_matcher.c 43 1 / 1 16 / 17 = 94% +string_test(match_node*,lsb_const_string*) .../heka_message_matcher.c 21 1 / 1 18 / 19 = 94% +encode_string(lsb_lua_sandbox*,lsb_output_buffer*,char,const char*,int) ../src/heka/message.c 254 1 / 1 2 / 2 = 100% +encode_field_object(lsb_lua_sandbox*,lsb_output_buffer*) ../src/heka/message.c 377 1 / 1 4 / 4 = 100% +mm_check(lua_State*) ../src/heka/sandbox.c 58 1 / 1 0 / 0 +mm_gc(lua_State*) ../src/heka/sandbox.c 66 1 / 1 0 / 0 +mm_create(lua_State*) ../src/heka/sandbox.c 91 1 / 1 4 / 4 = 100% +lsb_heka_stop_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 761 1 / 1 0 / 0 +lsb_heka_terminate_sandbox(lsb_heka_sandbox*,const char*) ../src/heka/sandbox.c 767 1 / 1 0 / 0 +lsb_heka_destroy_sandbox(lsb_heka_sandbox*) ../src/heka/sandbox.c 773 1 / 1 2 / 2 = 100% +lsb_heka_get_error(lsb_heka_sandbox*) ../src/heka/sandbox.c 853 1 / 1 2 / 2 = 100% +lsb_heka_get_lua_file(lsb_heka_sandbox*) ../src/heka/sandbox.c 859 1 / 1 2 / 2 = 100% +lsb_heka_get_stats(lsb_heka_sandbox*) ../src/heka/sandbox.c 865 1 / 1 2 / 2 = 100% +lsb_heka_is_running(lsb_heka_sandbox*) ../src/heka/sandbox.c 886 1 / 1 4 / 4 = 100% +lsb_heka_get_message(lsb_heka_sandbox*) ../src/heka/sandbox.c 894 1 / 1 2 / 2 = 100% +lsb_heka_get_type(lsb_heka_sandbox*) ../src/heka/sandbox.c 901 1 / 1 2 / 2 = 100% +check_hsr(lua_State*,int) ...rc/heka/stream_reader.c 18 1 / 1 0 / 0 +hsr_read_message(lua_State*) ...rc/heka/stream_reader.c 142 1 / 1 6 / 6 = 100% +hsr_gc(lua_State*) ...rc/heka/stream_reader.c 154 1 / 1 0 / 0 +libsize(const luaL_Reg*) ../src/luasandbox.c 50 1 / 1 2 / 2 = 100% +preload_modules(lua_State*) ../src/luasandbox.c 57 1 / 1 2 / 2 = 100% +instruction_usage(lsb_lua_sandbox*) ../src/luasandbox.c 113 1 / 1 0 / 0 +unprotected_panic(lua_State*) ../src/luasandbox.c 215 1 / 1 0 / 0 +get_int(lua_State*,int,const char*) ../src/luasandbox.c 223 1 / 1 0 / 0 +stop_hook(lua_State*,lua_Debug*) ../src/luasandbox.c 613 1 / 1 0 / 0 +lsb_usage(lsb_lua_sandbox*,lsb_usage_type,lsb_usage_stat) ../src/luasandbox.c 657 1 / 1 10 / 10 = 100% +lsb_get_error(lsb_lua_sandbox*) ../src/luasandbox.c 667 1 / 1 2 / 2 = 100% +lsb_set_error(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 673 1 / 1 4 / 4 = 100% +lsb_get_lua(lsb_lua_sandbox*) ../src/luasandbox.c 686 1 / 1 2 / 2 = 100% +lsb_get_lua_file(lsb_lua_sandbox*) ../src/luasandbox.c 692 1 / 1 2 / 2 = 100% +lsb_get_parent(lsb_lua_sandbox*) ../src/luasandbox.c 698 1 / 1 2 / 2 = 100% +lsb_get_state(lsb_lua_sandbox*) ../src/luasandbox.c 704 1 / 1 2 / 2 = 100% +lsb_pcall_teardown(lsb_lua_sandbox*) ../src/luasandbox.c 745 1 / 1 4 / 4 = 100% +lsb_terminate(lsb_lua_sandbox*,const char*) ../src/luasandbox.c 759 1 / 1 6 / 6 = 100% +lsb_add_output_function(lua_State*,lua_CFunction) ../src/luasandbox_output.c 22 1 / 1 0 / 0 +lsb_get_output_function(lua_State*,int) ../src/luasandbox_output.c 30 1 / 1 0 / 0 +get_serialize_function(lua_State*,int) .../luasandbox_serialize.c 93 1 / 1 0 / 0 +find_table_ref(table_ref_array*,const void*) .../luasandbox_serialize.c 150 1 / 1 4 / 4 = 100% +get_preservation_version(lua_State*) .../luasandbox_serialize.c 196 1 / 1 4 / 4 = 100% +serialize_table(lsb_lua_sandbox*,serialization_data*,size_t) .../luasandbox_serialize.c 215 1 / 1 6 / 6 = 100% +file_exists(const char*) .../luasandbox_serialize.c 477 1 / 1 2 / 2 = 100% +lsb_add_serialize_function(lua_State*,lua_CFunction) .../luasandbox_serialize.c 523 1 / 1 0 / 0 +lsb_serialize_binary(lsb_output_buffer*,const void*,size_t) .../luasandbox_serialize.c 531 1 / 1 11 / 11 = 100% +lsb_serialize_double(lsb_output_buffer*,double) .../luasandbox_serialize.c 558 1 / 1 6 / 6 = 100% +read_double_value(const char*,const char*,int,lsb_read_value*) ../src/util/heka_message.c 103 1 / 1 2 / 2 = 100% +process_varint(int,const char*,const char*,long long*) ../src/util/heka_message.c 116 1 / 1 4 / 4 = 100% +lsb_free_heka_message(lsb_heka_message*) ../src/util/heka_message.c 463 1 / 1 2 / 2 = 100% +lsb_write_heka_uuid(lsb_output_buffer*,const char*,size_t) ../src/util/heka_message.c 519 1 / 1 22 / 22 = 100% +lsb_destroy_message_matcher(lsb_message_matcher*) .../heka_message_matcher.c 175 1 / 1 6 / 6 = 100% +lsb_eval_message_matcher(lsb_message_matcher*,lsb_heka_message*) .../heka_message_matcher.c 195 1 / 1 0 / 0 +lsb_free_input_buffer(lsb_input_buffer*) ../src/util/input_buffer.c 36 1 / 1 2 / 2 = 100% +lsb_free_output_buffer(lsb_output_buffer*) ...rc/util/output_buffer.c 42 1 / 1 2 / 2 = 100% +lsb_pb_read_key(const char*,int*,int*) ../src/util/protobuf.c 15 1 / 1 8 / 8 = 100% +lsb_pb_write_key(lsb_output_buffer*,unsigned char,unsigned char) ../src/util/protobuf.c 25 1 / 1 2 / 2 = 100% +lsb_pb_read_varint(const char*,const char*,long long*) ../src/util/protobuf.c 36 1 / 1 22 / 22 = 100% +lsb_pb_write_varint(lsb_output_buffer*,unsigned long long) ../src/util/protobuf.c 75 1 / 1 2 / 2 = 100% +lsb_pb_write_bool(lsb_output_buffer*,int) ../src/util/protobuf.c 85 1 / 1 4 / 4 = 100% +lsb_pb_write_double(lsb_output_buffer*,double) ../src/util/protobuf.c 99 1 / 1 2 / 2 = 100% +lsb_pb_write_string(lsb_output_buffer*,char,const char*,size_t) ../src/util/protobuf.c 114 1 / 1 6 / 6 = 100% +lsb_pb_update_field_length(lsb_output_buffer*,size_t) ../src/util/protobuf.c 127 1 / 1 8 / 8 = 100% +lsb_init_running_stats(lsb_running_stats*) ...rc/util/running_stats.c 13 1 / 1 0 / 0 +lsb_update_running_stats(lsb_running_stats*,double) ...rc/util/running_stats.c 21 1 / 1 4 / 4 = 100% +lsb_sd_running_stats(lsb_running_stats*) ...rc/util/running_stats.c 37 1 / 1 2 / 2 = 100% +lsb_init_const_string(lsb_const_string*) ../src/util/string.c 11 1 / 1 0 / 0 +singlematch(int,const char*,const char*) ...c/util/string_matcher.c 119 1 / 1 4 / 4 = 100% +max_expand(MatchState*,const char*,const char*,const char*) ...c/util/string_matcher.c 156 1 / 1 10 / 10 = 100% +lsb_string_match(const char*,size_t,const char*) ...c/util/string_matcher.c 261 1 / 1 10 / 10 = 100% +lsb_lp2(unsigned long long) ../src/util/util.c 35 1 / 1 2 / 2 = 100% +lsb_get_time() ../src/util/util.c 75 1 / 1 0 / 0 +lsb_get_timestamp() ../src/util/util.c 114 1 / 1 0 / 0 +------------------------------------------------------------------------------------------------ -------------------------------- ----- --------------------- +Total 100% 1568 / 2029 = 77% diff --git a/docs/bloom_filter.md b/docs/bloom_filter.md deleted file mode 100644 index 8680c48..0000000 --- a/docs/bloom_filter.md +++ /dev/null @@ -1,48 +0,0 @@ -Lua Bloom Filter Library -======================== - -The library is implemented in the _bloom_filter_ table. - -Constructor ------------ -**bloom_filter.new** (items, probability) - -*Arguments* -- items (unsigned) The maximum number of items to be inserted into the filter (must be > 1) -- probability (double) The probability of false positives (must be between 0 and 1) - -*Return* - -A bloom filter object. - -Methods -------- -bool **add** (key) - -*Arguments* -- key (string/number) The key to add to the bloom filter. - -*Return* - -True if the key was added, false if it already existed. - -____ -bool **query** (key) - -*Arguments* -- key (string/number) The key to lookup in the bloom filter. - -*Return* - -True if the key exists, false if it doesn't. - -____ -void **clear** () - -*Arguments* - -none - -*Return* - -none diff --git a/docs/circular_buffer.md b/docs/circular_buffer.md deleted file mode 100644 index db4505b..0000000 --- a/docs/circular_buffer.md +++ /dev/null @@ -1,207 +0,0 @@ -Lua Circular Buffer Library -=========================== - -The library is a sliding window time series data store and is implemented in the _circular_buffer_ table. - -Constructor ------------ -**circular_buffer.new** (rows, columns, seconds_per_row, enable_delta) - -*Arguments* -- rows (unsigned) The number of rows in the buffer (must be > 1) -- columns (unsigned)The number of columns in the buffer (must be > 0) -- seconds_per_row (unsigned) The number of seconds each row represents (must be > 0). -- enable_delta (**optional, default false** bool) When true the changes made to the - circular buffer between delta outputs are tracked. - -*Return* - -A circular buffer object. - -Methods -------- -**Note:** All column arguments are 1 based. If the column is out of range for the configured circular buffer a fatal error is generated. - -double **add** (nanoseconds, column, value) - -*Arguments* -- nanosecond (unsigned) The number of nanosecond since the UNIX epoch. The value is - used to determine which row is being operated on. -- column (unsigned) The column within the specified row to perform an add operation on. -- value (double) The value to be added to the specified row/column. - -*Return* - -The value of the updated row/column or nil if the time was outside the range of the buffer. - -____ -double **set** (nanoseconds, column, value) - -*Arguments* -- nanosecond (unsigned) The number of nanosecond since the UNIX epoch. The value is - used to determine which row is being operated on. -- column (unsigned) The column within the specified row to perform a set operation on. -- value (double) The value to be overwritten at the specified row/column. - For aggregation methods "min" and "max" the value is only overwritten if it is smaller/larger than the current value. - -*Return* - -The resulting value of the row/column or nil if the time was outside the range of the buffer. - -____ -double **get** (nanoseconds, column) - -*Arguments* -- nanosecond (unsigned) The number of nanosecond since the UNIX epoch. The value is used - to determine which row is being operated on. -- column (unsigned) The column within the specified row to retrieve the data from. - -*Return* - -The value at the specifed row/column or nil if the time was outside the range of the buffer. - -____ -double, double, double **get_configuration** () - -*Arguments* -- none - -*Return* - -The circular buffer dimension values specified in the constructor. -- rows -- columns -- seconds_per_row - -____ -int **set_header** (column, name, unit, aggregation_method) - -*Arguments* -- column (unsigned) The column number where the header information is applied. -- name (string) Descriptive name of the column (maximum 15 characters). Any non alpha - numeric characters will be converted to underscores. (default: Column_N) -- unit (string - optional) The unit of measure (maximum 7 characters). Alpha numeric, - '/', and '*' characters are allowed everything else will be converted to underscores. - i.e. KiB, Hz, m/s (default: count) -- aggregation_method (string - optional) Controls how the column data is aggregated - when combining multiple circular buffers. - - **sum** The total is computed for the time/column (default). - - **min** The smallest value is retained for the time/column. - - **max** The largest value is retained for the time/column. - - **none** No aggregation will be performed the column. - -*Return* - -The column number passed into the function. - -____ -string, string, string **get_header** (column) - -*Arguments* -- column (unsigned) The column number of the header information to be retrieved. - -*Return* - -The current values of specified header column. -- name -- unit -- aggregation_method - -____ -double, int **compute** (function, column, start, end) - -*Arguments* -- function (string) The name of the compute function (sum|avg|sd|min|max|variance). -- column (unsigned) The column that the computation is performed against. -- start (optional - unsigned) The number of nanosecond since the UNIX epoch. Sets the - start time of the computation range; if nil the buffer's start time is used. -- end (optional- unsigned) The number of nanosecond since the UNIX epoch. Sets the - end time of the computation range (inclusive); if nil the buffer's end time is used. - The end time must be greater than or equal to the start time. - -*Returns* - -- The result of the computation for the specifed column over the given range or nil if the range fell outside of the buffer. -- The number of rows that contained a valid numeric value. - -____ -double, double **mannwhitneyu** (column, start_x, end_x, start_y, end_y, use_continuity) - -Computes the Mann-Whitney rank test on samples x and y. - -*Arguments* -- column (unsigned) The column that the computation is performed against. -- start_1 (unsigned) The number of nanosecond since the UNIX epoch. -- end_1 (unsigned) The number of nanosecond since the UNIX epoch. The end time must be greater than or equal to the start time. -- start_2 (unsigned). -- end_2 (unsigned). -- use_continuity (optional - bool) Whether a continuity correction (1/2) should be taken into account (default: true). - -*Returns* (nil if the range fell outside the buffer) - -- U_1 Mann-Whitney statistic. -- One-sided p-value assuming a asymptotic normal distribution. - -**Note:** Use only when the number of observation in each sample is > 20 and you have 2 independent samples of ranks. -Mann-Whitney U is significant if the u-obtained is LESS THAN or equal to the critical value of U. - -This test corrects for ties and by default uses a continuity correction. The reported p-value is for a one-sided -hypothesis, to get the two-sided p-value multiply the returned p-value by 2. - -____ -double **current_time** () - -*Arguments* -- none - -*Return* - -The time of the most current row in the circular buffer. - -____ -cbuf **format** (format) - Sets an internal flag to control the output format of the circular buffer data structure; if deltas are not enabled or there haven't been any modifications, nothing is output. - -*Arguments* -- format (string) - - **cbuf** The circular buffer full data set format. - - **cbufd** The circular buffer delta data set format. - -*Return* - -The circular buffer object. - -Output ------- -The circular buffer can be passed to the output() function. The output format -can be selected using the format() function. - -The cbuf (full data set) output format consists of newline delimited rows -starting with a json header row followed by the data rows with tab delimited -columns. The time in the header corresponds to the time of the first data row, -the time for the other rows is calculated using the seconds_per_row header value. - - {json header} - row1_col1\trow1_col2\n - . - . - . - rowN_col1\trowN_col2\n - -The cbufd (delta) output format consists of newline delimited rows starting with -a json header row followed by the data rows with tab delimited columns. The -first column is the timestamp for the row (time_t). The cbufd output will only -contain the rows that have changed and the corresponding delta values for each -column. - - {json header} - row14_timestamp\trow14_col1\trow14_col2\n - row10_timestamp\trow10_col1\trow10_col2\n - -Sample Cbuf Output ------------------- - - {"time":2,"rows":3,"columns":3,"seconds_per_row":60,"column_info":[{"name":"HTTP_200","unit":"count","aggregation":"sum"},{"name":"HTTP_400","unit":"count","aggregation":"sum"},{"name":"HTTP_500","unit":"count","aggregation":"sum"}]} - 10002 0 0 - 11323 0 0 - 10685 0 0 diff --git a/docs/cli/index.md b/docs/cli/index.md new file mode 100644 index 0000000..f0594f9 --- /dev/null +++ b/docs/cli/index.md @@ -0,0 +1,24 @@ +# Command Line Interface Tools + +## lsb_heka_cat + +A command-line utility for counting, viewing, filtering, and extracting messages +from a Heka protobuf log/stream. + +``` +usage: lsb_heka_cat [-t|-c|-h] [-m message_matcher] [-f] [-n #] +description: + -t output the messages in text format (default) + -c only output the message count + -h output the messages as a Heka protobuf stream + -f output appended data as the file grows + -n output the last # of messages (simple header check so not 100% accurate) + -m message_matcher expression (default "TRUE") + FILE name of the file to cat or '-' for stdin +notes: + All output is written to stdout and all log/error messages are written to stderr. + +``` +See the [message matcher](/util/message_matcher.md) documentation for more +details about the message_matcher expression. + diff --git a/docs/heka/analysis.md b/docs/heka/analysis.md new file mode 100644 index 0000000..bae001a --- /dev/null +++ b/docs/heka/analysis.md @@ -0,0 +1,210 @@ +# Analysis Sandbox Interface + +## Recommendations +Since the sandbox does not run in isolation there are some expectations of how +the host infrastructure behaves. The current recommendations are based on the +Hindsight reference implementation. + +## Disabled Functionality +- [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - collectgarbage, dofile, load, loadfile, loadstring, newproxy +- [coroutine](http://www.lua.org/manual/5.1/manual.html#5.2) + - the entire module is inaccessible +- [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - dump +- [io](http://www.lua.org/manual/5.1/manual.html#5.7) + - the entire module is inaccessible +- [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - getenv, execute, exit, remove, rename, setlocale, tmpname + +## Required Lua Functions (called by the host) + +### process_message + +Called when the host has a message available for analysis. Usually used in +combination with a [message matcher](/util/message_matcher.md) expression. + +Recommenation: specify this as a `message_matcher` configuration option. + +*Arguments* +* none + +*Return* +* status_code (number) + * success (less than or equal to zero) + * fatal error (greater than zero) +* status_message (optional: string) logged when the status code is less than + zero + +### timer_event + +Called when the host timer expires or on shutdown. + +Recommendation: specify this as a `ticker_interval` configuration option. + +*Arguments* +* ns (number) - nanosecond timestamp of the function call (it is actually + `time_t * 1e9` to keep the timestamp units consistent so it will only have a + one second resolution) +* shutdown (bool) - true if timer_event is being called due to a shutdown + +*Return* +* none + +## Available C Functions (called from the plugin) + +### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +### read_message + +Provides access to the Heka message data. Note that both fieldIndex and +arrayIndex are zero-based (i.e. the first element is 0) as opposed to Lua's +standard indexing, which is one-based. + +*Arguments* +* variableName (string) + * framed (returns the Heka message protobuf string including the framing + header) + * raw (returns the Heka message protobuf string) + * size (returns the size of the raw Heka message protobuf string) + * Uuid + * Type + * Logger + * Payload + * EnvVersion + * Hostname + * Timestamp + * Severity + * Pid + * Fields[*name*] +* fieldIndex (unsigned) - only used in combination with the Fields variableName +use to retrieve a specific instance of a repeated field *name*; zero indexed +* arrayIndex (unsigned) - only used in combination with the Fields variableName +use to retrieve a specific element out of a field containing an array; zero +indexed +* zeroCopy (bool, optional default false) - returns a userdata place holder for +the message variable (only valid for string types). Non string headers throw an +error during construction, non string fields throw an error on data retrieval. + +*Return* +* value (number, string, bool, nil, userdata depending on the type of variable + requested) + +### decode_message + +Converts a Heka protobuf encoded message string into a Lua table or throws an +error. + +*Arguments* +* heka_pb (string, userdata) - Heka protobuf binary string or a zero copy + userdata object containing a Heka protobuf binary string. + +*Return* +* msg ([Heka message table (array fields)](message.md#array-based-message-fields)) +with the value member always being an array (even if there is only a +single item). This format makes working with the output more consistent. The +wide variation in the inject table formats is to ease the construction of the +message especially when using an LPeg grammar transformation. + +### inject_message + +Creates a new Heka protocol buffer message using the contents of the specified +Lua table (overwriting whatever is in the payload buffer). `Timestamp`, +`Logger`, `Hostname` and `Pid` are restricted header values. An override +configuration option is provided `restricted_headers`; when true (default) the +headers are always set to the configuration values; when false the headers are +set to the values provided in the message table, if no value is provided it +defaults to the appropriate value. + +*Arguments* +* msg ([Heka message table](message.md)) + +*Return* +* none (throws an error if the table does not match the Heka message schema) + +### add_to_payload + +Appends the arguments to the payload buffer for incremental construction of the +final payload output (`inject_payload` finalizes the buffer and sends the +message to the infrastructure). This function is a rename of the generic sandbox +output function to improve the readability of the plugin code. + +*Arguments* +* arg (number, string, bool, nil, supported userdata) + +*Return* +* none (throws an error if arg is an unsupported type) + +### inject_payload + +This is a wrapper function for `inject_message` that is included for backwards +compatibility. The function creates a new Heka message using the contents of the +payload buffer (pre-populated with `add_to_payload`) and combined with any +additional payload_args passed here. The payload buffer is cleared after the +injection. The `payload_type` and `payload_name` arguments are two pieces of +optional metadata stored is message fields. The resulting message is structured +like this: +```lua +msg = { +Timestamp = +Uuid = "" +Hostname = "" +Logger = "" +Type = "inject_payload" +Payload = "" +Fields = { + payload_type = "txt", + payload_name = "", +} +``` + +*Arguments* + +* payload_type (optional: string, default "txt") - describes the content type of + the injected payload data +* payload_name (optional: string, default "") - names the content to aid in + downstream filtering +* arg3 (optional) -ame type restrictions as `add_to_payload` +* ... +* argN + +*Return* +* none (throws an error if arg is an unsupported type) + +### Modes of Operation + +#### Lock Step +* Receives one call to `process_message`, operates on the message, and returns + success (0) or failure (-1) + +#### Example simple counter plugin +```lua +-- cfg +message_matcher = "TRUE" +ticker_interval = 5 +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local cnt = 0 + +function process_message() + cnt = cnt + 1 + return 0 +end + +function timer_event() + inject_payload("txt", "count", cnt) +end +``` diff --git a/docs/heka/index.md b/docs/heka/index.md new file mode 100644 index 0000000..1fcd83d --- /dev/null +++ b/docs/heka/index.md @@ -0,0 +1,66 @@ +# Heka Sandbox Interface + +## Overview + +This document describes the 1.0 release of the Heka sandbox API built on the +[Generic Sandbox Interface](/sandbox.md). The 1.0 release is a refined +implementation of its predecessor which was developed in +[Heka](https://github.com/mozilla-services/heka). The goal is to decouple it from +Go and make it easily embeddable in any language. The Go version of Heka has +been deprecated and replaced by +[Hindsight](https://github.com/mozilla-services/hindsight). + +## Sandbox API Changes from the Go Heka Sandbox + +There are a few intentional changes between tho original Heka sandbox and this +version. + +### Changes + +#### Input Sandbox + +1. `inject_message` accepts a numeric or string checkpoint as the second +argument +1. `process_message` receives the checkpoint value as the first argument (if it +was provided by `inject_message`). + +#### Output Sandbox + +1. `process_message` receives a sequence_id as the first argument (if it was +provided by `update_checkpoint`). Extended return codes have been added to +support skipping, retrying, batching, and asynchronous output. + +#### Analysis/Output Sandbox + +1. `read_message` + * returns `nil` for optional header fields if they don't exist instead of an + empty string or zero + * added a `framed` parameter to retrieve the raw message with stream framing + * added a `size` parameter to retrieve size of the raw message without having + to copy it down +1. `timer_event` has a second parameter `shutdown` that is set to true when the +sandbox is exiting. + +### Additions + +#### Input Sandbox + +1. [create_stream_reader](input.md#createstreamreader) function was added. +1. [is_running](input.md#is_running) function was added. + +#### Output Sandbox + +1. [update_checkpoint](output.md#update_checkpoint) was added for batch and +asynchronous processing. +1. [create_message_matcher](output.md#createmessagematcher) function was added. + +### Removals + +1. The `write_message` API was removed; messages are immutable and this API +broke that rule. +1. The `read_next_field` API was removed; instead the raw message should be +decoded and the Lua table iterated. + +### Notes + +1. The `read_config` API in unchanged but now has access to the entire sandbox configuration. diff --git a/docs/heka/input.md b/docs/heka/input.md new file mode 100644 index 0000000..8b33812 --- /dev/null +++ b/docs/heka/input.md @@ -0,0 +1,245 @@ +# Input Sandbox Interface + +## Recommendations +Since the sandbox does not run in isolation there are some expectations of how +the host infrastructure behaves. The current recommendations are based on the +Hindsight reference implementation. + +## Disabled Functionality +- [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - dofile, load, loadfile, loadstring, newproxy +- [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - dump +- [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - exit, setlocale + +## Required Lua Functions (called by the host) + +### process_message + +Entry point for message creation. + +*Arguments* +* checkpoint (nil number, string) - value of the last checkpoint value passed +into `inject_message` + +*Return* +* status_code (number) + - success (less than or equal to zero) + - fatal error (greater than zero) +* status_message (optional: string) logged when the status code is less than +zero + +## Available C Functions (called from the plugin) + +### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +### is_running + +Provides a synchronization point for collecting statistics and communicating +shutdown status. + +*Arguments* +* none + +*Return* +* running (boolean) - true if a sandbox state is LSB_RUNNING, false if not or + throws an error if the request to the host fails. + +### decode_message + +Converts a Heka protobuf encoded message string into a Lua table. See +[decode_message](analysis.md#decodemessage) for details. + +### inject_message + +Sends a Heka protocol buffer message into the host. For the Heka message table +arguments `Timestamp`, `Logger`, `Hostname` and `Pid` are restricted header +values. An override configuration option is provided `restricted_headers`; when +true the headers are always set to the configuration values; when false +(default) the headers are set to the values provide in the message table, +if no value is provided it defaults to the appropriate value. + +*Arguments* +* msg ([Heka message table](message.md), + [Heka stream reader](#heka-stream-reader-methods), + Heka protobuf string, + or nil (if only updating the checkpoint)) +* checkpoint (optional: number, string) - checkpoint to be returned in the + `process_message` call + +*Return* +* none - throws an error on invalid input + +### create_stream_reader +Creates a Heka stream reader to enable parsing of a framed Heka protobuf stream +in a Lua sandbox. See: +[Example of a Heka protobuf reader](#example-of-a-heka-protobuf-stdin-reader) + +*Arguments* +* name (string) - name of the stream reader (used in the log) + +*Return* +* hsr (userdata) - Heka stream reader or an error is thrown + +#### Heka Stream Reader Methods + +##### find_message + +Locates a Heka message within the stream. + +```lua +local found, consumed, need = hsr:find_message(buf) + +``` + +*Arguments* +* buf (string, userdata (FILE*)) - buffer containing a Heka protobuf stream data + or a userdata file object +* decode (bool default: true) - true if the framed message should be protobuf + decoded + +*Return* +* found (bool) - true if a message was found +* consumed (number) - number of bytes consumed so the offset can be tracked for + checkpointing purposes +* need/read (number) - number of bytes needed to complete the message or fill + the underlying buffer or in the case of a file object the number of bytes + added to the buffer + +##### decode_message + +Converts a Heka protobuf encoded message string into a stream reader +representation. Note: this operation clears the internal stream reader buffer. + +*Arguments* +* heka_pb (string) - Heka protobuf binary string + +*Return* +* none - throws an error on failure + +##### read_message + +Provides access to the Heka message data within the reader object. The zeroCopy +flag is not accepted here. + +```lua +local ts = hsr:read_message("Timestamp") + +``` +See [read_message](analysis.md#readmessage) for details. + +## Modes of Operation + +### Run Once +* Set the `ticker_interval` to zero and return from `process_message` done. +* The `instruction_limit` configuration can be set if desired. + +#### Example startup ping +```lua +-- cfg +-- send a simple 'hello' messages every time the host is started +ticker_interval = 0 +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- the checkpoint is optional and not being used by this plugin +function process_message() + inject_message({Type = "hello"}) + return 0 +end + +``` + +### Polling + +* Set the `ticker_interval` greater than zero and non fatally (<=0) return from + `process_message`, when the ticker interval expires `process_message` will be + called again. +* The `instruction_limit` configuration can be set if desired. + +#### Example heartbeat ping +```lua +-- cfg +ticker_interval = 60 +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "io" + +local msg = { +Type = "uptime", +Payload = "", +} + +function process_message() + local fh = io.popen("uptime") + if fh then + msg.Payload = fh:read("*a") + fh:close() + else + return -1, "popen failed" + end + if msg.Payload then inject_message(msg) end + return 0 +end + +``` + +### Continuous + +* Don't return from `process_message`. +* The `instruction_limit` configuration **MUST** be set to zero. + +#### Example of a Heka protobuf stdin reader + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +--[[ +Reads a Heka protobuf stream from the stdin file handle + +-- .cfg +instruction_limit = 0 + +--]] + + +local stdin = require "io".stdin +require "string" + +local hsr = create_stream_reader(read_config("Logger")) + +function process_message() + local cnt = 0 + local found, consumed, read + repeat + repeat + found, consumed, read = hsr:find_message(stdin) + if found then + inject_message(hsr) + cnt = cnt + 1 + end + until not found + until read == 0 + return 0, string.format("processed %d messages", cnt) +end +``` diff --git a/docs/heka/message.md b/docs/heka/message.md new file mode 100644 index 0000000..6504c85 --- /dev/null +++ b/docs/heka/message.md @@ -0,0 +1,32 @@ +# Heka Message Table + +## Hash Based Message Fields + +```lua +{ +Uuid = "data", -- restricted header, auto generated if not a 16 byte raw binary UUID or a 36 character human readable UUID +Logger = "nginx", -- restricted header, defaults to the Logger configuration value +Hostname = "example.com", -- restricted header, defaults to the Hostname configuration value +Timestamp = 1e9, -- restricted header, defaults to the current time +Type = "TEST", +Payload = "Test Payload", +EnvVersion = "0.8", +Pid = 1234, -- restricted header, defaults to the Pid configuration value +Severity = 6, +Fields = { + http_status = 200, -- encoded as a double + request_size = {value=1413, value_type=2, representation="B"} -- encoded as an integer + } +} +``` + +## Array Based Message Fields +```lua +{ +-- Message headers are the same as above +Fields = { + {name="http_status" , value=200}, -- encoded as a double + {name="request_size", value=1413, value_type=2, representation="B"} -- encoded as an integer + } +} +``` diff --git a/docs/heka/output.md b/docs/heka/output.md new file mode 100644 index 0000000..c58a20e --- /dev/null +++ b/docs/heka/output.md @@ -0,0 +1,421 @@ +# Output Sandbox Interface + +## Recommendations +Since the sandbox does not run in isolation there are some expectations of how +the host infrastructure behaves. The current recommendations are based on the +Hindsight reference implementation. + +## Disabled Functionality +- [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - dofile, load, loadfile, loadstring, newproxy +- [string](http://www.lua.org/manual/5.1/manual.html#5.4) + - dump +- [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - exit, setlocale + +## Required Lua Functions (called by the host) + +### process_message + +Called when the host has a message available for analysis. Usually used in +combination with a [message matcher](/util/message_matcher.md) expression. + +Recommenation: specify this as a `message_matcher` configuration option. + +*Arguments* +* sequence_id (optional: lightuserdata) - pass in when `async_buffer_size` is + configured + +*Return* +* status_code (number) - see the [Modes of Operation](#modes-of-operation) for + a detail explanation of the return codes + * fatal error (greater than zero) + * success (0) + * non fatal failure (-1) + * skip (-2) + * retry (-3) + * batching (-4) + * async output (-5) +* status_message (optional: string) logged when the status code is -1 + +### timer_event + +Called when the host timer expires or on shutdown. + +Recommendation: specify this as a `ticker_interval` configuration option. + +*Arguments* +* ns (number) - nanosecond timestamp of the function call (it is actually + `time_t * 1e9` to keep the timestamp units consistent so it will only have a + one second resolution) +* shutdown (bool) - true if timer_event is being called due to a shutdown + +*Return* +* none + +## Available C Functions (called from the plugin) + +### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +### read_message + +Provides access to the Heka message data. See +[read_message](analysis.md#readmessage) for details. + +### decode_message + +Converts a Heka protobuf encoded message string into a Lua table. See +[decode_message](analysis.md#decodemessage) for details. + +### encode_message + +Returns a Heka protocol buffer message using the contents of the specified Lua +table. `Timestamp`, `Logger`, `Hostname` and `Pid` are restricted header values. +An override configuration option is provided `restricted_headers`; when true the +headers are always set to the configuration values; when false (default) the +headers are set to the values provided in the message table, if no value is +provided it defaults to the appropriate configuration value. + +Note: this operation uses the internal output buffer so it is goverened by the +`output_limit` configuration setting. + +*Arguments* +* msg ([Heka message table](message.md)) +* framed (bool default: false) A value of true includes the framing header + +*Return* +* heka_pb (string) - Heka protobuf binary string, framed as specified + or an error is thrown + +### inject_message + +Creates a new Heka protocol buffer message, in the input queue, using the +contents of the specified Lua table. The `restricted_headers` configuration +defaults to false (see encode_message above for a full description). + +*Arguments* +* msg ([Heka message table](message.md)) + +*Return* +* none (throws an error if the table does not match the Heka message schema) + +### create_message_matcher + +Returns a Heka protocol buffer message matcher; used to dynamically filter +messages sent to the output plugin. + +*Arguments* +* message_matcher [message matcher](/util/message_matcher.md) + +*Return* +* message_matcher (userdata) - or an error is thrown + The message matcher object has one method `eval` that returns true if the + current message matches, false if it does not. + +#### Example + +See: [heka_tcp_matcher.lua](https://mozilla-services.github.io/lua_sandbox_extensions/socket/sandboxes/heka/output/heka_tcp_matcher.html) + +### update_checkpoint + +#### Batch Mode +Advances the output checkpoint when in batching mode. The standard use case is +to call it from `timer_event` after successfully flushing a batch on +timeout/shutdown. + +*Arguments* +* none + +#### Asynchronous Mode +Advances the output checkpoint and optionally reports the number of failures +that occured. + +*Arguments* +* sequence_id (lightuserdata) - sequence_id for the message that was just + successfully delivered/acknowledged +* failures (optional: integer) - number of failures that occured in the + asynchronus processing (added to the failure count) + +*Return* +* none (throws an error on invalid arg types) + +### Modes of Operation + +#### Lock Step + +* `process_message` operates on the message and returns one of the following + values: + * success (0) - the message was successfully processed and the output + checkpoint is advanced + * failure (-1) - the message was not successfully processed + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * the checkpoint is advanced + * skip (-2) - the message was intentionally not processed and the checkpoint + is advanced + * retry (-3) - the message was not successfully processed and the host will + call `process_message` again, with the same message, after a one second + delay + +#### Example Payload Output + +```lua +-- cfg +message_matcher = "Type == 'inject_payload'" +ticker_interval = 0 + +--location where the payload is written +output_dir = "/tmp" +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "io" +require "string" + +local output_dir = read_config("output_dir") or "/tmp" + +function process_message() + local pt = read_message("Fields[payload_type]") + if type(pt) ~= "string" then return -1, "invalid payload_type" end + + local pn = read_message("Fields[payload_name]") or "" + if type(pn) ~= "string" then return -1, "invalid payload_name" end + + local logger = read_message("Logger") or "" + + pn = string.gsub(pn, "%W", "_") + pt = string.gsub(pt, "%W", "_") + logger = string.gsub(logger, "%W", "_") + + local fn = string.format("%s/%s.%s.%s", output_dir, logger, pn, pt) + local fh, err = io.open(fn, "w") + if err then return -1, err end + + local payload = read_message("Payload") or "" + fh:write(payload) + fh:close() + return 0 +end + +function timer_event(ns) + -- no op +end +``` + +#### Batching + +* `process_message` batches the message/transformation in memory or on disk and + returns one of the following values: + * batching (-4) - the message was successfully added to the batch + * failure (-1) - the message cannot be batch + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * skip (-2) - the message was intentionally not added to the batch + * retry (-3) - the message was not successfully added to the batch and the + host will call `process_message` again, with the same message, after a one + second delay + * success (0) - the batch has been successfully committed and the output + checkpoint is advanced to the most recent message + +#### Example Postgres Output + +```lua +-- cfg +message_matcher = "Type == 'logfile'" +memory_limit = 0 +ticker_interval = 60 + +buffer_max = 1000 +db_config = { + host = "example.com", + port = 5432, + name = "dev", + user = "test", + _password = "testpw", +} +``` + +```lua +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "os" +require "string" +require "table" + +local driver = require "luasql.postgres" + +local ticker_interval = read_config("ticker_interval") +local db_config = read_config("db_config") or error("db_config must be set") +local buffer_max = read_config("buffer_max") or 1000 +assert(buffer_max > 0, "buffer_max must be greater than zero") + +local env = assert(driver.postgres()) +local con, err = env:connect(db_config.name, db_config.user, db_config._password, db_config.host, db_config.port) +assert(con, err) + +local buffer = {} +local buffer_len = 0 +local table_name = "test_table" +local MAX_LENGTH = 65535 +local columns = { +-- column name field name field type field length + {"msg_Timestamp" ,"Timestamp" ,"TIMESTAMP" ,nil}, + {"payload" ,"Payload" ,"VARCHAR" ,MAX_LENGTH}, + {"sourceName" ,"Fields[sourceName]" ,"VARCHAR" ,30}, + {"sourceVersion" ,"Fields[sourceVersion]" ,"VARCHAR" ,12}, + {"submissionDate" ,"Fields[submissionDate]" ,"DATE" ,nil}, + {"sampleId" ,"Fields[sampleId]" ,"SMALLINT" ,nil}, +} + +local function make_create_table() + local pieces = {"CREATE TABLE IF NOT EXISTS ", table_name, " ("} + for i, c in ipairs(columns) do + table.insert(pieces, c[1]) + table.insert(pieces, " ") + table.insert(pieces, c[3]) + if c[4] ~= nil then + table.insert(pieces, "(") + table.insert(pieces, c[4]) + table.insert(pieces, ")") + end + if c[4] == MAX_LENGTH then + table.insert(pieces, " ENCODE LZO") + end + if i < #columns then + table.insert(pieces, ", ") + end + end + table.insert(pieces, ")") + return table.concat(pieces) +end +assert(con:execute(make_create_table())) + +local function bulk_load() + local cnt, err = con:execute(table.concat(buffer)) + if not err then + buffer = {} + buffer_len = 0 + else + return err + end +end + +local function esc_str(v) + if v == nil then return "NULL" end + + if type(v) ~= "string" then v = tostring(v) end + if string.len(v) > MAX_LENGTH then + v = "TRUNCATED:" .. string.sub(v, 1, MAX_LENGTH - 10) + end + + local escd = con:escape(v) + if not escd then return "NULL" end + + return string.format("'%s'", escd) +end + +local function esc_num(v) + if v == nil then return "NULL" end + if type(v) ~= "number" then return esc_str(v) end + return tostring(v) +end + +local function esc_ts(v) + if v == nil then return "NULL" end + if type(v) ~= "number" then return esc_str(v) end + local seconds = v / 1e9 + return table.concat({"(TIMESTAMP 'epoch' + ", seconds, " * INTERVAL '1 seconds')"}) +end + +local function make_insert(sep) + local pieces = {sep, "("} + for i=1,#columns do + if i > 1 then + table.insert(pieces, ",") + end + local col = columns[i] + if col[3] == "TIMESTAMP" then + table.insert(pieces, esc_ts(read_message(col[2]))) + elseif col[3] == "SMALLINT" then + table.insert(pieces, esc_num(read_message(col[2]))) + else + table.insert(pieces, esc_str(read_message(col[2]))) + end + end + table.insert(pieces, ")") + return table.concat(pieces) +end + +local last_load = 0 +function process_message() + local sep = "," + if buffer_len == 0 then + buffer_len = buffer_len + 1 + buffer[buffer_len] = string.format("INSERT INTO %s VALUES", table_name) + sep = " " + end + buffer_len = buffer_len + 1 + buffer[buffer_len] = make_insert(sep) + + if buffer_len - 1 >= buffer_max then + local err = bulk_load() + if err then + buffer[buffer_len] = nil + buffer_len = buffer_len - 1 + return -3, err + else + last_load = os.time() + return 0 + end + end + return -4 +end + +function timer_event(ns, shutdown) + if buffer_len > 1 and (shutdown or last_load + ticker_interval <= ns / 1e9)then + local err = bulk_load() + if not err then + update_checkpoint() + end + end +end +``` + +#### Asynchronous + +* `async_buffer_size` Recommendation: this configuration variable should be set +and consumed by the host +* `process_message` is called with a sequence_id parameter and asynchronously +sends the message/transformation to the destination and returns one of the +following values: + * asynchronous (-5) - the message was successfully queued + * failure (-1) - the message cannot be queue + * the failure count is incremented + * any optional error message is written to the log + * the message is skipped + * skip (-2) - the message was intentionally not queued + * retry (-3) - the message was not successfully queued and the host will call + `process_message` again, with the same message, after a one second delay +* When an asynchronously sent message is acknowledged +[update_checkpoint](#updatecheckpoint) **MUST** be called to advance the +checkpoint to that specific message + +#### Example Kafka Output +[kafka.lua](https://github.com/mozilla-services/lua_sandbox_extensions/blob/master/kafka/sandboxes/heka/output/kafka.lua) + diff --git a/docs/hyperloglog.md b/docs/hyperloglog.md deleted file mode 100644 index 184cf50..0000000 --- a/docs/hyperloglog.md +++ /dev/null @@ -1,49 +0,0 @@ -Lua HyperLogLog Library -======================= - -The library is implemented in the _hyperloglog_ table. - -Constructor ------------ -**hyperloglog.new** () - -*Arguments* - -none - -*Return* - -A HyperLogLog object. - -Methods -------- -bool **add** (key) - -*Arguments* -- key (string/number) The key to add to the HyperLogLog. - -*Return* - -True if add altered the estimate, false if it has not. - -____ -double **count** () - -*Arguments* - -none - -*Return* - -The approximated cardinality of the set. - -____ -void **clear** () - -*Arguments* - -none - -*Return* - -none diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..ef1bc66 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,27 @@ +# Overview + +Sandboxes provide a dynamic and isolated execution environment +for data parsing, transformation, and analysis. They allow access to data +without jeopardizing the integrity or performance of the processing +infrastructure. This broadens the audience that the data can be +exposed to and facilitates new uses of the data (i.e. debugging, monitoring, +dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, +etc.) + +The Lua sandbox is a library allowing customized control over the Lua execution +environment including functionality like global data preservation/restoration on +shutdown/startup, output collection in textual or binary formats and an array of +parsers for various data types (Nginx, Apache, Syslog, MySQL and many RFC grammars) + +These libraries and utilities have been mostly extracted from [Hindsight](https://github.com/mozilla-services/hindsight). +The goal was to decouple the Heka/Hindsight functionality from any particular +infrastructure and make it embeddable into any tool or language. + +## Features + +- small - memory requirements are as little as 8 KiB for a basic sandbox +- fast - microsecond execution times +- stateful - ability to resume where it left off after a restart/reboot +- isolated - failures are contained and malfunctioning sandboxes are terminated. + Containment is defined in terms of restriction to the operating system, + file system, libraries, memory use, Lua instruction use, and output size. diff --git a/docs/sandbox.md b/docs/sandbox.md new file mode 100644 index 0000000..d32839f --- /dev/null +++ b/docs/sandbox.md @@ -0,0 +1,160 @@ +# Generic Sandbox Interface + +## Configuration + +* **input_limit** - the largest input string that is allowed to be processed + (bytes (unsigned), default 65536, 0 for unlimited*). This is not directly used + by the sandbox it is made availble to modules to standardize the configuration. +* **output_limit** - the largest output string an input or analysis plugin can + inject into the host (bytes (unsigned), default 65536, 0 for unlimited*) +* **memory_limit** - the maximum amount of memory a plugin can use before being + terminated (bytes (unsigned), default 8388608, 0 for unlimited*) +* **instruction_limit** - the maximum number of Lua instructions a plugin can + execute in a single API function call (count (unsigned), default 1000000, 0 + for unlimited) +* **path** - The path used by require to search for a Lua loader. See + [package loaders](http://www.lua.org/manual/5.1/manual.html#pdf-package.loaders) + for the path syntax. By default no paths are set in the sandbox and + everything has been moved to a sandbox configuration table. +* **cpath** - The path used by require to search for a C loader (same notes as + above) +* **disabled_modules** - Hash specifying which modules should be completely + inaccessible. The existence of the key in the table will disable the module. +```lua +disabled_modules = {io = 1} +``` +* **remove_entries** - Hash specifying which functions within a module should be + inaccessible +```lua +remove_entries = { + os = {"getenv", "execute"}, + string = {"dump"} +} +``` +* **log_level** - syslog severity level, when set to debug (7) the print + function will be wired to the specified logger (default error (3)) +* *user defined* any other variable (string, bool, number, table) is passed + through as-is and available via [read_config](#readconfig) + +*_0 == SIZE_MAX which in not necessarily the upper limit of the +configuration range UINT_MAX_ + +## Lua functions exposed to C by the core sandbox + +There are no functions exposed by default, see lsb_pcall_setup() and +lsb_pcall_teardown() when defining an API. + +## Functions exposed to Lua by the core sandbox + +### require + +By default only the base library is loaded, additional libraries must be loaded +with require(). + +*Arguments* + +- libraryName (string) + +*Return* +- a table - For non user provided libraries the table is also globally + registered with the library name. User provided libraries may implement their + own semantics i.e. the grammar libraries return a table but do not globally + register the table name (see the individual module documentation for the + correct usage). + +*Notes* + +The following modules have been modified, as described, for use in the sandbox. + - [base library](http://www.lua.org/manual/5.1/manual.html#5.1) + - The require() function has been modified to not expose any of the package + table to the sandbox. + - [math](http://www.lua.org/manual/5.1/manual.html#5.6) + - Added Functions + - erf(x) - Returns the error function value for x. + - erfc(x)- Returns the complementary error function value for x. + - [os](http://www.lua.org/manual/5.1/manual.html#5.8) + - The local timezone is set to UTC in all sandboxes. + +### read_config + +Provides access to the sandbox configuration variables. + +*Arguments* +* key (string) - configuration key name + +*Return* +* value (string, number, bool, table) + +### output +Receives any number of arguments and appends data to the output buffer, which +cannot exceed the output_limit configuration parameter. See lsb_get_output() to +connect the output to the host application. + +*Arguments* +- arg (number, string, bool, nil, userdata implementing output support) - Lua + variable or literal to be appended the output buffer + +*Return* +- none + +### print +Receives any number of arguments and sends a debug message to the host's logger +function as a tab delimited message. The function clears and then uses the +output buffer so if pending output has been queued and not flushed it will be +lost (the same output_limit restrictions apply). Non-printable characters +are replaced with spaces to preserve the host's log integrity and any embedded +NULs terminate each argument. + +*Arguments* +- arg (anything that can be converted to a string with the tostring function) + +*Return* +- none + +**Note:** To extend the function set exposed to Lua see lsb_add_function() + +## Special Lua Global Variables + +- **_PRESERVATION_VERSION** (number) This variable is examined during state +restoration; if the previous version does not match the current version the +restoration is aborted and the sandbox starts cleanly. The version should be +incremented any time an incompatible change is made to the global data schema. + +When versioning is required, the use of a configuration variable name +`preservation_version` with the following syntax is recommended. +```lua +-- initial (allows the user to bump the version due to a cfg change) +_PRESERVATION_VERSION = read_config("preservation_version") or 0 + +-- if the plugin code alters the global data schema in an incompatible way this +-- ensures the state is properly cleared when there is a user provided version. +_PRESERVATION_VERSION = (read_config("preservation_version") or 0) + 1 +``` + +## How to interact with the sandbox (creating an API) + +The best place to start is with some examples: + +### Unit Test API + +[Unit Test API](https://mozilla-services.github.io/lua_sandbox/doxygen/test_2sandbox_8h.html) + +#### Lua Functions Exposed to C + +- int **process** (double) + - exposes a process function that takes a test case number as its argument + and returns and integer result code. +- void **report** (double) + - exposes a report function that takes a test case number as its argument + and returns nothing. + +#### C Functions Exposed to Lua + +- void **write_output** (optionalArg1, optionalArg2, optionalArgN) + - captures whatever is in the output buffer for use by the host application, + appending any optional arguments (optional arguments have the same + restriction as output). + +### Heka Sandbox API + +[Heka Sandbox API](https://mozilla-services.github.io/lua_sandbox/doxygen/heka_2sandbox_8h.html) diff --git a/docs/sandbox_api.md b/docs/sandbox_api.md deleted file mode 100644 index 597cf43..0000000 --- a/docs/sandbox_api.md +++ /dev/null @@ -1,105 +0,0 @@ -Sandbox API -=========== - -Sandboxes provide a dynamic and isolated execution environment -for data parsing, transformation, and analysis. They allow access to data -without jeopardizing the integrity or performance of the processing -infrastructure. This broadens the audience that the data can be -exposed to and facilitates new uses of the data (i.e. debugging, monitoring, -dynamic provisioning, SLA analysis, intrusion detection, ad-hoc reporting, -etc.) - -Features -======== -- small - memory requirements are as little as 6 KiB for a basic sandbox -- fast - microsecond execution times -- stateful - ability to resume where it left off after a restart/reboot -- isolated - failures are contained and malfunctioning sandboxes are terminated. - Containment is defined in terms of restriction to the operating system, - file system, libraries, memory use, Lua instruction use, and output size. - -Functions exposed to Lua by the core sandbox -============================================ -**require(libraryName)** - -By default only the base library is loaded additional libraries must be explicitly specified. - -*Arguments* - -- libraryName (string) - - **base library** - - The standard require() function is overridden by this version. - - Disabled functions: collectgarbage, coroutine, dofile, load, loadfile, loadstring, module, print. - - [bloom_filter](bloom_filter.md) - - [circular_buffer](circular_buffer.md) - - **cjson** http://www.kyne.com.au/~mark/software/lua-cjson-manual.html. With the following modifications: - - Loads the cjson module in a global cjson table - - The encode buffer is limited to the sandbox output_limit. - - The decode buffer will be roughly limited to one half of the sandbox memory_limit. - - The NULL value is not decoded to cjson.null it is simply discarded. - If the original behavior is desired use cjson.decode_null(true) to enable NULL decoding. - - The new() function has been disabled so only a single cjson parser can be created. - - The encode_keep_buffer() function has been disabled (the buffer is always reused). - - [hyperloglog](hyperloglog.md) - - **lpeg** loads the Lua Parsing Expression Grammar Library http://www.inf.puc-rio.br/~roberto/lpeg/lpeg.html - - **math** - - **os** - - The local timezone is set to UTC in all sandboxes. - - Disabled functions: execute, exit, remove, rename, setlocale, tmpname. - - **string** - - **struct** http://www.inf.puc-rio.br/~roberto/struct/ - - **table** - - _user provided_ - -*Return* -- a table - For non user provided libraries the table is also globally registered - with the library name. User provided libraries may implement there own semantics - i.e. the grammar libraries return a table but do not globally register the table name - (see the individual module documentation for the correct usage). - -____ -**output(arg0, arg1, ...argN)** - Appends data to the output buffer, which cannot exceed the output_limit - configuration parameter. See lsb_get_output() to connect the output to the - host application. - -*Arguments* -- arg (number, string, bool, nil, circular_buffer) Lua variable or literal to be appended the output buffer - -*Return* -- none - -**Note:** To extend the function set exposed to Lua see lsb_add_function() - - -Lua functions exposed to C by the core sandbox -============================================== -There are no functions exposed by default, see lsb_pcall_setup() and -lsb_pcall_teardown() when defining an API. - -How to interact with the sandbox (creating an API) -================================================== -The best place to start is with some examples: - -Unit Test API -============= -Lua Functions Exposed to C --------------------------- -- int **process** (double) - - exposes a process function that takes a test case number as its argument and returns and integer result code. -- void **report** (double) - - exposes a report function that takes a test case number as its argument and returns nothing. - -C Functions Exposed to Lua --------------------------- -- void **write_output** (optionalArg1, optionalArg2, optionalArgN) - - captures whatever is in the output buffer for use by the host application, appending any optional arguments - (optional arguments have the same restriction as output). -- void **write_message** (table) - - serializes the Lua table into a Heka protobuf message and captures the output. - -[Unit Test Source Code](https://github.com/mozilla-services/lua_sandbox/blob/master/src/test/test_lua_sandbox.c) - -Heka Sandbox API -================ -[Heka Sandbox](https://hekad.readthedocs.org/en/latest/sandbox/index.html#lua-sandbox) diff --git a/docs/util/message_matcher.md b/docs/util/message_matcher.md new file mode 100644 index 0000000..361c4a2 --- /dev/null +++ b/docs/util/message_matcher.md @@ -0,0 +1,91 @@ +# Message Matcher Syntax + +The message matcher allows sandboxes to select which messages they want to +consume (see [Heka Message Structure](/heka/message.md)) + +## Examples + +* Type == "test" && Severity == 6 +* (Severity == 7 || Payload == "Test Payload") && Type == "test" +* Fields[foo] != "bar" +* Fields[foo][1][0] == "alternate" +* Fields[MyBool] == TRUE +* TRUE +* Fields[created] =~ "^2015" +* Fields[string] =~ "foo.example.com"% -- literal pattern vs "foo%.example%.com" +* Fields[widget] != NIL +* Timestamp >= "2016-05-24T00:00:00Z" +* Timestamp >= 1464048000000000000 + +## Relational Operators + +* == equals +* != not equals +* > greater than +* >= greater than equals +* < less than +* <= less than equals +* =~ Lua pattern match +* !~ Lua negated pattern match + +## Logical Operators + +* Parentheses are used for grouping expressions +* && and (higher precedence) +* || or + +## Boolean + +* TRUE +* FALSE + +## Constants + +* NIL used to test the existence (!=) or non-existence (==) of optional headers or field variables + +## Message Variables + +* All message variables must be on the left hand side of the relational +comparison + +### String + +* Uuid - 16 byte raw binary type 4 UUID (useful for partitioning data) +* Type +* Logger +* Payload +* EnvVersion +* Hostname + +### Numeric + +* Timestamp - in addition to nanoseconds since the UNIX epoch an RFC3339 string is also accepted e.g., "2016-05-24T21:51:00Z" +* Severity +* Pid + +### Fields + +* Fields[_field_name_] - shorthand for Field[_field_name_][0][0] +* Fields[_field_name_][_field_index_] - shorthand for Field[_field_name_][_field_index_][0] +* Fields[_field_name_][_field_index_][_array_index_] the indices are restricted to 0-255 +* If a field type is mis-match for the relational comparison, false will be returned e.g., Fields[foo] == 6 where "foo" is a string + +## Quoted String + +* Single or double quoted strings are allowed +* The maximum string length is 255 bytes + +## Lua Pattern Matching Expression + +* Patterns are quoted string values + * An 'escape' pattern modifier of `%` is allowed e.g. `"foo.bar"%` is + treated as a literal instead of a pattern and behaves like the 'plain' + option on string.find(). If there are no pattern match characters in the + string this modifier is set automatically. +* See [Lua Patterns](http://www.lua.org/manual/5.1/manual.html#5.4.1) +* Capture groups are ignored + +## Additional Restrictions + +* Message matchers are restricted to 128 relational comparisons +* A NUL character '\0' is not allowed in a matcher string diff --git a/gen_gh_pages.lua b/gen_gh_pages.lua new file mode 100644 index 0000000..aff0b99 --- /dev/null +++ b/gen_gh_pages.lua @@ -0,0 +1,47 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "os" +require "io" +require "string" + + +local function output_menu(output_dir, version) + local fh = assert(io.open(string.format("%s/SUMMARY.md", output_dir), "w")) + fh:write(string.format("* [Lua Sandbox Library (%s)](README.md)\n\n", version)) + fh:write([[ +* [Generic Sandbox](sandbox.md) +* [Heka Sandbox](heka/index.md) + * [Input Interface](heka/input.md) + * [Analysis Interface](heka/analysis.md) + * [Output Interface](heka/output.md) + * [Message Schema](heka/message.md) + * [Message Matcher](util/message_matcher.md) +* [Command Line Tools](cli/index.md) +* [Source Documentation](https://mozilla-services.github.io/lua_sandbox/doxygen/index.html) +* [Sandbox Extensions](https://mozilla-services.github.io/lua_sandbox_extensions) +]]) + fh:close() +end + + +local args = {...} +local function main() + local output_dir = string.format("%s/gb-source", arg[3]) + local rv = os.execute(string.format("rsync -rav docs/ %s/", output_dir)) + if rv ~= 0 then error"rsync setup" end + + local fh = assert(io.open(string.format("%s/book.json", output_dir), "w")) + fh:write([[{"plugins" : ["collapsible-menu", "navigator"]}]]) + fh:close() + + os.execute(string.format("cd %s;gitbook install", output_dir)) + os.execute(string.format("mv %s/index.md %s/README.md", output_dir, output_dir)) + output_menu(output_dir, args[1]) + os.execute(string.format("gitbook build %s", output_dir)) + local rv = os.execute(string.format("rsync -rav %s/_book/ %s/", output_dir, "gh-pages/")) + if rv ~= 0 then error"rsync publish" end +end + +main() diff --git a/include/lua_sandbox.h b/include/lua_sandbox.h deleted file mode 100644 index 28b2a2a..0000000 --- a/include/lua_sandbox.h +++ /dev/null @@ -1,229 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// Generic Lua sandbox for dynamic data analysis @file -#ifndef lua_sandbox_h_ -#define lua_sandbox_h_ - -#include - -typedef enum { - LSB_UNKNOWN = 0, - LSB_RUNNING = 1, - LSB_TERMINATED = 2 -} lsb_state; - -typedef enum { - LSB_US_LIMIT = 0, - LSB_US_CURRENT = 1, - LSB_US_MAXIMUM = 2, - - LSB_US_MAX -} lsb_usage_stat; - -typedef enum { - LSB_UT_MEMORY = 0, - LSB_UT_INSTRUCTION = 1, - LSB_UT_OUTPUT = 2, - - LSB_UT_MAX -} lsb_usage_type; - -#define LSB_ERROR_SIZE 256 - -#ifdef _WIN32 -#if defined(luasandbox_EXPORTS) -#define LSB_EXPORT __declspec(dllexport) -#else -#define LSB_EXPORT __declspec(dllimport) -#endif -#else -#define LSB_EXPORT -#endif - -typedef struct lua_sandbox lua_sandbox; - -/** - * Allocates and initializes the structure around the Lua sandbox. - * - * @param parent Pointer to associate the owner to this sandbox. - * @param lua_file Filename of the Lua script to run in this sandbox. - * @param require_path Location of the common sandbox modules - * @param memory_limit Sets the sandbox memory limit (bytes). - * @param instruction_limit Sets the sandbox Lua instruction limit (count). - * This limit is per call to process_message or timer_event - * @param output_limit Sets the single message payload limit (bytes). This - * limit applies to the in memory output buffer. The buffer is reset back - * to zero when inject_message is called. - * @return lua_sandbox Sandbox pointer or NULL on failure. - */ -LSB_EXPORT lua_sandbox* lsb_create(void* parent, - const char* lua_file, - const char* require_path, - unsigned memory_limit, - unsigned instruction_limit, - unsigned output_limit); - -/** - * Initializes the Lua sandbox and loads/runs the Lua script that was specified - * in lua_create_sandbox. - * - * @param lsb Pointer to the sandbox. - * @param state_file Filename where the global data is read. Use a NULL or empty - * string for no data restoration. The global - * _PRESERVATION_VERSION variable will be examined during - * restoration; if the previous version does not match the - * current version the restoration will be aborted and the - * sandbox will start cleanly. _PRESERVATION_VERSION should be - * incremented any time an incompatible change is made to the - * global data schema. If no version is set the check will - * always succeed and a version of zero is assigned. - * - * @return int Zero on success, non-zero on failure. - */ -LSB_EXPORT int lsb_init(lua_sandbox* lsb, const char* state_file); - -/** - * Frees the memory associated with the sandbox. - * - * @param lsb Sandbox pointer to discard. - * @param state_file Filename where the sandbox global data is saved. Use a - * NULL or empty string for no preservation. - * - * @return NULL on success, pointer to an error message on failure that MUST BE - * FREED by the caller. - */ -LSB_EXPORT char* lsb_destroy(lua_sandbox* lsb, const char* state_file); - -/** - * Retrieve the sandbox usage statistics. - * - * @param lsb Pointer to the sandbox. - * @param lsb_usage_type Type of statistic to retrieve i.e. memory. - * @param lsb_usage_stat Type of statistic to retrieve i.e. current. - * - * @return unsigned Count or number of bytes depending on the statistic. - */ -LSB_EXPORT unsigned -lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, lsb_usage_stat ustat); -/** - * Retrieve the current sandbox status. - * - * @param lsb Pointer to the sandbox. - * - * @return lsb_state code - */ -LSB_EXPORT lsb_state lsb_get_state(lua_sandbox* lsb); - -/** - * Return the last error in human readable form. - * - * @param lsb Pointer to the sandbox. - * - * @return const char* error message - */ -LSB_EXPORT const char* lsb_get_error(lua_sandbox* lsb); - -/** - * Sets the last error string. - * - * @param lsb Pointer to the sandbox. - * @param err Error message. - * - * @return const char* error message - */ -LSB_EXPORT void lsb_set_error(lua_sandbox* lsb, const char* err); - -/** - * Access the Lua pointer. - * - * @param lsb Pointer to the sandbox. - * - * @return lua_State* The lua_State pointer. - */ -LSB_EXPORT lua_State* lsb_get_lua(lua_sandbox* lsb); - -/** - * Access the parent pointer stored in the sandbox. - * - * @param lsb Pointer to the sandbox. - * - * @return void* The parent pointer passed to init. - */ -LSB_EXPORT void* lsb_get_parent(lua_sandbox* lsb); - -/** - * Create a CFunction for use by the Sandbox. The Lua sandbox pointer is pushed - * to upvalue index 1. - * - * @param lsb Pointer to the sandbox. - * @param func Lua CFunction pointer. - * @param func_name Function name exposed to the Lua sandbox. - */ -LSB_EXPORT void lsb_add_function(lua_sandbox* lsb, lua_CFunction func, - const char* func_name); - -/** - * Retrieve the data in the output buffer and reset the buffer. The returned - * output string will remain valid until additional sandbox output is performed. - * The output should be copied if the application needs to hold onto it. - * - * @param lsb Pointer to the sandbox. - * @param len If len is not NULL, it will be set to the length of the string. - * - * @return const char* Pointer to the output buffer. - */ -LSB_EXPORT const char* lsb_get_output(lua_sandbox* lsb, size_t* len); - -/** - * Write an array of variables on the Lua stack to the output buffer. - * - * @param lsb Pointer to the sandbox. - * @param start Lua stack index of first variable. - * @param end Lua stack index of the last variable. - * @param append 0 to overwrite the output buffer, 1 to append the output to it - * - * - */ -LSB_EXPORT void lsb_output(lua_sandbox* lsb, int start, int end, int append); - -/** - * Write a Lua table (in a Heka protobuf structure) to the output buffer. - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index of the table. - * @param append 0 to overwrite the output buffer, 1 to append the output to it - * - * @return int 0 on success - */ -LSB_EXPORT int lsb_output_protobuf(lua_sandbox* lsb, int index, int append); - -/** - * Helper function to load the Lua function and set the instruction limits - * - * @param lsb Pointer to the sandbox. - * @param func_name Name of the function to load - * - * @return int 0 on success - */ -LSB_EXPORT int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name); - -/** - * Helper function to update the statistics after the call - * - * @param lsb Pointer to the sandbox. - */ -LSB_EXPORT void lsb_pcall_teardown(lua_sandbox* lsb); - -/** - * Shutdown the sandbox due to a fatal error. - * - * @param lsb Pointer to the sandbox. - * @param err Reason for termination - */ -LSB_EXPORT void lsb_terminate(lua_sandbox* lsb, const char* err); - -#endif diff --git a/include/luasandbox.h b/include/luasandbox.h new file mode 100644 index 0000000..32f8744 --- /dev/null +++ b/include/luasandbox.h @@ -0,0 +1,281 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Generic Lua sandbox for dynamic data analysis @file */ + +#ifndef luasandbox_h_ +#define luasandbox_h_ + +#include "luasandbox/error.h" + +#ifdef _WIN32 +#ifdef luasandbox_EXPORTS +#define LSB_EXPORT __declspec(dllexport) +#else +#define LSB_EXPORT __declspec(dllimport) +#endif +#else +#if __GNUC__ >= 4 +#define LSB_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_EXPORT +#endif +#endif + +#define LSB_ERROR_SIZE 256 + +#define LSB_SHUTTING_DOWN "shutting down" +#define LSB_CONFIG_TABLE "lsb_config" +#define LSB_THIS_PTR "lsb_this_ptr" +#define LSB_MEMORY_LIMIT "memory_limit" +#define LSB_INSTRUCTION_LIMIT "instruction_limit" +#define LSB_INPUT_LIMIT "input_limit" +#define LSB_OUTPUT_LIMIT "output_limit" +#define LSB_LOG_LEVEL "log_level" +#define LSB_LUA_PATH "path" +#define LSB_LUA_CPATH "cpath" +#define LSB_NIL_ERROR "" + +typedef enum { + LSB_UNKNOWN = 0, + LSB_RUNNING = 1, + LSB_TERMINATED = 2, + LSB_STOP = 3 +} lsb_state; + +typedef enum { + LSB_US_LIMIT = 0, + LSB_US_CURRENT = 1, + LSB_US_MAXIMUM = 2, + + LSB_US_MAX +} lsb_usage_stat; + +typedef enum { + LSB_UT_MEMORY = 0, + LSB_UT_INSTRUCTION = 1, + LSB_UT_OUTPUT = 2, + + LSB_UT_MAX +} lsb_usage_type; + +typedef struct lsb_lua_sandbox lsb_lua_sandbox; + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "luasandbox/lua.h" + +LSB_EXPORT extern lsb_err_id LSB_ERR_INIT; +LSB_EXPORT extern lsb_err_id LSB_ERR_LUA; +LSB_EXPORT extern lsb_err_id LSB_ERR_TERMINATED; + +/** + * Allocates and initializes the structure around the Lua sandbox allowing + * full specification of the sandbox configuration using a Lua configuration + * string. + * memory_limit = 1024*1024*1 + * instruction_limit = 10000 + * output_limit = 64*1024 + * path = '/modules/?.lua' + * cpath = '/modules/?.so' + * remove_entries = { + * [''] = + * {'collectgarbage','coroutine','dofile','load','loadfile','loadstring', + * 'newproxy','print'}, + * os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'} + * } + * disable_modules = {io = 1} + * + * + * @param parent Pointer to associate the owner to this sandbox. + * @param lua_file Filename of the Lua script to run in this sandbox. + * @param cfg Lua structure defining the full sandbox restrictions (may contain + * optional host configuration options, everything is available to + * the sandbox through the read_config API. + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @return lsb_lua_sandbox Sandbox pointer or NULL on failure. + */ +LSB_EXPORT lsb_lua_sandbox* +lsb_create(void *parent, const char *lua_file, const char *cfg, + lsb_logger *logger); + +/** + * Initializes the Lua sandbox and loads/runs the Lua script that was specified + * in lua_create_sandbox. + * + * @param lsb Pointer to the sandbox. + * @param state_file Filename where the global data is read. Use a NULL or empty + * string for no data restoration. The global + * _PRESERVATION_VERSION variable will be examined during + * restoration; if the previous version does not match the + * current version the restoration will be aborted and the + * sandbox will start cleanly. _PRESERVATION_VERSION should be + * incremented any time an incompatible change is made to the + * global data schema. If no version is set the check will + * always succeed and a version of zero is assigned. + * + * @return lsb_err_value NULL on success error message on failure + * + */ +LSB_EXPORT lsb_err_value +lsb_init(lsb_lua_sandbox *lsb, const char *state_file); + +/** + * Changes the sandbox state to LSB_STOP to allow for a clean exit. This call is + * not thread safe. + * + * @param lsb sandbox to clean stop + * + * @return + * + */ +LSB_EXPORT void lsb_stop_sandbox_clean(lsb_lua_sandbox *lsb); + +/** + * Aborts the running sandbox from a different thread of execution. A "shutting + * down" Lua error message is generated. + * + * @param lsb sandbox to abort + * + * @return + * + */ +LSB_EXPORT void lsb_stop_sandbox(lsb_lua_sandbox *lsb); + +/** + * Frees the memory associated with the sandbox. + * + * @param lsb Sandbox pointer to discard. + * + * @return NULL on success, pointer to an error message on failure that MUST BE + * FREED by the caller. + */ +LSB_EXPORT char* lsb_destroy(lsb_lua_sandbox *lsb); + +/** + * Retrieve the sandbox usage statistics. + * + * @param lsb Pointer to the sandbox. + * @param utype Type of statistic to retrieve i.e. memory. + * @param ustat Type of statistic to retrieve i.e. current. + * + * @return size_t Count or number of bytes depending on the statistic. + */ +LSB_EXPORT size_t lsb_usage(lsb_lua_sandbox *lsb, + lsb_usage_type utype, + lsb_usage_stat ustat); +/** + * Retrieve the current sandbox status. + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_state code + */ +LSB_EXPORT lsb_state lsb_get_state(lsb_lua_sandbox *lsb); + +/** + * Return the last error in human readable form. + * + * @param lsb Pointer to the sandbox. + * + * @return const char* error message + */ +LSB_EXPORT const char* lsb_get_error(lsb_lua_sandbox *lsb); + +/** + * Sets the last error string. + * + * @param lsb Pointer to the sandbox. + * @param err Error message. + * + * @return const char* error message + */ +LSB_EXPORT void lsb_set_error(lsb_lua_sandbox *lsb, const char *err); + +/** + * Access the Lua pointer. + * + * @param lsb Pointer to the sandbox. + * + * @return lua_State* The lua_State pointer. + */ +LSB_EXPORT lua_State* lsb_get_lua(lsb_lua_sandbox *lsb); + +/** + * Returns the filename of the Lua source. + * + * @param lsb Pointer to the sandbox. + * + * @return const char* filename. + */ +LSB_EXPORT const char* lsb_get_lua_file(lsb_lua_sandbox *lsb); + +/** + * Access the parent pointer stored in the sandbox. + * + * @param lsb Pointer to the sandbox. + * + * @return void* The parent pointer passed to init. + */ +LSB_EXPORT void* lsb_get_parent(lsb_lua_sandbox *lsb); + +/** + * Access the logger struct stored in the sandbox. The logger callback is only + * available to modules in debug mode (same as print). + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_logger Pointer to the logger struct + * + */ +LSB_EXPORT const lsb_logger* lsb_get_logger(lsb_lua_sandbox *lsb); + +/** + * Create a CFunction for use by the Sandbox. The Lua sandbox pointer is pushed + * to upvalue index 1. + * + * @param lsb Pointer to the sandbox. + * @param func Lua CFunction pointer. + * @param func_name Function name exposed to the Lua sandbox. + */ +LSB_EXPORT void lsb_add_function(lsb_lua_sandbox *lsb, + lua_CFunction func, + const char *func_name); + +/** + * Helper function to load the Lua function and set the instruction limits + * + * @param lsb Pointer to the sandbox. + * @param func_name Name of the function to load + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_EXPORT lsb_err_value +lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name); + +/** + * Helper function to update the statistics after the call + * + * @param lsb Pointer to the sandbox. + */ +LSB_EXPORT void lsb_pcall_teardown(lsb_lua_sandbox *lsb); + +/** + * Change the sandbox state to LSB_TERMINATED due to a fatal error. + * + * @param lsb Pointer to the sandbox. + * @param err Reason for termination + */ +LSB_EXPORT void lsb_terminate(lsb_lua_sandbox *lsb, const char *err); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/error.h b/include/luasandbox/error.h new file mode 100644 index 0000000..f1d0d61 --- /dev/null +++ b/include/luasandbox/error.h @@ -0,0 +1,29 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Error handling and logging @file */ + +#ifndef luasandbox_error_h_ +#define luasandbox_error_h_ + +// See Identify your Errors better with char[] +// http://accu.org/index.php/journals/2184 +typedef const char lsb_err_id[]; +typedef const char *lsb_err_value; +#define lsb_err_string(s) s ? s : "" + +typedef void (*lsb_logger_cb)(void *context, + const char *component, + int level, + const char *fmt, + ...); + +typedef struct lsb_logger { + void *context; + lsb_logger_cb cb; +} lsb_logger; + +#endif diff --git a/include/luasandbox/heka/sandbox.h b/include/luasandbox/heka/sandbox.h new file mode 100644 index 0000000..b524f52 --- /dev/null +++ b/include/luasandbox/heka/sandbox.h @@ -0,0 +1,409 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Heka sandbox implementation @file */ + +#ifndef luasandbox_heka_sandbox_h_ +#define luasandbox_heka_sandbox_h_ + +#include +#include + +#include "../../luasandbox.h" +#include "../error.h" +#include "../util/heka_message.h" + +#ifdef _WIN32 +#ifdef luasandboxheka_EXPORTS +#define LSB_HEKA_EXPORT __declspec(dllexport) +#else +#define LSB_HEKA_EXPORT __declspec(dllimport) +#endif +#else +#if __GNUC__ >= 4 +#define LSB_HEKA_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_HEKA_EXPORT +#endif +#endif + +#define LSB_HEKA_MAX_MESSAGE_SIZE "max_message_size" +#define LSB_HEKA_UPDATE_CHECKPOINT "update_checkpoint" +#define LSB_HEKA_THIS_PTR "lsb_heka_this_ptr" + +enum lsb_heka_pm_rv { + LSB_HEKA_PM_SENT = 0, + LSB_HEKA_PM_FAIL = -1, + LSB_HEKA_PM_SKIP = -2, + LSB_HEKA_PM_RETRY = -3, + LSB_HEKA_PM_BATCH = -4, + LSB_HEKA_PM_ASYNC = -5 +}; + +enum lsb_heka_im_rv { + LSB_HEKA_IM_SUCCESS = 0, + LSB_HEKA_IM_ERROR = 1, // generic error for backward compatibility + LSB_HEKA_IM_CHECKPOINT = 2, + LSB_HEKA_IM_LIMIT = 3, +}; + +typedef struct lsb_heka_sandbox lsb_heka_sandbox; + +typedef struct lsb_heka_stats { + unsigned long long mem_cur; + unsigned long long mem_max; + unsigned long long ins_max; + unsigned long long out_max; + unsigned long long im_cnt; + unsigned long long im_bytes; + unsigned long long pm_cnt; + unsigned long long pm_failures; + double pm_avg; + double pm_sd; + double te_avg; + double te_sd; +} lsb_heka_stats; + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "../lauxlib.h" +#include "../lua.h" + +LSB_HEKA_EXPORT extern lsb_err_id LSB_ERR_HEKA_INPUT; + +/** + * inject_message callback function provided by the host. Only one (or neither) + * of the checkpoint values will be set in a call. Numeric checkpoints can have + * a measurable performance benefit but are not always applicable so both + * options are provided to support various types of input plugins. This function + * can be called with a NULL pb pointer by the 'is_running' API or if only a + * checkpoint update is being performed; it should be treated as a no-op and + * used as a synchronization point to collect statistics and communicate + * shutdown status. + * + * @param parent Opaque pointer the host object owning this sandbox + * @param pb Pointer to a Heka protobuf encoded message being injected. + * @param pb_en Length of s + * @param cp_numeric Numeric based checkpoint (can be NAN) + * @param cp_string String based checkpoint (can be NULL) + * + * @return 0 on success + */ +typedef int (*lsb_heka_im_input)(void *parent, + const char *pb, + size_t pb_len, + double cp_numeric, + const char *cp_string); + +/** + * inject_message callback function provided by the host. + * + * @param parent Opaque pointer the host object owning this sandbox. + * @param pb Pointer to a Heka protobuf encoded message being injected. + * @param len Length of pb_len + * + * @return 0 on success + */ +typedef int (*lsb_heka_im_analysis)(void *parent, + const char *pb, + size_t pb_len); + +/** + * update_checkpoint callback function provided by the host. + * + * @param parent Opaque pointer the host object owning this sandbox. + * @param sequence_id Opaque pointer to the host message sequencing (passed into + * process_message). + * + * @return 0 on success + */ +typedef int (*lsb_heka_update_checkpoint)(void *parent, void *sequence_id); + +/** + * Create a sandbox supporting the Heka Input Plugin API + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified path to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param im inject_message callback + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_HEKA_EXPORT +lsb_heka_sandbox* lsb_heka_create_input(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_im_input im); + +/** + * Host access to the input sandbox process_message API. If a numeric + * checkpoint is set the string checkpoint is ignored. + * + * @param hsb Heka input sandbox + * @param cp_numeric NAN if no numeric checkpoint + * @param cp_string NULL if no string checkpoint + * @param profile Take a timing sample on this execution + * + * @return int + * >0 fatal error + * 0 success + * <0 non-fatal error (status message is logged) + * + */ +LSB_HEKA_EXPORT +int lsb_heka_pm_input(lsb_heka_sandbox *hsb, + double cp_numeric, + const char *cp_string, + bool profile); + +/** + * Create a sandbox supporting the Heka Analysis Plugin API + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified filename to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param im inject_message callback + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_HEKA_EXPORT +lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_im_analysis im); + +/** + * Host access to the analysis sandbox process_message API + * + * @param hsb Heka analysis sandbox + * @param msg Heka message to process + * @param profile Take a timing sample on this execution + * + * @return int + * >0 fatal error + * 0 success + * <0 non-fatal error (status message is logged) + * + */ +LSB_HEKA_EXPORT +int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + bool profile); + +/** + * Create a sandbox supporting the Heka Output Plugin API + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified path to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param ucp checkpoint_updated callback when using batch or async output + * + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_HEKA_EXPORT +lsb_heka_sandbox* lsb_heka_create_output(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_update_checkpoint ucp); + +/** + * Create a sandbox supporting the Heka Output Plugin API with + * inject_message support + * + * @param parent Opaque pointer the host object owning this sandbox + * @param lua_file Fully qualified path to the Lua source file + * @param state_file Fully qualified filename to the state preservation file + * (NULL if no preservation is required) + * @param lsb_cfg Full configuration string as a Lua table (NULL for lsb + * defaults) + * @param logger Struct for error reporting/debug printing (NULL to disable) + * @param ucp checkpoint_updated callback when using batch or async output + * @param im inject_message callback + * + * @return lsb_heka_sandbox* On success a pointer to the sandbox otherwise NULL + */ +LSB_HEKA_EXPORT +lsb_heka_sandbox* lsb_heka_create_output_im(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_update_checkpoint ucp, + lsb_heka_im_analysis im); +/** + * Host access to the output sandbox process_message API + * + * @param hsb Heka output sandbox + * @param msg Heka message to process + * @param sequence_id Opaque pointer to the message sequence id (only used for + * async output plugin otherwise it should be NULL) + * @param profile Take a timing sample on this execution + * + * @return int + * >0 fatal error + * 0 success + * -1 non-fatal_error (status message is logged) + * -2 skip + * -3 retry + * -4 batching + * -5 async output + * + */ +LSB_HEKA_EXPORT +int lsb_heka_pm_output(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + void *sequence_id, + bool profile); +/** + * Requests a long running input sandbox to stop. This call is not thread safe. + * + * @param hsb Heka sandbox to cleanly stop + * + * @return + * + */ +LSB_HEKA_EXPORT void +lsb_heka_stop_sandbox_clean(lsb_heka_sandbox *hsb); + +/** + * Aborts the running sandbox from a different thread of execution. A "shutting + * down" termination message is generated. Used to abort long runnning sandboxes + * such as an input sandbox. + * + * @param hsb Heka sandbox to forcibly stop + * + * @return + * + */ +LSB_HEKA_EXPORT void +lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb); + +/** + * Terminates the sandbox as if it had a fatal error (not thread safe). + * + * @param hsb Heka sandbox to terminate + * @param err Reason for termination + */ +LSB_HEKA_EXPORT void +lsb_heka_terminate_sandbox(lsb_heka_sandbox *hsb, const char *err); + +/** + * Frees all memory associated with the sandbox; hsb cannont be used after this + * point and the host should set it to NULL. + * + * @param hsb Heka sandbox to free + * + * @return NULL on success, pointer to an error message on failure that MUST BE + * FREED by the caller. + * + */ +LSB_HEKA_EXPORT char* +lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb); + +/** + * Host access to the timer_event API + * + * @param hsb Heka sandbox + * @param t Clock time of the timer_event execution + * @param shutdown Flag indicating the Host is shutting down allowing the + * sandbox to do any desired finialization) + * + * @return int 0 on success + */ +LSB_HEKA_EXPORT +int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown); + +/** + * Return the last error in human readable form. + * + * @param hsb Heka sandbox + * + * @return const char* error message + */ +LSB_HEKA_EXPORT const char* lsb_heka_get_error(lsb_heka_sandbox *hsb); + +/** + * Returns the filename of the Lua source. + * + * @param hsb Heka sandbox + * + * @return const char* filename. + */ +LSB_HEKA_EXPORT const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb); + +/** + * Retrieve the sandbox profiling/monitoring statistics. This call accesses + * internal data and is not thread safe. + * + * @param hsb Heka sandbox + * + * @return lsb_heka_stats A copy of the stats structure + */ +LSB_HEKA_EXPORT lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb); + +/** + * Convenience function to test if a sandbox is running. + * + * @param hsb Heka sandbox + * + * @return True if the sandbox has not been terminated + */ +LSB_HEKA_EXPORT bool lsb_heka_is_running(lsb_heka_sandbox *hsb); + +/** + * Queries the state of the sandbox. + * + * @param hsb + * + * @return lsb_state + * + */ +LSB_HEKA_EXPORT lsb_state lsb_heka_get_state(lsb_heka_sandbox *hsb); + +/** + * Retrieve the currently active sandbox message. This call returns a handle to + * internal data and is not thread safe. + * * + * @param hsb Heka sandbox + * + * @return const lsb_heka_message* NULL if there is no active message + */ +LSB_HEKA_EXPORT const lsb_heka_message* +lsb_heka_get_message(lsb_heka_sandbox *hsb); + +/** + * Retrieve the sandbox type. + * * + * @param hsb Heka sandbox + * + * @return char Heka sandbox type identifer + */ +LSB_HEKA_EXPORT char lsb_heka_get_type(lsb_heka_sandbox *hsb); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/heka/stream_reader.h b/include/luasandbox/heka/stream_reader.h new file mode 100644 index 0000000..18f83d0 --- /dev/null +++ b/include/luasandbox/heka/stream_reader.h @@ -0,0 +1,24 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight Heka stream reader structures @file */ + +#ifndef luasandbox_heka_stream_reader_h_ +#define luasandbox_heka_stream_reader_h_ + +#include "../util/heka_message.h" +#include "../util/input_buffer.h" + +#define LSB_HEKA_STREAM_READER "lsb.heka_stream_reader" + +typedef struct heka_stream_reader +{ + char *name; + lsb_heka_message msg; + lsb_input_buffer buf; +} heka_stream_reader; + +#endif diff --git a/include/luasandbox/lauxlib.h b/include/luasandbox/lauxlib.h new file mode 100644 index 0000000..3425823 --- /dev/null +++ b/include/luasandbox/lauxlib.h @@ -0,0 +1,174 @@ +/* +** $Id: lauxlib.h,v 1.88.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions for building Lua libraries +** See Copyright Notice in lua.h +*/ + + +#ifndef lauxlib_h +#define lauxlib_h + + +#include +#include + +#include "lua.h" + + +#if defined(LUA_COMPAT_GETN) +LUALIB_API int (luaL_getn) (lua_State *L, int t); +LUALIB_API void (luaL_setn) (lua_State *L, int t, int n); +#else +#define luaL_getn(L,i) ((int)lua_objlen(L, i)) +#define luaL_setn(L,i,j) ((void)0) /* no op! */ +#endif + +#if defined(LUA_COMPAT_OPENLIB) +#define luaI_openlib luaL_openlib +#endif + + +/* extra error code for `luaL_load' */ +#define LUA_ERRFILE (LUA_ERRERR+1) + + +typedef struct luaL_Reg { + const char *name; + lua_CFunction func; +} luaL_Reg; + + + +LUALIB_API void (luaI_openlib) (lua_State *L, const char *libname, + const luaL_Reg *l, int nup); +LUALIB_API void (luaL_register) (lua_State *L, const char *libname, + const luaL_Reg *l); +LUALIB_API int (luaL_getmetafield) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_callmeta) (lua_State *L, int obj, const char *e); +LUALIB_API int (luaL_typerror) (lua_State *L, int narg, const char *tname); +LUALIB_API int (luaL_argerror) (lua_State *L, int numarg, const char *extramsg); +LUALIB_API const char *(luaL_checklstring) (lua_State *L, int numArg, + size_t *l); +LUALIB_API const char *(luaL_optlstring) (lua_State *L, int numArg, + const char *def, size_t *l); +LUALIB_API lua_Number (luaL_checknumber) (lua_State *L, int numArg); +LUALIB_API lua_Number (luaL_optnumber) (lua_State *L, int nArg, lua_Number def); + +LUALIB_API lua_Integer (luaL_checkinteger) (lua_State *L, int numArg); +LUALIB_API lua_Integer (luaL_optinteger) (lua_State *L, int nArg, + lua_Integer def); + +LUALIB_API void (luaL_checkstack) (lua_State *L, int sz, const char *msg); +LUALIB_API void (luaL_checktype) (lua_State *L, int narg, int t); +LUALIB_API void (luaL_checkany) (lua_State *L, int narg); + +LUALIB_API int (luaL_newmetatable) (lua_State *L, const char *tname); +LUALIB_API void *(luaL_checkudata) (lua_State *L, int ud, const char *tname); + +LUALIB_API void (luaL_where) (lua_State *L, int lvl); +LUALIB_API int (luaL_error) (lua_State *L, const char *fmt, ...); + +LUALIB_API int (luaL_checkoption) (lua_State *L, int narg, const char *def, + const char *const lst[]); + +LUALIB_API int (luaL_ref) (lua_State *L, int t); +LUALIB_API void (luaL_unref) (lua_State *L, int t, int ref); + +LUALIB_API int (luaL_loadfile) (lua_State *L, const char *filename); +LUALIB_API int (luaL_loadbuffer) (lua_State *L, const char *buff, size_t sz, + const char *name); +LUALIB_API int (luaL_loadstring) (lua_State *L, const char *s); + +LUALIB_API lua_State *(luaL_newstate) (void); + + +LUALIB_API const char *(luaL_gsub) (lua_State *L, const char *s, const char *p, + const char *r); + +LUALIB_API const char *(luaL_findtable) (lua_State *L, int idx, + const char *fname, int szhint); + + + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define luaL_argcheck(L, cond,numarg,extramsg) \ + ((void)((cond) || luaL_argerror(L, (numarg), (extramsg)))) +#define luaL_checkstring(L,n) (luaL_checklstring(L, (n), NULL)) +#define luaL_optstring(L,n,d) (luaL_optlstring(L, (n), (d), NULL)) +#define luaL_checkint(L,n) ((int)luaL_checkinteger(L, (n))) +#define luaL_optint(L,n,d) ((int)luaL_optinteger(L, (n), (d))) +#define luaL_checklong(L,n) ((long)luaL_checkinteger(L, (n))) +#define luaL_optlong(L,n,d) ((long)luaL_optinteger(L, (n), (d))) + +#define luaL_typename(L,i) lua_typename(L, lua_type(L,(i))) + +#define luaL_dofile(L, fn) \ + (luaL_loadfile(L, fn) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_dostring(L, s) \ + (luaL_loadstring(L, s) || lua_pcall(L, 0, LUA_MULTRET, 0)) + +#define luaL_getmetatable(L,n) (lua_getfield(L, LUA_REGISTRYINDEX, (n))) + +#define luaL_opt(L,f,n,d) (lua_isnoneornil(L,(n)) ? (d) : f(L,(n))) + +/* +** {====================================================== +** Generic Buffer manipulation +** ======================================================= +*/ + + + +typedef struct luaL_Buffer { + char *p; /* current position in buffer */ + int lvl; /* number of strings in the stack (level) */ + lua_State *L; + char buffer[LUAL_BUFFERSIZE]; +} luaL_Buffer; + +#define luaL_addchar(B,c) \ + ((void)((B)->p < ((B)->buffer+LUAL_BUFFERSIZE) || luaL_prepbuffer(B)), \ + (*(B)->p++ = (char)(c))) + +/* compatibility only */ +#define luaL_putchar(B,c) luaL_addchar(B,c) + +#define luaL_addsize(B,n) ((B)->p += (n)) + +LUALIB_API void (luaL_buffinit) (lua_State *L, luaL_Buffer *B); +LUALIB_API char *(luaL_prepbuffer) (luaL_Buffer *B); +LUALIB_API void (luaL_addlstring) (luaL_Buffer *B, const char *s, size_t l); +LUALIB_API void (luaL_addstring) (luaL_Buffer *B, const char *s); +LUALIB_API void (luaL_addvalue) (luaL_Buffer *B); +LUALIB_API void (luaL_pushresult) (luaL_Buffer *B); + + +/* }====================================================== */ + + +/* compatibility with ref system */ + +/* pre-defined references */ +#define LUA_NOREF (-2) +#define LUA_REFNIL (-1) + +#define lua_ref(L,lock) ((lock) ? luaL_ref(L, LUA_REGISTRYINDEX) : \ + (lua_pushstring(L, "unlocked references are obsolete"), lua_error(L), 0)) + +#define lua_unref(L,ref) luaL_unref(L, LUA_REGISTRYINDEX, (ref)) + +#define lua_getref(L,ref) lua_rawgeti(L, LUA_REGISTRYINDEX, (ref)) + + +#define luaL_reg luaL_Reg + +#endif + + diff --git a/include/luasandbox/lua.h b/include/luasandbox/lua.h new file mode 100644 index 0000000..232fca9 --- /dev/null +++ b/include/luasandbox/lua.h @@ -0,0 +1,397 @@ +/* +** $Id: lua.h,v 1.218.1.7 2012/01/13 20:36:20 roberto Exp $ +** Lua - An Extensible Extension Language +** Lua.org, PUC-Rio, Brazil (http://www.lua.org) +** See Copyright Notice at the end of this file +*/ + + +#ifndef lua_h +#define lua_h + +#include +#include + + +#include "luaconf.h" + + +#define LUA_VERSION "Lua 5.1" +#define LUA_RELEASE "Lua 5.1.5" +#define LUA_VERSION_NUM 501 +#define LUA_COPYRIGHT "Copyright (C) 1994-2012 Lua.org, PUC-Rio" +#define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo & W. Celes" + + +/* mark for precompiled code (`Lua') */ +#define LUA_SIGNATURE "\033Lua" + +/* option for multiple returns in `lua_pcall' and `lua_call' */ +#define LUA_MULTRET (-1) + + +/* +** pseudo-indices +*/ +#define LUA_REGISTRYINDEX (-10000) +#define LUA_ENVIRONINDEX (-10001) +#define LUA_GLOBALSINDEX (-10002) +#define lua_upvalueindex(i) (LUA_GLOBALSINDEX-(i)) + + +/* thread status; 0 is OK */ +#define LUA_YIELD 1 +#define LUA_ERRRUN 2 +#define LUA_ERRSYNTAX 3 +#define LUA_ERRMEM 4 +#define LUA_ERRERR 5 + + +typedef struct lua_State lua_State; + +typedef int (*lua_CFunction) (lua_State *L); + + +/* +** functions that read/write blocks when loading/dumping Lua chunks +*/ +typedef const char * (*lua_Reader) (lua_State *L, void *ud, size_t *sz); + +typedef int (*lua_Writer) (lua_State *L, const void* p, size_t sz, void* ud); + + +/* +** prototype for memory-allocation functions +*/ +typedef void * (*lua_Alloc) (void *ud, void *ptr, size_t osize, size_t nsize); + + +/* +** basic types +*/ +#define LUA_TNONE (-1) + +#define LUA_TNIL 0 +#define LUA_TBOOLEAN 1 +#define LUA_TLIGHTUSERDATA 2 +#define LUA_TNUMBER 3 +#define LUA_TSTRING 4 +#define LUA_TTABLE 5 +#define LUA_TFUNCTION 6 +#define LUA_TUSERDATA 7 +#define LUA_TTHREAD 8 + +/* +** table types +*/ +#define LUA_TTEMPTY 0 +#define LUA_TTARRAY 1 +#define LUA_TTHASH 2 +#define LUA_TTMIXED 3 + + +/* minimum Lua stack available to a C function */ +#define LUA_MINSTACK 20 + + +/* +** generic extra include file +*/ +#if defined(LUA_USER_H) +#include LUA_USER_H +#endif + + +/* type of numbers in Lua */ +typedef LUA_NUMBER lua_Number; + + +/* type for integer functions */ +typedef LUA_INTEGER lua_Integer; + + + +/* +** state manipulation +*/ +LUA_API lua_State *(lua_newstate) (lua_Alloc f, void *ud); +LUA_API void (lua_close) (lua_State *L); +LUA_API lua_State *(lua_newthread) (lua_State *L); + +LUA_API lua_CFunction (lua_atpanic) (lua_State *L, lua_CFunction panicf); + + +/* +** basic stack manipulation +*/ +LUA_API int (lua_gettop) (lua_State *L); +LUA_API void (lua_settop) (lua_State *L, int idx); +LUA_API void (lua_pushvalue) (lua_State *L, int idx); +LUA_API void (lua_remove) (lua_State *L, int idx); +LUA_API void (lua_insert) (lua_State *L, int idx); +LUA_API void (lua_replace) (lua_State *L, int idx); +LUA_API int (lua_checkstack) (lua_State *L, int sz); + +LUA_API void (lua_xmove) (lua_State *from, lua_State *to, int n); + + +/* +** access functions (stack -> C) +*/ + +LUA_API int (lua_tabletype) (lua_State *L, int idx); +LUA_API int (lua_isnumber) (lua_State *L, int idx); +LUA_API int (lua_isstring) (lua_State *L, int idx); +LUA_API int (lua_iscfunction) (lua_State *L, int idx); +LUA_API int (lua_isuserdata) (lua_State *L, int idx); +LUA_API int (lua_type) (lua_State *L, int idx); +LUA_API const char *(lua_typename) (lua_State *L, int tp); + +LUA_API int (lua_equal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_rawequal) (lua_State *L, int idx1, int idx2); +LUA_API int (lua_lessthan) (lua_State *L, int idx1, int idx2); + +LUA_API lua_Number (lua_tonumber) (lua_State *L, int idx); +LUA_API lua_Integer (lua_tointeger) (lua_State *L, int idx); +LUA_API int (lua_toboolean) (lua_State *L, int idx); +LUA_API const char *(lua_tolstring) (lua_State *L, int idx, size_t *len); +LUA_API size_t (lua_objlen) (lua_State *L, int idx); +LUA_API lua_CFunction (lua_tocfunction) (lua_State *L, int idx); +LUA_API void *(lua_touserdata) (lua_State *L, int idx); +LUA_API lua_State *(lua_tothread) (lua_State *L, int idx); +LUA_API const void *(lua_topointer) (lua_State *L, int idx); + + +/* +** push functions (C -> stack) +*/ +LUA_API void (lua_pushnil) (lua_State *L); +LUA_API void (lua_pushnumber) (lua_State *L, lua_Number n); +LUA_API void (lua_pushinteger) (lua_State *L, lua_Integer n); +LUA_API void (lua_pushlstring) (lua_State *L, const char *s, size_t l); +LUA_API void (lua_pushstring) (lua_State *L, const char *s); +LUA_API const char *(lua_pushvfstring) (lua_State *L, const char *fmt, + va_list argp); +LUA_API const char *(lua_pushfstring) (lua_State *L, const char *fmt, ...); +LUA_API void (lua_pushcclosure) (lua_State *L, lua_CFunction fn, int n); +LUA_API void (lua_pushboolean) (lua_State *L, int b); +LUA_API void (lua_pushlightuserdata) (lua_State *L, void *p); +LUA_API int (lua_pushthread) (lua_State *L); + + +/* +** get functions (Lua -> stack) +*/ +LUA_API void (lua_gettable) (lua_State *L, int idx); +LUA_API void (lua_getfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawget) (lua_State *L, int idx); +LUA_API void (lua_rawgeti) (lua_State *L, int idx, int n); +LUA_API void (lua_createtable) (lua_State *L, int narr, int nrec); +LUA_API void *(lua_newuserdata) (lua_State *L, size_t sz); +LUA_API int (lua_getmetatable) (lua_State *L, int objindex); +LUA_API void (lua_getfenv) (lua_State *L, int idx); + + +/* +** set functions (stack -> Lua) +*/ +LUA_API void (lua_settable) (lua_State *L, int idx); +LUA_API void (lua_setfield) (lua_State *L, int idx, const char *k); +LUA_API void (lua_rawset) (lua_State *L, int idx); +LUA_API void (lua_rawseti) (lua_State *L, int idx, int n); +LUA_API int (lua_setmetatable) (lua_State *L, int objindex); +LUA_API int (lua_setfenv) (lua_State *L, int idx); + + +/* +** `load' and `call' functions (load and run Lua code) +*/ +LUA_API void (lua_call) (lua_State *L, int nargs, int nresults); +LUA_API int (lua_pcall) (lua_State *L, int nargs, int nresults, int errfunc); +LUA_API int (lua_cpcall) (lua_State *L, lua_CFunction func, void *ud); +LUA_API int (lua_load) (lua_State *L, lua_Reader reader, void *dt, + const char *chunkname); + +LUA_API int (lua_dump) (lua_State *L, lua_Writer writer, void *data); + + +/* +** coroutine functions +*/ +LUA_API int (lua_yield) (lua_State *L, int nresults); +LUA_API int (lua_resume) (lua_State *L, int narg); +LUA_API int (lua_status) (lua_State *L); + +/* +** garbage-collection function and options +*/ + +#define LUA_GCSTOP 0 +#define LUA_GCRESTART 1 +#define LUA_GCCOLLECT 2 +#define LUA_GCCOUNT 3 +#define LUA_GCCOUNTB 4 +#define LUA_GCSTEP 5 +#define LUA_GCSETPAUSE 6 +#define LUA_GCSETSTEPMUL 7 + +LUA_API int (lua_gc) (lua_State *L, int what, int data); + + +/* +** miscellaneous functions +*/ + +LUA_API int (lua_error) (lua_State *L); + +LUA_API int (lua_next) (lua_State *L, int idx); + +LUA_API void (lua_concat) (lua_State *L, int n); + +LUA_API lua_Alloc (lua_getallocf) (lua_State *L, void **ud); +LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud); + + + +/* +** =============================================================== +** some useful macros +** =============================================================== +*/ + +#define lua_pop(L,n) lua_settop(L, -(n)-1) + +#define lua_newtable(L) lua_createtable(L, 0, 0) + +#define lua_register(L,n,f) (lua_pushcfunction(L, (f)), lua_setglobal(L, (n))) + +#define lua_pushcfunction(L,f) lua_pushcclosure(L, (f), 0) + +#define lua_strlen(L,i) lua_objlen(L, (i)) + +#define lua_isfunction(L,n) (lua_type(L, (n)) == LUA_TFUNCTION) +#define lua_istable(L,n) (lua_type(L, (n)) == LUA_TTABLE) +#define lua_islightuserdata(L,n) (lua_type(L, (n)) == LUA_TLIGHTUSERDATA) +#define lua_isnil(L,n) (lua_type(L, (n)) == LUA_TNIL) +#define lua_isboolean(L,n) (lua_type(L, (n)) == LUA_TBOOLEAN) +#define lua_isthread(L,n) (lua_type(L, (n)) == LUA_TTHREAD) +#define lua_isnone(L,n) (lua_type(L, (n)) == LUA_TNONE) +#define lua_isnoneornil(L, n) (lua_type(L, (n)) <= 0) + +#define lua_pushliteral(L, s) \ + lua_pushlstring(L, "" s, (sizeof(s)/sizeof(char))-1) + +#define lua_setglobal(L,s) lua_setfield(L, LUA_GLOBALSINDEX, (s)) +#define lua_getglobal(L,s) lua_getfield(L, LUA_GLOBALSINDEX, (s)) + +#define lua_tostring(L,i) lua_tolstring(L, (i), NULL) + + + +/* +** compatibility macros and functions +*/ + +#define lua_open() luaL_newstate() + +#define lua_getregistry(L) lua_pushvalue(L, LUA_REGISTRYINDEX) + +#define lua_getgccount(L) lua_gc(L, LUA_GCCOUNT, 0) + +#define lua_Chunkreader lua_Reader +#define lua_Chunkwriter lua_Writer + + +/* hack */ +LUA_API void lua_setlevel (lua_State *from, lua_State *to); + + +/* +** {====================================================================== +** Debug API +** ======================================================================= +*/ + + +/* +** Event codes +*/ +#define LUA_HOOKCALL 0 +#define LUA_HOOKRET 1 +#define LUA_HOOKLINE 2 +#define LUA_HOOKCOUNT 3 +#define LUA_HOOKTAILRET 4 + + +/* +** Event masks +*/ +#define LUA_MASKCALL (1 << LUA_HOOKCALL) +#define LUA_MASKRET (1 << LUA_HOOKRET) +#define LUA_MASKLINE (1 << LUA_HOOKLINE) +#define LUA_MASKCOUNT (1 << LUA_HOOKCOUNT) + +typedef struct lua_Debug lua_Debug; /* activation record */ + + +/* Functions to be called by the debuger in specific events */ +typedef void (*lua_Hook) (lua_State *L, lua_Debug *ar); + + +LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar); +LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar); +LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n); +LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n); +LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n); + +LUA_API int lua_sethook (lua_State *L, lua_Hook func, int mask, int count); +LUA_API lua_Hook lua_gethook (lua_State *L); +LUA_API int lua_gethookmask (lua_State *L); +LUA_API int lua_gethookcount (lua_State *L); +LUA_API int lua_gethookcountremaining (lua_State *L); + + +struct lua_Debug { + int event; + const char *name; /* (n) */ + const char *namewhat; /* (n) `global', `local', `field', `method' */ + const char *what; /* (S) `Lua', `C', `main', `tail' */ + const char *source; /* (S) */ + int currentline; /* (l) */ + int nups; /* (u) number of upvalues */ + int linedefined; /* (S) */ + int lastlinedefined; /* (S) */ + char short_src[LUA_IDSIZE]; /* (S) */ + /* private part */ + int i_ci; /* active function */ +}; + +/* }====================================================================== */ + + +/****************************************************************************** +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ + + +#endif diff --git a/include/luasandbox/luaconf.h b/include/luasandbox/luaconf.h new file mode 100644 index 0000000..fa748aa --- /dev/null +++ b/include/luasandbox/luaconf.h @@ -0,0 +1,765 @@ +/* +** $Id: luaconf.h,v 1.82.1.7 2008/02/11 16:25:08 roberto Exp $ +** Configuration file for Lua +** See Copyright Notice in lua.h +*/ + + +#ifndef lconfig_h +#define lconfig_h + +#include +#include + + +/* +** ================================================================== +** Search for "@@" to find all configurable definitions. +** =================================================================== +*/ + + +/* +@@ LUA_ANSI controls the use of non-ansi features. +** CHANGE it (define it) if you want Lua to avoid the use of any +** non-ansi feature or library. +*/ +#if defined(__STRICT_ANSI__) +#define LUA_ANSI +#endif + + +#if !defined(LUA_ANSI) && defined(_WIN32) +#define LUA_WIN +#endif + +#if defined(LUA_USE_LINUX) +#define LUA_USE_POSIX +#define LUA_USE_DLOPEN /* needs an extra library: -ldl */ +#define LUA_USE_READLINE /* needs some extra libraries */ +#endif + +#if defined(LUA_USE_MACOSX) +#define LUA_USE_POSIX +#define LUA_DL_DYLD /* does not need extra library */ +#endif + + + +/* +@@ LUA_USE_POSIX includes all functionallity listed as X/Open System +@* Interfaces Extension (XSI). +** CHANGE it (define it) if your system is XSI compatible. +*/ +#if defined(LUA_USE_POSIX) +#define LUA_USE_MKSTEMP +#define LUA_USE_ISATTY +#define LUA_USE_POPEN +#define LUA_USE_ULONGJMP +#endif + + +/* +@@ LUA_PATH and LUA_CPATH are the names of the environment variables that +@* Lua check to set its paths. +@@ LUA_INIT is the name of the environment variable that Lua +@* checks for initialization code. +** CHANGE them if you want different names. +*/ +#define LUA_PATH "LUA_PATH" +#define LUA_CPATH "LUA_CPATH" +#define LUA_INIT "LUA_INIT" + + +/* +@@ LUA_PATH_DEFAULT is the default path that Lua uses to look for +@* Lua libraries. +@@ LUA_CPATH_DEFAULT is the default path that Lua uses to look for +@* C libraries. +** CHANGE them if your machine has a non-conventional directory +** hierarchy or if you want to install your libraries in +** non-conventional directories. +*/ +#if defined(_WIN32) +/* +** In Windows, any exclamation mark ('!') in the path is replaced by the +** path of the directory of the executable file of the current process. +*/ +#define LUA_LDIR "!\\lua\\" +#define LUA_CDIR "!\\" +#define LUA_PATH_DEFAULT \ + ".\\?.lua;" LUA_LDIR"?.lua;" LUA_LDIR"?\\init.lua;" \ + LUA_CDIR"?.lua;" LUA_CDIR"?\\init.lua" +#define LUA_CPATH_DEFAULT \ + ".\\?.dll;" LUA_CDIR"?.dll;" LUA_CDIR"loadall.dll" + +#else +#define LUA_ROOT "/usr/local/" +#define LUA_LDIR LUA_ROOT "share/lua/5.1/" +#define LUA_CDIR LUA_ROOT "lib/lua/5.1/" +#define LUA_PATH_DEFAULT \ + "./?.lua;" LUA_LDIR"?.lua;" LUA_LDIR"?/init.lua;" \ + LUA_CDIR"?.lua;" LUA_CDIR"?/init.lua" +#define LUA_CPATH_DEFAULT \ + "./?.so;" LUA_CDIR"?.so;" LUA_CDIR"loadall.so" +#endif + + +/* +@@ LUA_DIRSEP is the directory separator (for submodules). +** CHANGE it if your machine does not use "/" as the directory separator +** and is not Windows. (On Windows Lua automatically uses "\".) +*/ +#if defined(_WIN32) +#define LUA_DIRSEP "\\" +#else +#define LUA_DIRSEP "/" +#endif + + +/* +@@ LUA_PATHSEP is the character that separates templates in a path. +@@ LUA_PATH_MARK is the string that marks the substitution points in a +@* template. +@@ LUA_EXECDIR in a Windows path is replaced by the executable's +@* directory. +@@ LUA_IGMARK is a mark to ignore all before it when bulding the +@* luaopen_ function name. +** CHANGE them if for some reason your system cannot use those +** characters. (E.g., if one of those characters is a common character +** in file/directory names.) Probably you do not need to change them. +*/ +#define LUA_PATHSEP ";" +#define LUA_PATH_MARK "?" +#define LUA_EXECDIR "!" +#define LUA_IGMARK "-" + + +/* +@@ LUA_INTEGER is the integral type used by lua_pushinteger/lua_tointeger. +** CHANGE that if ptrdiff_t is not adequate on your machine. (On most +** machines, ptrdiff_t gives a good choice between int or long.) +*/ +#define LUA_INTEGER ptrdiff_t + + +/* +@@ LUA_API is a mark for all core API functions. +@@ LUALIB_API is a mark for all standard library functions. +** CHANGE them if you need to define those functions in some special way. +** For instance, if you want to create one Windows DLL with the core and +** the libraries, you may want to use the following definition (define +** LUA_BUILD_AS_DLL to get it). +*/ +#if defined(LUA_BUILD_AS_DLL) + +#if defined(LUA_CORE) || defined(LUA_LIB) +#define LUA_API __declspec(dllexport) +#else +#define LUA_API __declspec(dllimport) +#endif + +#else +#if __GNUC__ >= 4 +#define LUA_API __attribute__ ((visibility ("default"))) +#else +#define LUA_API +#endif +#endif + +/* more often than not the libs go together with the core */ +#define LUALIB_API LUA_API + + +/* +@@ LUAI_FUNC is a mark for all extern functions that are not to be +@* exported to outside modules. +@@ LUAI_DATA is a mark for all extern (const) variables that are not to +@* be exported to outside modules. +** CHANGE them if you need to mark them in some special way. Elf/gcc +** (versions 3.2 and later) mark them as "hidden" to optimize access +** when Lua is compiled as a shared library. +*/ +#if defined(luaall_c) +#define LUAI_FUNC static +#define LUAI_DATA /* empty */ + +#elif defined(__GNUC__) && ((__GNUC__*100 + __GNUC_MINOR__) >= 302) && \ + defined(__ELF__) +#define LUAI_FUNC __attribute__((visibility("hidden"))) extern +#define LUAI_DATA LUAI_FUNC + +#else +#define LUAI_FUNC extern +#define LUAI_DATA extern +#endif + + + +/* +@@ LUA_QL describes how error messages quote program elements. +** CHANGE it if you want a different appearance. +*/ +#define LUA_QL(x) "'" x "'" +#define LUA_QS LUA_QL("%s") + + +/* +@@ LUA_IDSIZE gives the maximum size for the description of the source +@* of a function in debug information. +** CHANGE it if you want a different size. +*/ +#define LUA_IDSIZE 60 + + +/* +** {================================================================== +** Stand-alone configuration +** =================================================================== +*/ + +#if defined(lua_c) || defined(luaall_c) + +/* +@@ lua_stdin_is_tty detects whether the standard input is a 'tty' (that +@* is, whether we're running lua interactively). +** CHANGE it if you have a better definition for non-POSIX/non-Windows +** systems. +*/ +#if defined(LUA_USE_ISATTY) +#include +#define lua_stdin_is_tty() isatty(0) +#elif defined(LUA_WIN) +#include +#include +#define lua_stdin_is_tty() _isatty(_fileno(stdin)) +#else +#define lua_stdin_is_tty() 1 /* assume stdin is a tty */ +#endif + + +/* +@@ LUA_PROMPT is the default prompt used by stand-alone Lua. +@@ LUA_PROMPT2 is the default continuation prompt used by stand-alone Lua. +** CHANGE them if you want different prompts. (You can also change the +** prompts dynamically, assigning to globals _PROMPT/_PROMPT2.) +*/ +#define LUA_PROMPT "> " +#define LUA_PROMPT2 ">> " + + +/* +@@ LUA_PROGNAME is the default name for the stand-alone Lua program. +** CHANGE it if your stand-alone interpreter has a different name and +** your system is not able to detect that name automatically. +*/ +#define LUA_PROGNAME "lua" + + +/* +@@ LUA_MAXINPUT is the maximum length for an input line in the +@* stand-alone interpreter. +** CHANGE it if you need longer lines. +*/ +#define LUA_MAXINPUT 512 + + +/* +@@ lua_readline defines how to show a prompt and then read a line from +@* the standard input. +@@ lua_saveline defines how to "save" a read line in a "history". +@@ lua_freeline defines how to free a line read by lua_readline. +** CHANGE them if you want to improve this functionality (e.g., by using +** GNU readline and history facilities). +*/ +#if defined(LUA_USE_READLINE) +#include +#include +#include +#define lua_readline(L,b,p) ((void)L, ((b)=readline(p)) != NULL) +#define lua_saveline(L,idx) \ + if (lua_strlen(L,idx) > 0) /* non-empty line? */ \ + add_history(lua_tostring(L, idx)); /* add it to history */ +#define lua_freeline(L,b) ((void)L, free(b)) +#else +#define lua_readline(L,b,p) \ + ((void)L, fputs(p, stdout), fflush(stdout), /* show prompt */ \ + fgets(b, LUA_MAXINPUT, stdin) != NULL) /* get line */ +#define lua_saveline(L,idx) { (void)L; (void)idx; } +#define lua_freeline(L,b) { (void)L; (void)b; } +#endif + +#endif + +/* }================================================================== */ + + +/* +@@ LUAI_GCPAUSE defines the default pause between garbage-collector cycles +@* as a percentage. +** CHANGE it if you want the GC to run faster or slower (higher values +** mean larger pauses which mean slower collection.) You can also change +** this value dynamically. +*/ +#define LUAI_GCPAUSE 200 /* 200% (wait memory to double before next GC) */ + + +/* +@@ LUAI_GCMUL defines the default speed of garbage collection relative to +@* memory allocation as a percentage. +** CHANGE it if you want to change the granularity of the garbage +** collection. (Higher values mean coarser collections. 0 represents +** infinity, where each step performs a full collection.) You can also +** change this value dynamically. +*/ +#define LUAI_GCMUL 200 /* GC runs 'twice the speed' of memory allocation */ + + + +/* +@@ LUA_COMPAT_GETN controls compatibility with old getn behavior. +** CHANGE it (define it) if you want exact compatibility with the +** behavior of setn/getn in Lua 5.0. +*/ +#undef LUA_COMPAT_GETN + +/* +@@ LUA_COMPAT_LOADLIB controls compatibility about global loadlib. +** CHANGE it to undefined as soon as you do not need a global 'loadlib' +** function (the function is still available as 'package.loadlib'). +*/ +#undef LUA_COMPAT_LOADLIB + +/* +@@ LUA_COMPAT_VARARG controls compatibility with old vararg feature. +** CHANGE it to undefined as soon as your programs use only '...' to +** access vararg parameters (instead of the old 'arg' table). +*/ +#define LUA_COMPAT_VARARG + +/* +@@ LUA_COMPAT_MOD controls compatibility with old math.mod function. +** CHANGE it to undefined as soon as your programs use 'math.fmod' or +** the new '%' operator instead of 'math.mod'. +*/ +#define LUA_COMPAT_MOD + +/* +@@ LUA_COMPAT_LSTR controls compatibility with old long string nesting +@* facility. +** CHANGE it to 2 if you want the old behaviour, or undefine it to turn +** off the advisory error when nesting [[...]]. +*/ +#define LUA_COMPAT_LSTR 1 + +/* +@@ LUA_COMPAT_GFIND controls compatibility with old 'string.gfind' name. +** CHANGE it to undefined as soon as you rename 'string.gfind' to +** 'string.gmatch'. +*/ +#define LUA_COMPAT_GFIND + +/* +@@ LUA_COMPAT_OPENLIB controls compatibility with old 'luaL_openlib' +@* behavior. +** CHANGE it to undefined as soon as you replace to 'luaL_register' +** your uses of 'luaL_openlib' +*/ +#define LUA_COMPAT_OPENLIB + + + +/* +@@ luai_apicheck is the assert macro used by the Lua-C API. +** CHANGE luai_apicheck if you want Lua to perform some checks in the +** parameters it gets from API calls. This may slow down the interpreter +** a bit, but may be quite useful when debugging C code that interfaces +** with Lua. A useful redefinition is to use assert.h. +*/ +#if defined(LUA_USE_APICHECK) +#include +#define luai_apicheck(L,o) { (void)L; assert(o); } +#else +#define luai_apicheck(L,o) { (void)L; } +#endif + + +/* +@@ LUAI_BITSINT defines the number of bits in an int. +** CHANGE here if Lua cannot automatically detect the number of bits of +** your machine. Probably you do not need to change this. +*/ +/* avoid overflows in comparison */ +#if INT_MAX-20 < 32760 +#define LUAI_BITSINT 16 +#elif INT_MAX > 2147483640L +/* int has at least 32 bits */ +#define LUAI_BITSINT 32 +#else +#error "you must define LUA_BITSINT with number of bits in an integer" +#endif + + +/* +@@ LUAI_UINT32 is an unsigned integer with at least 32 bits. +@@ LUAI_INT32 is an signed integer with at least 32 bits. +@@ LUAI_UMEM is an unsigned integer big enough to count the total +@* memory used by Lua. +@@ LUAI_MEM is a signed integer big enough to count the total memory +@* used by Lua. +** CHANGE here if for some weird reason the default definitions are not +** good enough for your machine. (The definitions in the 'else' +** part always works, but may waste space on machines with 64-bit +** longs.) Probably you do not need to change this. +*/ +#if LUAI_BITSINT >= 32 +#define LUAI_UINT32 unsigned int +#define LUAI_INT32 int +#define LUAI_MAXINT32 INT_MAX +#define LUAI_UMEM size_t +#define LUAI_MEM ptrdiff_t +#else +/* 16-bit ints */ +#define LUAI_UINT32 unsigned long +#define LUAI_INT32 long +#define LUAI_MAXINT32 LONG_MAX +#define LUAI_UMEM unsigned long +#define LUAI_MEM long +#endif + + +/* +@@ LUAI_MAXCALLS limits the number of nested calls. +** CHANGE it if you need really deep recursive calls. This limit is +** arbitrary; its only purpose is to stop infinite recursion before +** exhausting memory. +*/ +#define LUAI_MAXCALLS 20000 + + +/* +@@ LUAI_MAXCSTACK limits the number of Lua stack slots that a C function +@* can use. +** CHANGE it if you need lots of (Lua) stack space for your C +** functions. This limit is arbitrary; its only purpose is to stop C +** functions to consume unlimited stack space. (must be smaller than +** -LUA_REGISTRYINDEX) +*/ +#define LUAI_MAXCSTACK 8000 + + + +/* +** {================================================================== +** CHANGE (to smaller values) the following definitions if your system +** has a small C stack. (Or you may want to change them to larger +** values if your system has a large C stack and these limits are +** too rigid for you.) Some of these constants control the size of +** stack-allocated arrays used by the compiler or the interpreter, while +** others limit the maximum number of recursive calls that the compiler +** or the interpreter can perform. Values too large may cause a C stack +** overflow for some forms of deep constructs. +** =================================================================== +*/ + + +/* +@@ LUAI_MAXCCALLS is the maximum depth for nested C calls (short) and +@* syntactical nested non-terminals in a program. +*/ +#define LUAI_MAXCCALLS 200 + + +/* +@@ LUAI_MAXVARS is the maximum number of local variables per function +@* (must be smaller than 250). +*/ +#define LUAI_MAXVARS 200 + + +/* +@@ LUAI_MAXUPVALUES is the maximum number of upvalues per function +@* (must be smaller than 250). +*/ +#define LUAI_MAXUPVALUES 60 + + +/* +@@ LUAL_BUFFERSIZE is the buffer size used by the lauxlib buffer system. +*/ +#define LUAL_BUFFERSIZE BUFSIZ + +/* }================================================================== */ + + + + +/* +** {================================================================== +@@ LUA_NUMBER is the type of numbers in Lua. +** CHANGE the following definitions only if you want to build Lua +** with a number type different from double. You may also need to +** change lua_number2int & lua_number2integer. +** =================================================================== +*/ + +#define LUA_NUMBER_DOUBLE +#define LUA_NUMBER double + +/* +@@ LUAI_UACNUMBER is the result of an 'usual argument conversion' +@* over a number. +*/ +#define LUAI_UACNUMBER double + + +/* +@@ LUA_NUMBER_SCAN is the format for reading numbers. +@@ LUA_NUMBER_FMT is the format for writing numbers. +@@ lua_number2str converts a number to a string. +@@ LUAI_MAXNUMBER2STR is maximum size of previous conversion. +@@ lua_str2number converts a string to a number. +*/ +#define LUA_NUMBER_SCAN "%lf" +#define LUA_NUMBER_FMT "%.14g" +#define lua_number2str(s,n) sprintf((s), LUA_NUMBER_FMT, (n)) +#define LUAI_MAXNUMBER2STR 32 /* 16 digits, sign, point, and \0 */ +#define lua_str2number(s,p) strtod((s), (p)) + + +/* +@@ The luai_num* macros define the primitive operations over numbers. +*/ +#if defined(LUA_CORE) +#include +#define luai_numadd(a,b) ((a)+(b)) +#define luai_numsub(a,b) ((a)-(b)) +#define luai_nummul(a,b) ((a)*(b)) +#define luai_numdiv(a,b) ((a)/(b)) +#define luai_nummod(a,b) ((a) - floor((a)/(b))*(b)) +#define luai_numpow(a,b) (pow(a,b)) +#define luai_numunm(a) (-(a)) +#define luai_numeq(a,b) ((a)==(b)) +#define luai_numlt(a,b) ((a)<(b)) +#define luai_numle(a,b) ((a)<=(b)) +#define luai_numisnan(a) (!luai_numeq((a), (a))) +#endif + + +/* +@@ lua_number2int is a macro to convert lua_Number to int. +@@ lua_number2integer is a macro to convert lua_Number to lua_Integer. +** CHANGE them if you know a faster way to convert a lua_Number to +** int (with any rounding method and without throwing errors) in your +** system. In Pentium machines, a naive typecast from double to int +** in C is extremely slow, so any alternative is worth trying. +*/ + +/* On a Pentium, resort to a trick */ +#if defined(LUA_NUMBER_DOUBLE) && !defined(LUA_ANSI) && !defined(__SSE2__) && \ + (defined(__i386) || defined (_M_IX86) || defined(__i386__)) + +/* On a Microsoft compiler, use assembler */ +#if defined(_MSC_VER) + +#define lua_number2int(i,d) __asm fld d __asm fistp i +#define lua_number2integer(i,n) lua_number2int(i, n) + +/* the next trick should work on any Pentium, but sometimes clashes + with a DirectX idiosyncrasy */ +#else + +union luai_Cast { double l_d; long l_l; }; +#define lua_number2int(i,d) \ + { volatile union luai_Cast u; u.l_d = (d) + 6755399441055744.0; (i) = u.l_l; } +#define lua_number2integer(i,n) lua_number2int(i, n) + +#endif + + +/* this option always works, but may be slow */ +#else +#define lua_number2int(i,d) ((i)=(int)(d)) +#define lua_number2integer(i,d) ((i)=(lua_Integer)(d)) + +#endif + +/* }================================================================== */ + + +/* +@@ LUAI_USER_ALIGNMENT_T is a type that requires maximum alignment. +** CHANGE it if your system requires alignments larger than double. (For +** instance, if your system supports long doubles and they must be +** aligned in 16-byte boundaries, then you should add long double in the +** union.) Probably you do not need to change this. +*/ +#define LUAI_USER_ALIGNMENT_T union { double u; void *s; long l; } + + +/* +@@ LUAI_THROW/LUAI_TRY define how Lua does exception handling. +** CHANGE them if you prefer to use longjmp/setjmp even with C++ +** or if want/don't to use _longjmp/_setjmp instead of regular +** longjmp/setjmp. By default, Lua handles errors with exceptions when +** compiling as C++ code, with _longjmp/_setjmp when asked to use them, +** and with longjmp/setjmp otherwise. +*/ +#if defined(__cplusplus) +/* C++ exceptions */ +#define LUAI_THROW(L,c) throw(c) +#define LUAI_TRY(L,c,a) try { a } catch(...) \ + { if ((c)->status == 0) (c)->status = -1; } +#define luai_jmpbuf int /* dummy variable */ + +#elif defined(LUA_USE_ULONGJMP) +/* in Unix, try _longjmp/_setjmp (more efficient) */ +#define LUAI_THROW(L,c) _longjmp((c)->b, 1) +#define LUAI_TRY(L,c,a) if (_setjmp((c)->b) == 0) { a } +#define luai_jmpbuf jmp_buf + +#else +/* default handling with long jumps */ +#define LUAI_THROW(L,c) longjmp((c)->b, 1) +#define LUAI_TRY(L,c,a) if (setjmp((c)->b) == 0) { a } +#define luai_jmpbuf jmp_buf + +#endif + + +/* +@@ LUA_MAXCAPTURES is the maximum number of captures that a pattern +@* can do during pattern-matching. +** CHANGE it if you need more captures. This limit is arbitrary. +*/ +#define LUA_MAXCAPTURES 32 + + +/* +@@ lua_tmpnam is the function that the OS library uses to create a +@* temporary name. +@@ LUA_TMPNAMBUFSIZE is the maximum size of a name created by lua_tmpnam. +** CHANGE them if you have an alternative to tmpnam (which is considered +** insecure) or if you want the original tmpnam anyway. By default, Lua +** uses tmpnam except when POSIX is available, where it uses mkstemp. +*/ +#if defined(loslib_c) || defined(luaall_c) + +#if defined(LUA_USE_MKSTEMP) +#include +#define LUA_TMPNAMBUFSIZE 32 +#define lua_tmpnam(b,e) { \ + strcpy(b, "/tmp/lua_XXXXXX"); \ + e = mkstemp(b); \ + if (e != -1) close(e); \ + e = (e == -1); } + +#else +#define LUA_TMPNAMBUFSIZE L_tmpnam +#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } +#endif + +#endif + + +/* +@@ lua_popen spawns a new process connected to the current one through +@* the file streams. +** CHANGE it if you have a way to implement it in your system. +*/ +#if defined(LUA_USE_POPEN) + +#define lua_popen(L,c,m) ((void)L, fflush(NULL), popen(c,m)) +#define lua_pclose(L,file) ((void)L, (pclose(file) != -1)) + +#elif defined(LUA_WIN) + +#define lua_popen(L,c,m) ((void)L, _popen(c,m)) +#define lua_pclose(L,file) ((void)L, (_pclose(file) != -1)) + +#else + +#define lua_popen(L,c,m) ((void)((void)c, m), \ + luaL_error(L, LUA_QL("popen") " not supported"), (FILE*)0) +#define lua_pclose(L,file) ((void)((void)L, file), 0) + +#endif + +/* +@@ LUA_DL_* define which dynamic-library system Lua should use. +** CHANGE here if Lua has problems choosing the appropriate +** dynamic-library system for your platform (either Windows' DLL, Mac's +** dyld, or Unix's dlopen). If your system is some kind of Unix, there +** is a good chance that it has dlopen, so LUA_DL_DLOPEN will work for +** it. To use dlopen you also need to adapt the src/Makefile (probably +** adding -ldl to the linker options), so Lua does not select it +** automatically. (When you change the makefile to add -ldl, you must +** also add -DLUA_USE_DLOPEN.) +** If you do not want any kind of dynamic library, undefine all these +** options. +** By default, _WIN32 gets LUA_DL_DLL and MAC OS X gets LUA_DL_DYLD. +*/ +#if defined(LUA_USE_DLOPEN) +#define LUA_DL_DLOPEN +#endif + +#if defined(LUA_WIN) +#define LUA_DL_DLL +#endif + + +/* +@@ LUAI_EXTRASPACE allows you to add user-specific data in a lua_State +@* (the data goes just *before* the lua_State pointer). +** CHANGE (define) this if you really need that. This value must be +** a multiple of the maximum alignment required for your machine. +*/ +#define LUAI_EXTRASPACE 0 + + +/* +@@ luai_userstate* allow user-specific actions on threads. +** CHANGE them if you defined LUAI_EXTRASPACE and need to do something +** extra when a thread is created/deleted/resumed/yielded. +*/ +#define luai_userstateopen(L) ((void)L) +#define luai_userstateclose(L) ((void)L) +#define luai_userstatethread(L,L1) ((void)L) +#define luai_userstatefree(L) ((void)L) +#define luai_userstateresume(L,n) ((void)L) +#define luai_userstateyield(L,n) ((void)L) + + +/* +@@ LUA_INTFRMLEN is the length modifier for integer conversions +@* in 'string.format'. +@@ LUA_INTFRM_T is the integer type correspoding to the previous length +@* modifier. +** CHANGE them if your system supports long long or does not support long. +*/ + +#if defined(LUA_USELONGLONG) + +#define LUA_INTFRMLEN "ll" +#define LUA_INTFRM_T long long + +#else + +#define LUA_INTFRMLEN "l" +#define LUA_INTFRM_T long + +#endif + + + +/* =================================================================== */ + +/* +** Local configuration. You can use this space to add your redefinitions +** without modifying the main part of the file. +*/ + + + +#endif + diff --git a/include/luasandbox/lualib.h b/include/luasandbox/lualib.h new file mode 100644 index 0000000..ae047d9 --- /dev/null +++ b/include/luasandbox/lualib.h @@ -0,0 +1,55 @@ +/* +** $Id: lualib.h,v 1.36.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua standard libraries +** See Copyright Notice in lua.h +*/ + + +#ifndef lualib_h +#define lualib_h + +#include "lua.h" + + +/* Key to file-handle type */ +#define LUA_FILEHANDLE "FILE*" + +#define LUA_BASELIBNAME "" +LUALIB_API int (luaopen_base) (lua_State *L); + +#define LUA_COLIBNAME "coroutine" +LUALIB_API int (luaopen_coroutine) (lua_State *L); + +#define LUA_TABLIBNAME "table" +LUALIB_API int (luaopen_table) (lua_State *L); + +#define LUA_IOLIBNAME "io" +LUALIB_API int (luaopen_io) (lua_State *L); + +#define LUA_OSLIBNAME "os" +LUALIB_API int (luaopen_os) (lua_State *L); + +#define LUA_STRLIBNAME "string" +LUALIB_API int (luaopen_string) (lua_State *L); + +#define LUA_MATHLIBNAME "math" +LUALIB_API int (luaopen_math) (lua_State *L); + +#define LUA_DBLIBNAME "debug" +LUALIB_API int (luaopen_debug) (lua_State *L); + +#define LUA_LOADLIBNAME "package" +LUALIB_API int (luaopen_package) (lua_State *L); + + +/* open all previous libraries */ +LUALIB_API void (luaL_openlibs) (lua_State *L); + + + +#ifndef lua_assert +#define lua_assert(x) ((void)0) +#endif + + +#endif diff --git a/include/luasandbox/test/mu_test.h b/include/luasandbox/test/mu_test.h new file mode 100644 index 0000000..313617e --- /dev/null +++ b/include/luasandbox/test/mu_test.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Unit test macros and global data @file */ + +#ifndef luasandbox_test_mu_test_h_ +#define luasandbox_test_mu_test_h_ + +#include + +#ifdef _WIN32 +#if _MSC_VER < 1900 +#define snprintf _snprintf +#endif +#endif + +#if defined(_MSC_VER) +#define PRIuSIZE "Iu" +#else +#define PRIuSIZE "zu" +#endif + +#define MU_ERR_LEN 1024 + +#define mu_assert(cond, ...) \ +do { \ + if (!(cond)) { \ + int cnt = snprintf(mu_err, MU_ERR_LEN, "line: %d (%s) ", __LINE__, #cond); \ + if (cnt > 0 && cnt < MU_ERR_LEN) { \ + cnt = snprintf(mu_err+cnt, MU_ERR_LEN-cnt, __VA_ARGS__); \ + if (cnt > 0 && cnt < MU_ERR_LEN) { \ + return mu_err; \ + } \ + } \ + mu_err[MU_ERR_LEN - 1] = 0; \ + return mu_err; \ + } \ +} while (0) + +#define mu_assert_rv(rv, fn) \ +do { \ + int result = fn; \ + if (rv != result) { \ + int cnt = snprintf(mu_err, MU_ERR_LEN, "line: %d %s expected: %d " \ + " received: %d", __LINE__, #fn, rv, result); \ + if (cnt > 0 && cnt < MU_ERR_LEN) { \ + return mu_err; \ + } \ + mu_err[MU_ERR_LEN - 1] = 0; \ + return mu_err; \ + } \ +} while (0) + +#define mu_run_test(test) \ +do { \ + char *message = test(); \ + mu_tests_run++; \ + if (message) \ + return message; \ +} while (0) + +int mu_tests_run = 0; +char mu_err[MU_ERR_LEN] = { 0 }; + +#endif diff --git a/include/luasandbox/test/sandbox.h b/include/luasandbox/test/sandbox.h new file mode 100644 index 0000000..8465fa0 --- /dev/null +++ b/include/luasandbox/test/sandbox.h @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Test interface for the generic lua sandbox @file */ + +#ifndef luasandbox_test_sandbox_h_ +#define luasandbox_test_sandbox_h_ + +#include + +#include "../../luasandbox.h" +#include "../error.h" + +#ifdef _WIN32 +#ifdef luasandboxtest_EXPORTS +#define LSB_TEST_EXPORT __declspec(dllexport) +#else +#define LSB_TEST_EXPORT __declspec(dllimport) +#endif +#else +#if __GNUC__ >= 4 +#define LSB_TEST_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_TEST_EXPORT +#endif +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif +#include "../lua.h" + +/** + * Global variable to store the test output + * + */ +LSB_TEST_EXPORT extern const char *lsb_test_output; + +/** + * Global variable storing the length of the current test output string + * + */ +LSB_TEST_EXPORT extern size_t lsb_test_output_len; + +/** + * Generaly purpose stderr logger + * + */ +LSB_TEST_EXPORT extern lsb_logger lsb_test_logger; + +/** + * Function to emulate processing data + * + * @param lsb Pointer to a lua sandbox + * @param tc Test case number to control how the sandbox state is updated + * + * @return LSB_TEST_EXPORT int Status value returned from the sandbox + */ +LSB_TEST_EXPORT int lsb_test_process(lsb_lua_sandbox *lsb, double tc); + +/** + * Function to emulate outputting summary data + * + * @param lsb Pointer to a lua sandbox + * @param tc Test case number to control what data is returned and how the state + * is updated + * + * @return LSB_TEST_EXPORT int 0 on success, 1 on failure + */ +LSB_TEST_EXPORT int lsb_test_report(lsb_lua_sandbox *lsb, double tc); + +/** + * Callback for collecting output, places a pointer to the results in the global + * lsb_test_output variable )not thread safe) + * + * @param lua Lua state + * + * @return LSB_TEST_EXPORT int 0 or lua_error + */ +LSB_TEST_EXPORT int lsb_test_write_output(lua_State *lua); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/heka_message.h b/include/luasandbox/util/heka_message.h new file mode 100644 index 0000000..b6b4aab --- /dev/null +++ b/include/luasandbox/util/heka_message.h @@ -0,0 +1,227 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Heka message representation @file */ + +#ifndef luasandbox_util_heka_message_h_ +#define luasandbox_util_heka_message_h_ + +#include +#include + +#include "input_buffer.h" +#include "output_buffer.h" +#include "string.h" +#include "util.h" + +#define LSB_UUID_SIZE 16 +#define LSB_UUID_STR_SIZE 36 +#define LSB_HDR_FRAME_SIZE 3 +#define LSB_MIN_HDR_SIZE 14 +#define LSB_MAX_HDR_SIZE (255 + LSB_HDR_FRAME_SIZE) + +#define LSB_UUID "Uuid" +#define LSB_TIMESTAMP "Timestamp" +#define LSB_TYPE "Type" +#define LSB_LOGGER "Logger" +#define LSB_SEVERITY "Severity" +#define LSB_PAYLOAD "Payload" +#define LSB_ENV_VERSION "EnvVersion" +#define LSB_PID "Pid" +#define LSB_HOSTNAME "Hostname" +#define LSB_FIELDS "Fields" +#define LSB_RAW "raw" +#define LSB_FRAMED "framed" +#define LSB_SIZE "size" + +typedef enum { + LSB_PB_UUID = 1, + LSB_PB_TIMESTAMP = 2, + LSB_PB_TYPE = 3, + LSB_PB_LOGGER = 4, + LSB_PB_SEVERITY = 5, + LSB_PB_PAYLOAD = 6, + LSB_PB_ENV_VERSION = 7, + LSB_PB_PID = 8, + LSB_PB_HOSTNAME = 9, + LSB_PB_FIELDS = 10 +} lsb_pb_message; + +typedef enum { + LSB_PB_NAME = 1, + LSB_PB_VALUE_TYPE = 2, + LSB_PB_REPRESENTATION = 3, + LSB_PB_VALUE_STRING = 4, + LSB_PB_VALUE_BYTES = 5, + LSB_PB_VALUE_INTEGER = 6, + LSB_PB_VALUE_DOUBLE = 7, + LSB_PB_VALUE_BOOL = 8 +} lsb_pb_field; + +typedef enum { + LSB_PB_STRING = 0, + LSB_PB_BYTES = 1, + LSB_PB_INTEGER = 2, + LSB_PB_DOUBLE = 3, + LSB_PB_BOOL = 4 +} lsb_pb_value_types; + +typedef struct lsb_heka_field +{ + lsb_const_string name; + lsb_const_string representation; + lsb_const_string value; + lsb_pb_value_types value_type; +} lsb_heka_field; + +typedef struct lsb_heka_message +{ + lsb_const_string raw; + lsb_const_string uuid; + lsb_const_string type; + lsb_const_string logger; + lsb_const_string payload; + lsb_const_string env_version; + lsb_const_string hostname; + + lsb_heka_field *fields; + + long long timestamp; + int severity; + int pid; + int fields_len; + int fields_size; +} lsb_heka_message; + +typedef enum { + LSB_READ_NIL, + LSB_READ_NUMERIC, + LSB_READ_STRING, + LSB_READ_BOOL +} lsb_read_type; + +typedef struct { + union + { + lsb_const_string s; + double d; + } u; + lsb_read_type type; +} lsb_read_value; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Zero the structure and allocate memory for at least 'size' fields + * + * @param m Heka message structure + * @param num_fields Preallocated number of fields (must be >0) + * + * @return lsb_err_value NULL on success error message on failure + * + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_init_heka_message(lsb_heka_message *m, int num_fields); + +/** + * Frees the memory allocated for the message fields + * + * @param m Heka message structure + * + */ +LSB_UTIL_EXPORT void lsb_free_heka_message(lsb_heka_message *m); + +/** + * Resets the message headers and fields zeroing the allocated memory + * + * @param m Heka message structure + * + */ +LSB_UTIL_EXPORT void lsb_clear_heka_message(lsb_heka_message *m); + +/** + * Locates a framed Heka message in an input buffer + * + * @param m Heka message structure + * @param ib Input buffer + * @param decode True if the framed message should be protobuf decoded + * @param discarded_bytes Used to track stream corruption + * @param logger Logger structure (can be set to NULL to disable logging) + * + * @return bool True on success + */ +LSB_UTIL_EXPORT bool lsb_find_heka_message(lsb_heka_message *m, + lsb_input_buffer *ib, + bool decode, + size_t *discarded_bytes, + lsb_logger *logger); + +/** + * Decodes an array of bytes into a Heka message. The message structure is + * cleared before decoding. + * + * @param m Heka message structure + * @param buf Protobuf array + * @param len Length of the protobuf array + * @param logger Logger structure (can be set to NULL to disable logging) + * + * @return bool True on success + * + */ +LSB_UTIL_EXPORT bool lsb_decode_heka_message(lsb_heka_message *m, + const char *buf, + size_t len, + lsb_logger *logger); + +/** + * Reads a dynamic field from the Heka message + * + * @param m Heka meassage structure + * @param name Field name + * @param fi Field index + * @param ai Array index into the field + * @param val Value structure to be populated by the read + * + * @return bool True on success + */ +LSB_UTIL_EXPORT bool lsb_read_heka_field(const lsb_heka_message *m, + lsb_const_string *name, + int fi, + int ai, + lsb_read_value *val); + +/** + * Writes a binary UUID to the output buffer + * + * @param ob Pointer to the output data buffer. + * @param uuid Uuid string to output (can be NULL, binary, human readable UUID) + * @param len Length of UUID (16 or 36 anything else will cause a new UUID to be + * created). + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len); + +/** + * Writes the Heka framing header to the specified buffer. + * + * @param buf Buffer to write the header to must be at least LSB_MIN_HDR_SIZE + * size. + * @param len Length of the message to encode into the header + * + * @return LSB_UTIL_EXPORT size_t + */ +LSB_UTIL_EXPORT size_t lsb_write_heka_header(char *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/heka_message_matcher.h b/include/luasandbox/util/heka_message_matcher.h new file mode 100644 index 0000000..b0c74bb --- /dev/null +++ b/include/luasandbox/util/heka_message_matcher.h @@ -0,0 +1,55 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher @file */ + +#ifndef luasandbox_util_heka_message_matcher_h_ +#define luasandbox_util_heka_message_matcher_h_ + +#include + +#include "heka_message.h" + +typedef struct lsb_message_matcher lsb_message_matcher; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Parses a message matcher expression and returns the matcher + * + * @param exp Expression to parse into a matcher + * + * @return lsb_message_matcher* + */ +LSB_UTIL_EXPORT lsb_message_matcher* +lsb_create_message_matcher(const char *exp); + +/** + * Frees all memory associated with a message matcher instance + * + * @param mm Message matcher + */ +LSB_UTIL_EXPORT void lsb_destroy_message_matcher(lsb_message_matcher *mm); + +/** + * Evaluates the message matcher against the provided message + * + * @param mm Message matcher + * @param m Heka message + * + * @return bool True if the message is a match + */ +LSB_UTIL_EXPORT bool +lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/input_buffer.h b/include/luasandbox/util/input_buffer.h new file mode 100644 index 0000000..48d83ef --- /dev/null +++ b/include/luasandbox/util/input_buffer.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream input buffer @file */ + +#ifndef luasandbox_util_input_buffer_h_ +#define luasandbox_util_input_buffer_h_ + +#include +#include + +#include "util.h" + +typedef struct lsb_input_buffer +{ + char *buf; + size_t size; + size_t maxsize; + size_t readpos; + size_t scanpos; + size_t msglen; +} lsb_input_buffer; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Initialize the provided input buffer + * + * @param b Input buffer + * @param max_message_size The maximum message size the buffer will handle + * before erroring (the internal buffer will contain extra space + * for the header) + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size); + +/** + * Frees the memory internally allocated by the buffer and resets the state + * + * @param b Input buffer + */ +LSB_UTIL_EXPORT void lsb_free_input_buffer(lsb_input_buffer *b); + +/** + * Expands the input buffer (if necessary) to accomadate the requested number of + * bytes. The expansion happens in power of two increments up to the maxsize. + * The buffer never shrinks in size. + * + * @param b Input buffer + * @param len The length of the data being added to the buffer + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_expand_input_buffer(lsb_input_buffer *b, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/output_buffer.h b/include/luasandbox/util/output_buffer.h new file mode 100644 index 0000000..6a325f4 --- /dev/null +++ b/include/luasandbox/util/output_buffer.h @@ -0,0 +1,119 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream output buffer @file */ + +#ifndef luasandbox_util_output_buffer_h_ +#define luasandbox_util_output_buffer_h_ + +#include + +#include "util.h" + +#define LSB_OUTPUT_SIZE 1024 + +typedef struct lsb_output_buffer { + char *buf; + size_t maxsize; + size_t size; + size_t pos; +} lsb_output_buffer; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Initialize the provided input buffer + * + * @param b Output buffer + * @param max_message_size The maximum message size the buffer will handle + * before erroring + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size); + +/** + * Frees the memory internally allocated by the buffer and resets the state + * + * @param b Output buffer + */ +LSB_UTIL_EXPORT void lsb_free_output_buffer(lsb_output_buffer *b); + +/** + * Resize the output buffer when more space is needed. + * + * @param b Output buffer to resize. + * @param needed Number of additional bytes needed. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value lsb_expand_output_buffer(lsb_output_buffer *b, + size_t needed); + +/** + * Append a character to the output buffer. + * + * @param b Pointer the b buffer. + * @param ch Character to append to the b. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value lsb_outputc(lsb_output_buffer *b, char ch); + +/** + * Append a formatted string to the output buffer. + * + * @param b Pointer the b buffer. + * @param fmt Printf format specifier. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_outputf(lsb_output_buffer *b, const char *fmt, ...); + +/** + * Append a fixed string to the output buffer. + * + * @param b Pointer the b buffer. + * @param str String to append to the b. + * @param len Length of the string to append + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_outputs(lsb_output_buffer *b, const char *str, size_t len); + +/** + * More efficient output of a double to a string. NaN/Inf check and then calls + * lsb_outputfd. + * + * @param b Pointer the output buffer. + * @param d Double value to convert to a string. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value lsb_outputd(lsb_output_buffer *b, double d); + + +/** + * More efficient output of a double to a string; no NaN or Inf outputs. + * + * @param b Pointer the output buffer. + * @param d Double value to convert to a string. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value lsb_outputfd(lsb_output_buffer *b, double d); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/protobuf.h b/include/luasandbox/util/protobuf.h new file mode 100644 index 0000000..a03ba06 --- /dev/null +++ b/include/luasandbox/util/protobuf.h @@ -0,0 +1,143 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Generic protobuf utility functions @file */ + +#ifndef luasandbox_util_protobuf_h_ +#define luasandbox_util_protobuf_h_ + +#include +#include + +#include "output_buffer.h" +#include "util.h" + +#define LSB_MAX_VARINT_BYTES 10 + +typedef enum { + LSB_PB_WT_VARINT = 0, + LSB_PB_WT_FIXED64 = 1, + LSB_PB_WT_LENGTH = 2, + LSB_PB_WT_SGROUP = 3, + LSB_PB_WT_EGROUP = 4, + LSB_PB_WT_FIXED32 = 5 +} lsb_pb_wire_types; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Extract the tag and wiretype from a protobuf key + * + * @param p Key + * @param tag Tag Id + * @param wiretype Wiretype Id + * + * @return LSB_EXPORT const char* + */ +LSB_UTIL_EXPORT +const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype); + +/** + * Writes a field key (tag id/wire type) to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param wiretype Field wire type. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, + unsigned char wiretype); + +/** + * Reads the varint into the provided variable + * + * @param p Start of buffer + * @param e End of buffer + * @param vi Varint value + * + * @return const char* Position in the buffer after the varint + */ +LSB_UTIL_EXPORT +const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi); + + +/** + * Outputs the varint to an existing buffer + * + * @param buf Pointer to buffer with at least LSB_MAX_VARINT_BYTES available, + * @param i Number to be encoded. + * + * @return int Number of bytes written + */ +LSB_UTIL_EXPORT int lsb_pb_output_varint(char *buf, unsigned long long i); + +/** + * Writes a varint encoded number to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param i Number to be encoded. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i); + +/** + * Writes a bool to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param i Number to be encoded. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value lsb_pb_write_bool(lsb_output_buffer *ob, int i); + +/** + * Writes a double to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param i Double to be encoded. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_pb_write_double(lsb_output_buffer *ob, double i); + +/** + * Writes a string to the output buffer. + * + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param s String to output. + * @param len Length of s. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len); + +/** + * Updates the field length in the output buffer once the size is known, this + * allows for single pass encoding. + * + * @param ob Pointer to the output data buffer. + * @param len_pos Position in the output buffer where the length should be + * written. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_UTIL_EXPORT lsb_err_value +lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/running_stats.h b/include/luasandbox/util/running_stats.h new file mode 100644 index 0000000..af04082 --- /dev/null +++ b/include/luasandbox/util/running_stats.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Calculates the running count, sum, mean, variance, and standard deviation + * @file */ + +#ifndef lsb_util_running_stats_h_ +#define lsb_util_running_stats_h_ + +#include "util.h" + +typedef struct lsb_running_stats +{ + double count; + double mean; + double sum; +} lsb_running_stats; + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Zeros out the stats counters + * + * @param s Stat structure to zero out + */ +LSB_UTIL_EXPORT void lsb_init_running_stats(lsb_running_stats *s); + +/** + * Value to add to the running stats + * + * @param s Stat structure + * @param d Value to add + */ +LSB_UTIL_EXPORT void lsb_update_running_stats(lsb_running_stats *s, double d); + +/** + * Return the standard deviation of the stats + * + * @param s Stat structure + * + * @return double Standard deviation of the stats up to this point + */ +LSB_UTIL_EXPORT double lsb_sd_running_stats(lsb_running_stats *s); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/string.h b/include/luasandbox/util/string.h new file mode 100644 index 0000000..e4e5070 --- /dev/null +++ b/include/luasandbox/util/string.h @@ -0,0 +1,56 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Heka message string @file */ + +#ifndef luasandbox_util_string_h_ +#define luasandbox_util_string_h_ + +#include "util.h" + +typedef struct lsb_const_string +{ + const char *s; + size_t len; +} lsb_const_string; + + +#ifdef __cplusplus +extern "C" +{ +#endif + +/** + * Initializes the struct to zero. + * + * @param s Pointer to the struct + * + */ +LSB_UTIL_EXPORT void lsb_init_const_string(lsb_const_string *s); + + +/** + * Lua string unescape. The returned string is always NUL terminated, but can + * contain other NULs in its body. + * + * @param d Pointer to the destination array where the content is to be + * unescaped. + * @param s C string to be unescaped + * @param dlen The length of the destination array (must be 1 byte larger than + * the source string (for inclusion of the NUL terminator). After + * successful conversion the final length of the escaped string is + * written back to this value as it may not equal strlen(d). + * + * @return char* A pointer to d or NULL on error. + */ +LSB_UTIL_EXPORT +char* lsb_lua_string_unescape(char *d, const char *s, size_t *dlen); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/string_matcher.h b/include/luasandbox/util/string_matcher.h new file mode 100644 index 0000000..22d9642 --- /dev/null +++ b/include/luasandbox/util/string_matcher.h @@ -0,0 +1,50 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** C API for the Lua string pattern matcher @file */ + +#ifndef luasandbox_util_string_matcher_h_ +#define luasandbox_util_string_matcher_h_ + +#include +#include + +#include "util.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Matches a string using a Lua string match pattern + * + * @param s String to match + * @param len Length of the string + * @param p Lua match pattern + * http://www.lua.org/manual/5.1/manual.html#pdf-string.match + * + * @return bool True if the string matches the pattern + */ +LSB_UTIL_EXPORT bool lsb_string_match(const char *s, size_t len, const char *p); + + +/** + * Searches for a string literal within a string + * + * @param s String to search + * @param ls Length of the string + * @param p Literal match string + * @param lp Length of the match string + * + * @return bool True if the string contains the literal + */ +LSB_UTIL_EXPORT bool lsb_string_find(const char *s, size_t ls, const char *p, size_t lp); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox/util/util.h b/include/luasandbox/util/util.h new file mode 100644 index 0000000..bccc66a --- /dev/null +++ b/include/luasandbox/util/util.h @@ -0,0 +1,89 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Shared types and structures @file */ + +#ifndef luasandbox_util_util_h_ +#define luasandbox_util_util_h_ + +#include +#include +#include + +#include "../error.h" + +#ifdef _WIN32 +#ifdef luasandboxutil_EXPORTS +#define LSB_UTIL_EXPORT __declspec(dllexport) +#else +#define LSB_UTIL_EXPORT __declspec(dllimport) +#endif +#else +#if __GNUC__ >= 4 +#define LSB_UTIL_EXPORT __attribute__ ((visibility ("default"))) +#else +#define LSB_UTIL_EXPORT +#endif +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_NULL; +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_OOM; +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_FULL; +LSB_UTIL_EXPORT extern lsb_err_id LSB_ERR_UTIL_PRANGE; + +/** + * Hacker's Delight - Henry S. Warren, Jr. page 48 + * + * @param x + * + * @return size_t Least power of 2 greater than or equal to x + */ +LSB_UTIL_EXPORT size_t lsb_lp2(unsigned long long x); + +/** + * Read a file into a string + * + * @param fn Filename to read + * + * @return char* NULL on failure otherwise a pointer to the file contents (must + * be freed by the caller). + */ +LSB_UTIL_EXPORT char* lsb_read_file(const char *fn); + +/** + * Retrieves the highest resolution timer available converted to nanoseconds + * + * @return unsigned long long + */ +LSB_UTIL_EXPORT unsigned long long lsb_get_time(); + +/** + * Retrieves the highest resolution time since Jan 1, 1970 converted to + * nanoseconds + * + * @return unsigned long long (time_ns) + */ +LSB_UTIL_EXPORT long long lsb_get_timestamp(); + +/** + * Sets the timezone environment variable for the time conversion functions + * + * @param tz Timezone string (if NULL uses UTC) + * + * @return bool True if the environment variable is successfully set + */ +LSB_UTIL_EXPORT bool lsb_set_tz(const char *tz); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox_output.h b/include/luasandbox_output.h new file mode 100644 index 0000000..27851b7 --- /dev/null +++ b/include/luasandbox_output.h @@ -0,0 +1,117 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox output generation/retrieval functions @file */ + +#ifndef luasandbox_output_h_ +#define luasandbox_output_h_ + +#include + +#include "luasandbox.h" + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include "luasandbox/lua.h" + +/** + * Add a output function to the environment table. The environment table must be + * on the top of the stack. This function will receive the userdata and + * lsb_output_buffer struct as pointers on the Lua stack. + * + * lsb_output_buffer* output = (output_data*)lua_touserdata(lua, -1); + * ud_object* ud = (ud_object*)lua_touserdata(lua, -2); + * + * @param lua Pointer the Lua state. + * @param fp Function pointer to the outputter. + * + * @return int Zero on success, non-zero on failure. + */ +LSB_EXPORT void lsb_add_output_function(lua_State *lua, lua_CFunction fp); + +/** + * Utility function to retrieve a user data output function + * + * @param lua + * @param index + * + * @return lua_CFunction + */ +LSB_EXPORT lua_CFunction lsb_get_output_function(lua_State *lua, int index); + +/** + * Add a zero copy function to the environment table. The environment table must + * be on the top of the stack. This function will receive the userdata as a + * pointer on the Lua stack. + * + * ud_object* ud = (ud_object*)lua_touserdata(lua, -1); + * + * @param lua Pointer the Lua state. + * @param fp Function pointer to the zero copy function. + * + * @return int Number of segments (pointer and length for each) + */ +LSB_EXPORT void lsb_add_zero_copy_function(lua_State *lua, lua_CFunction fp); + + +/** + * Utility function to retrieve a user data zero copy function + * + * @param lua + * @param index + * + * @return lua_CFunction + */ +LSB_EXPORT lua_CFunction lsb_get_zero_copy_function(lua_State *lua, int index); + +/** + * Write an array of variables on the Lua stack to the output buffer. + * + * @param lsb Pointer to the sandbox. + * @param start Lua stack index of first variable. + * @param end Lua stack index of the last variable. + * @param append 0 to overwrite the output buffer, 1 to append the output to it + * + */ +LSB_EXPORT void +lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append); + +/** + * Write an array of variables on the Lua stack to the output buffer. After + * adding support for coroutines we need an extra variable to specify the + * correct Lua state. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer the Lua state + * @param start Lua stack index of first variable. + * @param end Lua stack index of the last variable. + * @param append 0 to overwrite the output buffer, 1 to append the output to it + * + */ +LSB_EXPORT void +lsb_output_coroutine(lsb_lua_sandbox *lsb, lua_State *lua, int start, + int end, int append); + +/** + * Retrieve the data in the output buffer and reset the buffer. The returned + * output string will remain valid until additional sandbox output is performed. + * The output should be copied if the application needs to hold onto it. + * + * @param lsb Pointer to the sandbox. + * @param len If len is not NULL, it will be set to the length of the string. + * + * @return const char* Pointer to the output buffer. + */ +LSB_EXPORT const char* lsb_get_output(lsb_lua_sandbox *lsb, size_t *len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/luasandbox_serialize.h b/include/luasandbox_serialize.h new file mode 100644 index 0000000..d7d8209 --- /dev/null +++ b/include/luasandbox_serialize.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua sandbox serialization @file */ + +#ifndef luasandbox_serialize_h_ +#define luasandbox_serialize_h_ + +#include + +#include "luasandbox/util/output_buffer.h" +#include "luasandbox_output.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#include "luasandbox/lua.h" + +/** + * Add a serialization function to the environment table. The environment table + * must be on the top of the stack. This function will receive the userdata, + * fully qualified variable name, and lsb_output_buffer struct as pointers on the + * Lua stack. + * + * lsb_output_buffer* output = (output_data*)lua_touserdata(lua, -1); + * const char *key = (const char*)lua_touserdata(lua, -2); + * ud_object* ud = (ud_object*)lua_touserdata(lua, -3); + * + * @param lua Pointer the Lua state. + * @param fp Function pointer to the serializer. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_EXPORT void +lsb_add_serialize_function(lua_State *lua, lua_CFunction fp); + +/** + * Serializes a binary data to a Lua string. + * + * @param output Pointer the output buffer. + * @param src Pointer to the binary data. + * @param len Length in bytes of the data to output. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_EXPORT lsb_err_value +lsb_serialize_binary(lsb_output_buffer *output, const void *src, size_t len); + +/** + * More efficient serialization of a double to a string + * + * @param output Pointer the output buffer. + * @param d Double value to convert to a string. + * + * @return lsb_err_value NULL on success error message on failure + */ +LSB_EXPORT lsb_err_value +lsb_serialize_double(lsb_output_buffer *output, double d); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/modules/cbufd.lua b/modules/cbufd.lua deleted file mode 100644 index 1424f63..0000000 --- a/modules/cbufd.lua +++ /dev/null @@ -1,49 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Imports -local l = require "lpeg" -l.locale(l) -local tonumber = tonumber - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - ---[[ cbufd grammar -sample input: -{"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]} -1379660520 12075 159901 -1379660280 11837 154880 - -output table: -1 - 1=12075 (number) - 2=159901 (number) - time=1379660520000000000 (number) - -2 - 1=11837 (number) - 2=154880 (number) - time=1379660280000000000 (number) - -header={"time":1379574900,"rows":1440,"columns":2,"seconds_per_row":60,"column_info":[{"name":"Requests","unit":"count","aggregation":"sum"},{"name":"Total_Size","unit":"KiB","aggregation":"sum"}]} ---]] - -local function not_a_number() - return 0/0 -end - -local eol = l.P"\n" -local header = l.Cg((1 - eol)^1, "header") * eol -local timestamp = l.digit^1 / "%0000000000" / tonumber -local sign = l.P"-" -local float = l.digit^1 * "." * l.digit^1 -local nan = l.P"nan" / not_a_number -local number = (sign^-1 * (float + l.digit^1)) / tonumber -+ nan -local row = l.Ct(l.Cg(timestamp, "time") * ("\t" * number)^1 * eol) - -grammar = l.Ct(header * row^1) * -1 - -return M diff --git a/modules/common_log_format.lua b/modules/common_log_format.lua deleted file mode 100644 index 51be364..0000000 --- a/modules/common_log_format.lua +++ /dev/null @@ -1,429 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Imports -local l = require "lpeg" -l.locale(l) -local string = require "string" -local dt = require "date_time" -local ip = require "ip_address" -local tonumber = tonumber -local ipairs = ipairs -local pairs = pairs -local error = error -local type = type - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - ---[[ Nginx access log grammar generation and parsing --]] - -local pct_encoded = l.P"%" * l.xdigit * l.xdigit -local unreserved = l.alnum + l.S"-._~" -local sub_delims = l.S"!$&'()*+,;=" - -local last_literal = l.space -local integer = l.digit^1 / tonumber -local double = l.digit^1 * "." * l.digit^1 / tonumber -local msec_time = double / dt.seconds_to_ns -local host = l.Ct(l.Cg(ip.v4, "value") * l.Cg(l.Cc"ipv4", "representation")) - + l.Ct(l.Cg(ip.v6, "value") * l.Cg(l.Cc"ipv6", "representation")) - + l.Ct(l.Cg((unreserved + pct_encoded + sub_delims)^1, "value") * l.Cg(l.Cc"hostname", "representation")) -local http_status = l.digit * l.digit * l.digit / tonumber - -local nginx_error_levels = l.Cg(( - l.P"debug" / "7" -+ l.P"info" / "6" -+ l.P"notice" / "5" -+ l.P"warn" / "4" -+ l.P"error" / "3" -+ l.P"crit" / "2" -+ l.P"alert" / "1" -+ l.P"emerg" / "0") -/ tonumber, "Severity") - -local nginx_upstream_sep = ", " -local nginx_upstream_gsep = " : " -local nginx_upstream_addr = l.C((1 - l.S(nginx_upstream_sep))^1) -local nginx_upstream_addrs = nginx_upstream_addr * (nginx_upstream_sep * nginx_upstream_addr)^0 -local nginx_upstream_times = double * (nginx_upstream_sep * double)^0 -local nginx_upstream_lengths = integer * (nginx_upstream_sep * integer)^0 -local nginx_upstream_statuses = http_status * (nginx_upstream_sep * http_status)^0 - -local nginx_format_variables = { - --arg_* - --args - binary_remote_addr = l.P(1) * l.P(1) * l.P(1) * l.P(1) - , body_bytes_sent = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - , bytes_sent = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - , connection = integer - , connection_requests = integer - , content_length = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - --content_type - --cookie_* - --document_root - --document_uri - , host = host - , hostname = host - --http_* - , https = l.P"on"^-1 - , is_args = l.P"?"^-1 - --limit_rate - , msec = msec_time - , nginx_version = l.digit^1 * "." * l.digit^1 * "." * l.digit^1 - , pid = integer - , pipe = l.P"p" + "." - , proxy_protocol_addr = host - --query_string - --realpath_root - , remote_addr = host - , remote_port = integer - --remote_user - --request - --request_body - --request_body_file - , request_completion = l.P"OK"^-1 - --request_filename - , request_length = l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"B", "representation")) - , request_method = l.P"GET" + "POST" + "HEAD" + "PUT" + "DELETE" + "OPTIONS" + "TRACE" + "CONNECT" - , request_time = l.Ct(l.Cg(double, "value") * l.Cg(l.Cc"s", "representation")) - --request_uri - , scheme = l.P"https" + "http" - --sent_http_* - , server_addr = host - , server_name = host - , server_port = integer - , server_protocol = l.P"HTTP/" * l.digit^1 * "." * l.digit^1 - , status = http_status - , tcpinfo_rtt = integer - , tcpinfo_rttvar = integer - , tcpinfo_snd_cwnd = integer - , tcpinfo_rcv_space = integer - , time_iso8601 = dt.rfc3339 / dt.time_to_ns - , time_local = dt.clf_timestamp / dt.time_to_ns - --uri - - -- Upstream Module Grammars - -- http://nginx.org/en/docs/http/ngx_http_upstream_module.html#variables - , upstream_addr = l.Ct(nginx_upstream_addrs * (nginx_upstream_gsep * nginx_upstream_addrs)^0) + "-" - , upstream_cache_status = l.P"HIT" + "MISS" + "EXPIRED" + "BYPASS" + "STALE" + "UPDATING" + "REVALIDATED" + "-" - , upstream_cache_last_modified = dt.build_strftime_grammar("%a, %d %b %Y %T GMT") / dt.time_to_ns + "-" - , upstream_response_length = l.Ct(l.Cg(l.Ct(nginx_upstream_lengths * (nginx_upstream_gsep * nginx_upstream_lengths)^0), "value") * l.Cg(l.Cc"B", "representation")) + "-" - , upstream_response_time = l.Ct(l.Cg(l.Ct(nginx_upstream_times * (nginx_upstream_gsep * nginx_upstream_times)^0), "value") * l.Cg(l.Cc"s", "representation")) + "-" - , upstream_status = l.Ct(nginx_upstream_statuses * (nginx_upstream_gsep * nginx_upstream_statuses)^0) + "-" - --upstream_http_* handled by the generic grammar -} - -local function get_string_grammar(s) - if last_literal ~= '"' then - return l.Cg((l.P(1) - last_literal)^0, s) - else - return l.Cg((l.P'\\"' + (l.P(1) - l.P'"'))^0, s) - end -end - -local function nginx_lookup_grammar(var) - local g = nginx_format_variables[var] - if not g then - g = get_string_grammar(var) - elseif var == "time_local" or var == "time_iso8601" or var == "msec" then - g = l.Cg(g, "time") - else - g = l.Cg(g, var) - end - - return g -end - - -local apache_format_variables = { - ["%"] = l.P"%" - , a = l.Cg(host, "remote_addr") - , A = l.Cg(host, "server_addr") - , B = l.Cg(nginx_format_variables["body_bytes_sent"], "body_bytes_sent") - , b = l.Cg(l.Ct(l.Cg(integer + l.P"-" / function () return 0 end, "value") * l.Cg(l.Cc"B", "representation")), "body_bytes_sent") - , D = l.Cg(l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"us", "representation")), "request_time") - , f = "request_filename" - , h = l.Cg(host, "remote_addr") - , H = l.Cg(nginx_format_variables["server_protocol"], "server_protocol") - , k = l.Cg(integer, "connection_requests") - , l = "remote_user" - , L = "request_log_id" - , m = l.Cg(nginx_format_variables["request_method"], "request_method") - , p = l.Cg(integer, "server_port") - , P = l.Cg(integer, "pid") - , q = "query_string" - , r = "request" - , R = "request_handler" - , s = l.Cg(nginx_format_variables["status"], "status") - , t = l.P"[" * l.Cg(nginx_format_variables["time_local"], "time") * "]" - , T = l.Cg(l.Ct(l.Cg(integer, "value") * l.Cg(l.Cc"s", "representation")), "request_time") - , u = "remote_user" - , U = "uri" - , v = l.Cg(host, "server_name") - , V = l.Cg(host, "server_name") - , X = l.Cg(l.S"X+-", "connection_status") - , I = l.Cg(nginx_format_variables["request_length"], "request_length") - , O = l.Cg(nginx_format_variables["request_length"], "response_length") - , S = l.Cg(nginx_format_variables["request_length"], "total_length") -} - -local function port_format(a) - if a == "canonical" or a == "local" then - return l.Cg(integer, "server_port") - elseif a == "remote" then - return l.Cg(integer, "remote_port") - end - error("unknown port format: " .. a) -end - -local function pid_format(a) - if a == "pid" then - return l.Cg(integer, "pid") - elseif a == "tid" then - return l.Cg(integer, "tid") - elseif a == "hextid" then - return l.Cg(l.xdigit^1 / function (n) return tonumber(n, 16) end, "tid") - end - error("unknown pid format: " .. a) -end - -local function time_format(a) - local pos = 1 - if string.find(a, "^end:") then - pos = 5 - elseif string.find(a, "^begin:") then - pos = 7 - end - - if string.find(a, "^sec", pos) then - return l.Cg(l.digit^1 / dt.seconds_to_ns, "time") - elseif string.find(a, "^msec_frac", pos) or string.find(a, "^usec_frac", pos) then - return l.Cg(l.digit^1 / function (t) return tonumber("." .. t) end, "sec_frac") - elseif string.find(a, "^msec", pos) then - return l.Cg(l.digit^1 / function (t) return tonumber(t) * 1e6 end, "time") - elseif string.find(a, "^usec", pos) then - return l.Cg(l.digit^1 / function (t) return tonumber(t) * 1e3 end, "time") - end - - return l.Cg(dt.build_strftime_grammar(a) / dt.time_to_ns, "time") -end - -local apache_format_arguments = { - a = function(a) return apache_format_variables["a"] end - , C = function(a) return "cookie_" .. a end - , e = function(a) return "env_" .. a end - , i = function(a) return "http_" .. string.gsub(string.lower(a), "-", "_") end -- make it compatible with the Nginx naming convention - , n = function(a) return "module_" .. a end - , o = function(a) return "sent_http_" .. string.gsub(string.lower(a), "-", "_") end -- make it compatible with the Nginx naming convention - , p = port_format - , P = pid_format - , t = time_format -} - -local function apache_lookup_variable(var) - local g = apache_format_variables[var] - if not g then - error("unknown variable: " .. var) - elseif type(g) == "string" then - g = get_string_grammar(g) - end - - return g -end - -local function apache_lookup_argument(t) - local f = apache_format_arguments[t.variable] - if not f then - error("unknown variable: " .. t.variable) - end - - local g = f(t.argument) - if not g then - error(string.format("variable: %s invalid arguments: %s", t.variable, t.arguments)) - elseif type(g) == "string" then - g = get_string_grammar(g) - end - - return g -end - - -local function space_grammar() - last_literal = l.space - return l.space^1 -end - -local function literal_grammar(var) - last_literal = string.sub(var, -1) - return l.P(var) -end - - -local escape_chars = {t = "\9", n = "\10", ['"'] = '"', ["\\"] = "\\"} -local function substitute_escape(seq) - seq = string.sub(seq, 2) - local e = escape_chars[seq] - if not e then - e = string.char(tonumber(seq)) - end - return e -end - -local escape_sequence = l.Cs(((l.P"\\" * (l.S'tn"\\' + l.digit^-3)) / substitute_escape + 1)^0) -local function escape_literal_grammar(var) - local literal = l.match(escape_sequence, var) - last_literal = string.sub(literal, -1) - return l.P(literal) -end - ---[[ User Agent normalization -Derived from the Persona user agent parser -https://github.com/mozilla/persona/blob/dev/lib/coarse_user_agent_parser.js -]]-- - --- find the version of the last product in the user agent string -local function ua_basic(ua) - local cur, last = 0, nil - while cur do - cur = ua:find("/", cur+1) - if cur then - last = cur - end - end - - if last then - return tonumber(ua:match("^(%d+)", last+1)) - end -end - --- find the version of the product identified by the keyword -local function ua_keyword(kw) - return function(ua) - local i = ua:find(kw) - if i then - local s = i + kw:len() - if s then - return tonumber(ua:match("^(%d+)", s)) - end - end - end -end - -local ua_os_matchers = { - -- search, replace - {"iPod" ,"iPod" } - , {"iPad" ,"iPad" } - , {"iPhone" ,"iPhone" } - , {"Android" ,"Android" } - , {"BlackBerry" ,"BlackBerry" } - , {"Linux" ,"Linux" } - , {"Macintosh" ,"Macintosh" } - , {"Mozilla/5.0 (Mobile;" ,"FirefoxOS" } - --http://en.wikipedia.org/wiki/Microsoft_Windows#Timeline_of_releases - , {"Windows NT 6.3" ,"Windows 8.1" } - , {"Windows NT 6.2" ,"Windows 8" } - , {"Windows NT 6.1" ,"Windows 7" } - , {"Windows NT 6.0" ,"Windows Vista"} - , {"Windows NT 5.1" ,"Windows XP" } - , {"Windows NT 5.0" ,"Windows 2000" } -} - -local ua_browser_matchers = { - {"Chrome" , ua_keyword("Chrome/")} - , {"Opera Mini" , ua_basic} - , {"Opera Mobi" , ua_basic} - , {"Opera" , ua_basic} - , {"MSIE" , ua_keyword("MSIE ")} - , {"Safari" , ua_basic} - , {"Firefox" , ua_keyword("Firefox/")} -} - ---[[ Public Interface --]] - --- Returns an LPeg grammar based on the Nginx log_format configuration string. -function build_nginx_grammar(log_format) - last_literal = l.space - local ws = l.space^1 / space_grammar - local variable = l.P"$" * ((l.alnum + "_")^1 / nginx_lookup_grammar) - local literal = (l.P(1) - (ws + variable))^1 / literal_grammar - local item = ws + variable + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(log_format) - if not t then - error("could not parse the log_format configuration") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - --- Returns the user agent browser, version, and os -function normalize_user_agent(ua) - if not ua then return end - - local browser, version, os - for i, v in ipairs(ua_os_matchers) do - if ua:find(v[1], 1, true) then - os = v[2] - break - end - end - - for i, v in ipairs(ua_browser_matchers) do - if ua:find(v[1], 1, true) then - browser = v[1] - version = v[2](ua) - break - end - end - - return browser, version, os -end - --- Returns an LPeg grammar based on the Apache LogFormat configuration string. -function build_apache_grammar(log_format) - last_literal = l.space - local ws = l.space^1 / space_grammar - local http_status_code = l.digit * l.digit * l.digit - local http_status_codes = l.P"!"^-1 * http_status_code * ("," * http_status_code)^0 - local variable = l.alpha + "%" - local argument = "{" * l.Cg((l.P(1) - "}")^1, "argument") * "}" - local directive = l.P"%" * l.S"<>"^-1 * (variable / apache_lookup_variable) - + "%" * http_status_codes^-1 * l.Ct(argument * l.Cg(variable, "variable")) / apache_lookup_argument - local literal = (l.P(1) - (ws + directive))^1 / escape_literal_grammar - local item = ws + directive + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(log_format) - if not t then - error("could not parse the log_format configuration") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - -nginx_error_grammar = l.Ct(l.Cg(dt.build_strftime_grammar("%Y/%m/%d %T") / dt.time_to_ns, "time") - * l.space * "[" * nginx_error_levels * "]" - * l.space * l.Cg(l.digit^1 / tonumber, "Pid") * "#" - * l.Cg(l.Ct(l.Cg(l.digit^1 / tonumber, "tid") * ": " * (l.Cg(l.digit^1 / tonumber, "connection") * " ")^-1), "Fields") - * l.Cg(l.P(1)^0, "Payload")) - -return M diff --git a/modules/date_time.lua b/modules/date_time.lua deleted file mode 100644 index 3973d0a..0000000 --- a/modules/date_time.lua +++ /dev/null @@ -1,258 +0,0 @@ --- Imports -local l = require "lpeg" -l.locale(l) -local os = require "os" -local string = require "string" -local tonumber = tonumber -local ipairs = ipairs -local error = error - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - ---[[ Utility Functions --]] --- Converts a time table into the number of nanoseconds since the UNIX epoch -function time_to_ns(t) - if not t then return 0 end - if t.time_t then return t.time_t * 1e9 end - - local offset = 0 - if t.offset_hour then - offset = (t.offset_hour * 60 * 60) + (t.offset_min * 60) - if t.offset_sign == "+" then offset = offset * -1 end - end - - if t.period then - t.hour = tonumber(t.hour) - if t.period == "AM" then - if t.hour == 12 then - t.hour = 0 - end - elseif t.period == "PM" then - if t.hour < 12 then - t.hour = t.hour + 12 - end - end - end - - local frac = 0 - if t.sec_frac then - frac = t.sec_frac - end - - local ost = os.time(t) - if not ost then - return 0 - end - - return (ost + frac + offset) * 1e9 -end - ---Converts a second value into the number of nanoseconds since the UNIX epoch -function seconds_to_ns(sec) - return tonumber(sec) * 1e9 -end - - ---[[ Generic Grammars --]] -date_fullyear = l.Cg(l.digit * l.digit * l.digit * l.digit, "year") - -date_month = l.Cg(l.P"0" * l.R"19" - + "1" * l.R"02", "month") -date_mabbr = l.Cg( - l.P"Jan" /"1" - + l.P"Feb" /"2" - + l.P"Mar" /"3" - + l.P"Apr" /"4" - + l.P"May" /"5" - + l.P"Jun" /"6" - + l.P"Jul" /"7" - + l.P"Aug" /"8" - + l.P"Sep" /"9" - + l.P"Oct" /"10" - + l.P"Nov" /"11" - + l.P"Dec" /"12" - , "month") - -date_mfull = l.Cg( - l.P"January" /"1" - + l.P"February" /"2" - + l.P"March" /"3" - + l.P"April" /"4" - + l.P"May" /"5" - + l.P"June" /"6" - + l.P"July" /"7" - + l.P"August" /"8" - + l.P"September" /"9" - + l.P"October" /"10" - + l.P"November" /"11" - + l.P"December" /"12" - , "month") - -date_mday = l.Cg(l.P"0" * l.R"19" - + l.R"12" * l.R"09" - + "3" * l.R"01", "day") - -date_mday_sp = l.Cg(l.P" " * l.R"19" - + l.S"12" * l.digit - + "3" * l.S"01", "day") - -time_hour = l.Cg(l.R"01" * l.digit - + "2" * l.R"03", "hour") - -time_minute = l.Cg(l.R"05" * l.digit, "min") - -time_second = l.Cg(l.R"05" * l.digit - + "60", "sec") -- include leap second - -time_secfrac = l.Cg(l.P"." * l.digit^1 / tonumber, "sec_frac") - -timezone = - l.P"UTC" * l.Cg(l.Cc"+", "offset_sign") * l.Cg(l.Cc"00" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"PST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"08" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"PDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"07" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"MST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"07" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"MDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"06" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"CST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"06" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"CDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"05" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"EST" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"05" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.P"EDT" * l.Cg(l.Cc"-", "offset_sign") * l.Cg(l.Cc"04" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - + l.alpha^-5 * l.Cg(l.Cc"+", "offset_sign") * l.Cg(l.Cc"00" / tonumber, "offset_hour") * l.Cg(l.Cc"00" / tonumber, "offset_min") - -timezone_offset = l.Cg(l.S"+-", "offset_sign") * l.Cg(time_hour / tonumber, "offset_hour") * l.Cg(time_minute / tonumber, "offset_min") - ---[[ RFC3339 grammar -sample input: 1999-05-05T23:23:59.217-07:00 - -output table: -hour=23 (string) -min=23 (string) -year=1999 (string) -month=05 (string) -day=05 (string) -sec=59 (string) -*** conditional table members *** -sec_frac=0.217 (number) -offset_sign=- (string) -offset_hour=7 (number) -offset_min=0 (number) ---]] -rfc3339_time_numoffset = l.Cg(l.S"+-", "offset_sign") -* l.Cg(time_hour / tonumber, "offset_hour") -* ":" -* l.Cg(time_minute / tonumber, "offset_min") - -rfc3339_time_offset = l.P"Z" + rfc3339_time_numoffset -rfc3339_partial_time = time_hour * ":" * time_minute * ":" * time_second * time_secfrac^-1 -rfc3339_full_date = date_fullyear * "-" * date_month * "-" * date_mday -rfc3339_full_time = rfc3339_partial_time * rfc3339_time_offset -rfc3339 = l.Ct(rfc3339_full_date * "T" * rfc3339_full_time) - ---[[ strftime Grammars --]] -local century = os.date("%Y"):sub(1,2) -local strftime_specifiers = {} -strftime_specifiers["a"] = l.P"Mon" + "Tue" + "Wed" + "Thu" + "Fri" + "Sat" + "Sun" -strftime_specifiers["A"] = l.P"Monday" + "Tuesday" + "Wednesday" + "Thursday" + "Friday" + "Saturday" + "Sunday" -strftime_specifiers["b"] = date_mabbr -strftime_specifiers["B"] = date_mfull -strftime_specifiers["C"] = l.digit * l.digit -strftime_specifiers["y"] = l.Cg((l.digit * l.digit) / function (yy) return century .. yy end, "year") -strftime_specifiers["d"] = date_mday -strftime_specifiers["D"] = date_month * "/" * date_mday * "/" * strftime_specifiers["y"] -strftime_specifiers["e"] = date_mday_sp -strftime_specifiers["F"] = date_fullyear * "-" * date_month * "-" * date_mday -strftime_specifiers["g"] = strftime_specifiers["y"] -strftime_specifiers["G"] = date_fullyear -strftime_specifiers["h"] = date_mabbr -strftime_specifiers["H"] = time_hour -strftime_specifiers["I"] = l.Cg(l.P"0" * l.digit - + "1" * l.R"02", "hour") -strftime_specifiers["j"] = l.P"00" * l.R"19" - + "0" * l.digit * l.digit - + l.S"12" * l.digit * l.digit - + "3" * l.R"05" * l.digit - + "36" * l.R"06" -strftime_specifiers["k"] = l.Cg(l.S" 1" * l.digit - + "2" * l.R"03", "hour") -strftime_specifiers["l"] = l.Cg(l.space * l.digit - + "1" * l.R"02", "hour") -strftime_specifiers["m"] = date_month -strftime_specifiers["M"] = time_minute -strftime_specifiers["n"] = l.P"\n" -strftime_specifiers["p"] = l.Cg(l.P"AM" + "PM", "period") -strftime_specifiers["r"] = strftime_specifiers["I"] * ":" * time_minute * ":" * time_second * " " * strftime_specifiers["p"] -strftime_specifiers["R"] = time_hour * ":" * time_minute -strftime_specifiers["s"] = l.Cg(l.digit^1 / tonumber, "time_t") -strftime_specifiers["S"] = l.Cg(l.R"05" * l.digit - + "6" * l.S"01", "sec") -strftime_specifiers["t"] = l.P"\t" -strftime_specifiers["T"] = time_hour * ":" * time_minute * ":" * time_second -strftime_specifiers["u"] = l.R"17" -strftime_specifiers["U"] = l.R"04" * l.digit - + l.P"5" * l.R"03" -strftime_specifiers["V"] = strftime_specifiers["U"] -strftime_specifiers["w"] = l.R"06" -strftime_specifiers["W"] = strftime_specifiers["U"] -strftime_specifiers["x"] = strftime_specifiers["D"] -strftime_specifiers["X"] = strftime_specifiers["T"] -strftime_specifiers["Y"] = date_fullyear -strftime_specifiers["z"] = timezone_offset -strftime_specifiers["Z"] = timezone -strftime_specifiers["%"] = l.P"%" -if os.date("%c"):find("^%d") then -- windows - strftime_specifiers["c"] = strftime_specifiers["D"] * " " * strftime_specifiers["T"] - strftime_specifiers["z"] = timezone -- todo depending on the registry this could be the full name and not the abbreviation -else - strftime_specifiers["c"] = strftime_specifiers["a"] * " " * date_mabbr * " " * date_mday_sp * " " * strftime_specifiers["T"] * " " * date_fullyear -end - -local function strftime_lookup_grammar(var) - local g = strftime_specifiers[var] - if not g then - error("unknown strftime specifier: " .. var) - end - return g -end - --- Returns an LPeg grammar based on the strftime format. -function build_strftime_grammar(format) - local ws = l.space / function () return l.space end - local variable = l.P"%" * ((l.alnum + "%") / strftime_lookup_grammar) - local literal = (l.P(1) - (ws + variable))^1 / function (lit) return l.P(lit) end - local item = ws + variable + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(format) - if not t then - error("could not parse the strftime format") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - - ---[[ Common Log Format Grammars --]] --- strftime format %d/%b/%Y:%H:%M:%S %z. -clf_timestamp = l.Ct(date_mday * "/" * date_mabbr * "/" * date_fullyear * ":" * strftime_specifiers["T"] * " " * timezone_offset) - - ---[[ RFC3164 Grammars --]] -local sp = l.P" " --- since we consume multiple spaces this will also handle date-rfc3164-buggyday -rfc3164_timestamp = l.Ct(date_mabbr * sp^1 * l.Cg(l.digit^-2, "day") * sp^1 * strftime_specifiers["T"] * l.Cg(l.Cc("") / function() return os.date("%Y") end, "year")) - - ---[[ Database Grammars --]] -mysql_timestamp = l.Ct(date_fullyear * date_month * date_mday * time_hour * time_minute * time_second) - -pgsql_timestamp = l.Ct(rfc3339_full_date * sp * strftime_specifiers["T"] ) - -return M diff --git a/modules/ip_address.lua b/modules/ip_address.lua deleted file mode 100644 index c4c9503..0000000 --- a/modules/ip_address.lua +++ /dev/null @@ -1,35 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Imports -local l = require "lpeg" -l.locale(l) - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local d8 = "1" * l.digit * l.digit - + "2" * l.R"04" * l.digit - + "25" * l.R"05" - + l.R"19" * l.digit - + l.digit -v4 = d8 * "." * d8 * "." * d8 * "." * d8 - - -local h16 = l.xdigit * l.xdigit^-3 -local hg = h16 * ":" -local ls32 = hg * h16 - + v4 - -v6 = hg * hg * hg * hg * hg * hg * ls32 -+ "::" * hg * hg * hg * hg * hg * ls32 -+ h16^-1 * "::" * hg * hg * hg * hg * ls32 -+ (hg * hg^-1 + ":") * ":" * hg * hg * hg * ls32 -+ (hg * hg^-2 + ":") * ":" * hg * hg * ls32 -+ (hg * hg^-3 + ":") * ":" * hg * ls32 -+ (hg * hg^-4 + ":") * ":" * ls32 -+ (hg * hg^-5 + ":") * ":" * h16 -+ (hg * hg^-6 + ":") * ":" - -return M diff --git a/modules/mysql.lua b/modules/mysql.lua deleted file mode 100644 index c9c3081..0000000 --- a/modules/mysql.lua +++ /dev/null @@ -1,62 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Imports -local l = require "lpeg" -l.locale(l) -local tonumber = tonumber - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - -local space = l.space^1 -local sep = l.P"\n" -local sql_end = l.P";" * (l.P"\n" + -1) -local line = (l.P(1) - sep)^0 * sep -local float = l.digit^1 * "." * l.digit^1 - -local time = l.P"# Time: " * line - -local user_name = l.alpha^1 * "[" * l.alpha^1 * "]" -local host_name = l.alpha^0 * l.space^0 * "[" * l.Cg((l.P(1) - "]")^1, "Hostname") * "]" -local user = l.P"# User@Host: " * user_name * space * "@" * space * host_name * sep - -local query_time = l.P"# Query_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Query_time") -local lock_time = l.P"Lock_time: " * l.Cg(l.Ct(l.Cg(float / tonumber, "value") * l.Cg(l.Cc"s", "representation")), "Lock_time") -local rows_sent = l.P"Rows_sent: " * l.Cg(l.digit^1 / tonumber, "Rows_sent") -local rows_examined = l.P"Rows_examined: " * l.Cg(l.digit^1 / tonumber, "Rows_examined") -local query = query_time * space * lock_time * space * rows_sent * space * rows_examined * sep - -local use_db = l.P"use " * line - -local last_insert = l.P"last_insert_id=" * l.digit^1 * "," -local insert = l.P"insert_id=" * l.digit^1 * "," -local timestamp = l.P"timestamp=" * l.Cg((l.digit^1 / "%0000000000"), "Timestamp") -local set = l.P"SET " * last_insert^0 * insert^0 * timestamp * ";" * sep - -local admin = l.P"# administrator command: " * line - -local sql = l.Cg((l.P(1) - sql_end)^0 * sql_end, "Payload") - --- Maria DB extensions -local yes_no = l.C(l.P"Yes" + "No") -local thread_id = l.P"# Thread_id: " * l.Cg(l.digit^1 / tonumber, "Thread_id") - * l.P" Schema: " * l.Cg(l.alnum^0, "Schema") - * l.P" QC_hit: " * l.Cg(yes_no, "QC_hit") * sep - -local full_scan = l.P"# Full_scan: " * l.Cg(yes_no, "Full_scan") - * l.P" Full_join: " * l.Cg(yes_no, "Full_join") - * l.P" Tmp_table: " * l.Cg(yes_no, "Tmp_table") - * l.P" Tmp_table_on_disk: " * l.Cg(yes_no, "Tmp_table_on_disk") * sep - * l.P"# Filesort: " * l.Cg(yes_no, "Filesort") - * " Filesort_on_disk: " * l.Cg(yes_no, "Filesort_on_disk") - * " Merge_passes: " * l.Cg(l.digit^1 / tonumber, "Merge_passes") * sep - -slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) -mariadb_slow_query_grammar = l.Ct(time^0 * user * l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) - -short_slow_query_grammar = l.Ct(l.Cg(l.Ct(query), "Fields") * use_db^0 * set * admin^0 * sql) -mariadb_short_slow_query_grammar= l.Ct(l.Cg(l.Ct(thread_id * query * full_scan^0), "Fields") * use_db^0 * set * admin^0 * sql) - -return M diff --git a/modules/syslog.lua b/modules/syslog.lua deleted file mode 100644 index 9e986dc..0000000 --- a/modules/syslog.lua +++ /dev/null @@ -1,250 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Imports -local l = require "lpeg" -l.locale(l) -local math = require "math" -local string = require "string" -local dt = require "date_time" -local ip = require "ip_address" -local tonumber = tonumber -local error = error -local type = type -local ipairs = ipairs - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - --- http://tools.ietf.org/html/rfc5424#page-8 -local octet = l.P(1) -local utf8_string = octet^0 -local sp = l.P" " -local printusascii = l.R"!~" -local nonzero_digit = l.R"19" -local digit = l.R"09" -local nilvalue = l.P"-" -local bom = l.P("\239\187\191") -local msg_any = octet^0 -local msg_utf8 = bom * utf8_string -local msg = msg_utf8 + msg_any -local hostname = nilvalue + printusascii^-255 -local sd_name = printusascii^-32 -local param_name = sd_name -local param_value = utf8_string -local sd_param = param_name * '="' * param_value * '"' -local sd_id = sd_name -local sd_element = l.P"[" * sd_id * (sp * sd_param)^0 * "]" -local syslog_facility = digit^-3 / tonumber -local syslog_severity = digit / tonumber - -local function convert_pri(pri) - pri = tonumber(pri) - local facility = math.floor(pri/8) - local severity = pri % 8 - - return {facility = facility, severity = severity} -end -local pri = digit^-3 / convert_pri - --- https://github.com/rsyslog/rsyslog/blob/35ef2408dfec0e8abdebd33c74578f9bb3299f20/runtime/msg.c#L298 -local syslog_severity_text = ( - (l.P"debug" + "DEBUG") / "7" -+ (l.P"info" + "INFO") / "6" -+ (l.P"notice" + "NOTICE") / "5" -+ (l.P"warning" + "WARNING") / "4" -+ (l.P"warn" + "WARN") / "4" -+ (l.P"error" + "ERROR") / "3" -+ (l.P"err" + "ERR") / "3" -+ (l.P"crit" + "CRIT") / "2" -+ (l.P"alert" + "ALERT") / "1" -+ (l.P"emerg" + "EMERG") / "0" -+ (l.P"panic" + "PANIC") / "0" -) / tonumber - -local syslog_facility_text = ( - (l.P"kern" + "KERN") / "0" -+ (l.P"user" + "USER") / "1" -+ (l.P"mail" + "MAIL") / "2" -+ (l.P"daemon" + "DAEMON") / "3" -+ (l.P"auth" + "AUTH") / "4" -+ (l.P"security" + "SECURITY") / "4" -+ (l.P"syslog" + "SYSLOG") / "5" -+ (l.P"lpr" + "LPR") / "6" -+ (l.P"news" + "NEWS") / "7" -+ (l.P"uucp" + "UUCP") / "8" -+ (l.P"cron" + "CRON") / "9" -+ (l.P"authpriv" + "AUTHPRIV") / "10" -+ (l.P"ftp" + "FTP") / "11" -+ (l.P"ntp" + "NTP") / "12" -+ (l.P"audit" + "AUDIT") / "13" -+ (l.P"alert" + "ALERT") / "14" -+ (l.P"clock" + "CLOCK") / "15" -+ (l.P"local0" + "LOCAL0") / "16" -+ (l.P"local1" + "LOCAL1") / "17" -+ (l.P"local2" + "LOCAL2") / "18" -+ (l.P"local3" + "LOCAL3") / "19" -+ (l.P"local4" + "LOCAL4") / "20" -+ (l.P"local5" + "LOCAL5") / "21" -+ (l.P"local6" + "LOCAL6") / "22" -+ (l.P"local7" + "LOCAL7") / "23" -) / tonumber - -local time_formats = { -["date-rfc3164"] = dt.rfc3164_timestamp, -["date-rfc3164-buggyday"] = dt.rfc3164_timestamp, -["date-mysql"] = dt.mysql_timestamp, -["date-pgsql"] = dt.pgsql_timestamp, -["date-rfc3339"] = dt.rfc3339, -["date-unixtimestamp"] = digit^1, -["date-subseconds"] = digit^1 -} - -local function lookup_time_format(property) - local f - if property.options then - local format = string.lower(property.options) - f = time_formats[format] - end - if property.name == "timereported" then - property.name = "timestamp" - end - - if not f then - f = time_formats["date-rfc3164"] - end - - if format == "date-unixtimestamp" then - f = f / dt.seconds_to_ns - elseif format == "date-subseconds" then - f = f / tonumber - else - f = f / dt.time_to_ns - end - - return f -end - --- http://rsyslog-5-8-6-doc.neocities.org/property_replacer.html -local programname = (printusascii - l.S" :[")^0 -local rsyslog_properties = { - -- special case msg since the rsyslog template can break the rfc5424 msg rules - rawmsg = octet^0, - hostname = hostname, - source = hostname, - fromhost = hostname, - ["fromhost-ip"] = ip.v4 + ip.v6, - syslogtag = l.Ct(l.Cg(programname, "programname") * ("[" * l.Cg(l.digit^1 / tonumber, "pid") * "]")^-1 * l.P":"^-1), - programname = programname, - pri = pri, -- pri table with facility and severity keys - ["pri-text"] = syslog_facility_text * "." * syslog_severity_text * "<" * pri * ">", - iut = l.digit^1, - syslogfacility = syslog_facility, - ["syslogfacility-text"] = syslog_facility_text, - syslogseverity = syslog_severity, - ["syslogseverity-text"] = syslog_severity_text, - syslogpriority = syslog_severity, - ["syslogpriority-text"] = syslog_severity_text, - timegenerated = lookup_time_format, - timereported = lookup_time_format, - timestamp = lookup_time_format, - ["protocol-version"] = digit^1, - ["structured-data"] = nilvalue + sd_element, - ["app-name"] = nilvalue + printusascii^-48, - procid = nilvalue + printusascii^-128, - msgid = nilvalue + printusascii^-32, - inputname = printusascii^0, -- doesn't appear to generate output (don't use it) - ["$bom"] = bom, - ["$now"] = dt.rfc3339_full_date, - ["$year"] = dt.date_fullyear, - ["$month"] = dt.date_month, - ["$day"] = dt.date_mday, - ["$hour"] = dt.time_hour, - ["$hhour"] = l.P"0" * l.S"01", - ["$qhour"] = l.P"0" * l.R"03", - ["$minute"] = dt.time_minute -} - -local function space_grammar() - return l.space -end - -local last_literal = l.P"\n" -local function rsyslog_lookup(property) - if property.options and property.options == "sp-if-no-1st-sp" then - return sp^1 - end - - property.name = string.lower(property.name) - local g - if property.name == "msg" then - g = l.Cg((l.P(1) - last_literal)^0, property.name) -- todo account for an escaped literal - else - g = rsyslog_properties[property.name] - end - - if not g then - error(string.format("invalid rsyslog property: '%s'", property.name)) - end - - if type(g) == "function" then - g = g(property) - end - last_literal = l.P"\n" - return l.Cg(g, property.name) -end - -local escape_chars = {t = "\9", n = "\10", r = "\13", ['"'] = '"', ["'"] = "'" - , ["\\"] = "\\", ["%"] = "%", a = "\7", b = "\8", f = "\12", v = "\11"} -local function substitute_escape(seq) - seq = string.sub(seq, 2) - local e = escape_chars[seq] - if not e then - e = string.char(tonumber(seq)) - end - return e -end -local escape_sequence = l.Cs(((l.P"\\" * (l.S'tnr"\'\\%abfv' + digit^-3)) / substitute_escape + 1)^0) -local function literal_grammar(var) - local literal = l.match(escape_sequence, var) - last_literal = string.sub(literal, -1) - return l.P(literal) -end - --- --- Public Interface --- - --- http://rsyslog-5-8-6-doc.neocities.org/rsyslog_conf_templates.html -function build_rsyslog_grammar(template) - local ws = l.space / space_grammar - local options = l.P":" * l.Cg((1 - l.P"%")^0, "options") -- todo support multiple options - local tochar = l.Cg((1 - l.S":%")^0, "tochar") - local fromchar = l.Cg((1 - l.P":")^0, "fromchar") - local substr = l.P":" * fromchar * ":" * tochar - local propname = l.Cg((l.alnum + l.S"-$")^1, "name") - local property = l.P"%" * l.Ct(propname * substr^-1 * options^-1) / rsyslog_lookup * l.P"%" - local literal = (l.P(1) - (ws + property))^1 / literal_grammar - local item = ws + property + literal - - local p = l.Ct(item * (item)^0) - local t = p:match(template) - if not t then - error("could not parse the rsyslog template") - end - - local grammar = nil - for i,v in ipairs(t) do - if not grammar then - grammar = v - else - grammar = grammar * v - end - end - return l.Ct(grammar) -end - -severity = syslog_severity_text - -return M diff --git a/modules/util.lua b/modules/util.lua deleted file mode 100644 index bed263c..0000000 --- a/modules/util.lua +++ /dev/null @@ -1,31 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - --- Imports -local pairs = pairs -local type = type -local string = require "string" - -local M = {} -setfenv(1, M) -- Remove external access to contain everything in the module - --- flattens a Lua table so it can be encoded as a protobuf fields object -function table_to_fields(t, fields, parent) - for k,v in pairs(t) do - if parent then - full_key = string.format("%s.%s", parent, k) - else - full_key = k - end - if type(v) == "table" then - table_to_fields(v, fields, full_key) - else - if type(v) ~= "userdata" then - fields[full_key] = v - end - end - end -end - -return M diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0cca751..beff13e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -2,67 +2,70 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -set(LUA_SANDBOX_SRC -lua_sandbox.c -lua_sandbox_private.c -lua_serialize.c -lua_serialize_protobuf.c -lua_circular_buffer.c -lua_bloom_filter.c -lua_hyperloglog.c -cephes.c -xxhash.c -redis_hyperloglog.c -struct.c -) - if(MSVC) - add_definitions(-D_CRT_SECURE_NO_WARNINGS) - set(LUA_SANDBOX_LIBS - "${EP_BASE}/lib/lua.lib" - "${EP_BASE}/lib/lpeg.lib" - "${EP_BASE}/lib/cjson.lib" - ) - add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.dll") - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.lib") -elseif(MINGW) - add_definitions(-D_MINGW) - set(LUA_SANDBOX_LIBS - "${EP_BASE}/lib/liblua.dll" - "${EP_BASE}/lib/liblpeg.dll" - "${EP_BASE}/lib/libcjson.dll" - ) - add_library(luasandbox SHARED ${LUA_SANDBOX_SRC}) - set_target_properties(luasandbox PROPERTIES LINK_FLAGS -s) - if (ADDRESS_MODEL EQUAL 32) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32") - set_target_properties(luasandbox PROPERTIES LINK_FLAGS "-s -m32") - endif() - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.dll") + add_definitions( + -DLUA_BUILD_AS_DLL + -D_CRT_SECURE_NO_WARNINGS + ) else() - if (LUA_JIT) - if(NOT CMAKE_SYSTEM_NAME MATCHES "FreeBSD") - set(LINK_DL "-ldl") - endif() - endif() - - set(LUA_SANDBOX_LIBS - "${EP_BASE}/lib/liblua.a" - "${EP_BASE}/lib/liblpeg.a" - "${EP_BASE}/lib/libcjson.a" - ${LINK_DL} -lm - ) - add_library(luasandbox STATIC ${LUA_SANDBOX_SRC}) - install(DIRECTORY "${EP_BASE}/lib/" DESTINATION lib FILES_MATCHING PATTERN "*.a") - install(DIRECTORY "${EP_BASE}/include/" DESTINATION include) + add_definitions( + -DLUA_USE_POSIX + -DLUA_USE_DLOPEN + -DLUA_USE_STRTODHEX + -DLUA_USE_LONGLONG + -DLUA_USE_GMTIME_R + ) endif() -target_link_libraries(luasandbox ${LUA_SANDBOX_LIBS}) +set(LUA_SRC +lua/lapi.c +lua/lauxlib.c +lua/lbaselib.c +lua/lcode.c +lua/ldblib.c +lua/ldebug.c +lua/ldo.c +lua/ldump.c +lua/lfunc.c +lua/lgc.c +lua/linit.c +lua/liolib.c +lua/llex.c +lua/lmathlib.c +lua/lmem.c +lua/loadlib.c +lua/lobject.c +lua/lopcodes.c +lua/loslib.c +lua/lparser.c +lua/lstate.c +lua/lstring.c +lua/lstrlib.c +lua/ltable.c +lua/ltablib.c +lua/ltm.c +lua/lundump.c +lua/lvm.c +lua/lzio.c +) + +set(LUA_SANDBOX_SRC +luasandbox.c +luasandbox_output.c +luasandbox_serialize.c +) -add_dependencies(luasandbox ${LUA_PROJECT} lpeg-0_12 lua-cjson-2_1_0) -install(TARGETS luasandbox DESTINATION lib) -install(DIRECTORY "${CMAKE_SOURCE_DIR}/include/" DESTINATION include) -install(DIRECTORY "${LUA_INCLUDE_DIR}/" DESTINATION include) +add_library(luasandbox SHARED ${LUA_SANDBOX_SRC} ${LUA_SRC}) +set_target_properties(luasandbox PROPERTIES VERSION ${PROJECT_VERSION} SOVERSION 0) +target_link_libraries(luasandbox luasandboxutil ${CMAKE_DL_LIBS}) +if(LIBM_LIBRARY) + target_link_libraries(luasandbox ${LIBM_LIBRARY}) +endif() +install(TARGETS luasandbox DESTINATION ${CMAKE_INSTALL_LIBDIR}) +add_subdirectory(util) +add_subdirectory(heka) +if(NOT WIN32) # todo need to add getopt support for Windows + add_subdirectory(cli) +endif() add_subdirectory(test) diff --git a/src/cephes.c b/src/cephes.c deleted file mode 100644 index 9cfabdc..0000000 --- a/src/cephes.c +++ /dev/null @@ -1,33 +0,0 @@ -/* -* Cephes Math Library Release 2.2: June, 1992 -* Copyright 1984, 1987, 1988, 1992 by Stephen L. Moshier -* Direct inquiries to 30 Frost Street, Cambridge, MA 02140 -*/ - -/* extracted the needed function from ntdr.c */ - -#include - -const double SQRTH = 0.70710678118654752440; /* sqrt(2)/2 */ - -double ndtr(double a) -{ - double x, y, z; - - if (isnan(a)) { - return a; - } - - x = a * SQRTH; - z = fabs(x); - - if (z < SQRTH) y = 0.5 + 0.5 * erf(x); - - else { - y = 0.5 * erfc(z); - - if (x > 0) y = 1.0 - y; - } - - return (y); -} diff --git a/src/cephes.h b/src/cephes.h deleted file mode 100644 index 5eb340e..0000000 --- a/src/cephes.h +++ /dev/null @@ -1,8 +0,0 @@ -/* -* Cephes Math Library Release 2.2: June, 1992 -* Copyright 1984, 1987, 1988, 1992 by Stephen L. Moshier -* Direct inquiries to 30 Frost Street, Cambridge, MA 02140 -*/ - -/* Normal Distribution Function */ -double ndtr(double a); diff --git a/src/cli/CMakeLists.txt b/src/cli/CMakeLists.txt new file mode 100644 index 0000000..4d580b8 --- /dev/null +++ b/src/cli/CMakeLists.txt @@ -0,0 +1,12 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +add_executable(lsb_heka_cat lsb_heka_cat) +target_link_libraries(lsb_heka_cat luasandboxutil) + +if(LIBM_LIBRARY) + target_link_libraries(lsb_heka_cat ${LIBM_LIBRARY}) +endif() + +install(TARGETS lsb_heka_cat DESTINATION ${CMAKE_INSTALL_BINDIR}) diff --git a/src/cli/lsb_heka_cat.c b/src/cli/lsb_heka_cat.c new file mode 100644 index 0000000..0743ca0 --- /dev/null +++ b/src/cli/lsb_heka_cat.c @@ -0,0 +1,455 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lua_sandbox Heka file stream cat @file */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" +#include "luasandbox/util/input_buffer.h" +#include "luasandbox/util/protobuf.h" +#include "luasandbox/util/util.h" + +typedef void (*output_function)(lsb_heka_message *msg); + +static void +log_cb(void *context, const char *component, int level, const char *fmt, ...) +{ + (void)context; + (void)component; + (void)level; + va_list args; + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fwrite("\n", 1, 1, stderr); +} +static lsb_logger logger = { .context = NULL, .cb = log_cb }; + + +// todo this function is duplicated in util/heka_message.c +// we may want to add it to the API if it is generally useful +static const char* +read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) +{ + if (wiretype != LSB_PB_WT_LENGTH) { + return NULL; + } + + long long vi; + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || vi > e - p) { + return NULL; + } + s->s = p; + s->len = (size_t)vi; + return p + vi; +} + + +static void output_cs(const char *key, lsb_const_string *cs, bool eol) +{ + if (cs->s) { + fprintf(stdout, "%s: %.*s", key, (int)cs->len, cs->s); + } else { + fprintf(stdout, "%s: ", key); + } + if (eol) { + fprintf(stdout, "\n"); + } +} + + +static void output_text(lsb_heka_message *msg) +{ + static char tstr[64]; + if (!msg->raw.s) return; + + fprintf(stdout, ":Uuid: %02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx" + "-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx\n", + msg->uuid.s[0], msg->uuid.s[1], msg->uuid.s[2], msg->uuid.s[3], + msg->uuid.s[4], msg->uuid.s[5], msg->uuid.s[6], msg->uuid.s[7], + msg->uuid.s[8], msg->uuid.s[9], msg->uuid.s[10], msg->uuid.s[11], + msg->uuid.s[12], msg->uuid.s[13], msg->uuid.s[14], msg->uuid.s[15]); + fprintf(stdout, ":Timestamp: "); + + time_t t = floor(msg->timestamp / 1e9); + double frac = msg->timestamp - t * 1e9; + struct tm *tms = gmtime(&t); + strftime(tstr, sizeof(tstr) - 1, "%Y-%m-%dT%H:%M:%S", tms); + fprintf(stdout, "%s.%09lldZ\n", tstr, (long long)frac); + output_cs(":Type", &msg->type, true); + output_cs(":Logger", &msg->logger, true); + fprintf(stdout, ":Severity: %d\n", msg->severity); + output_cs(":Payload", &msg->payload, true); + output_cs(":EnvVersion", &msg->env_version, true); + if (msg->pid == INT_MIN) { + fprintf(stdout, ":Pid: \n"); + } else { + fprintf(stdout, ":Pid: %d\n", msg->pid); + } + output_cs(":Hostname", &msg->hostname, true); + fprintf(stdout, ":Fields:\n"); + for (int i = 0; i < msg->fields_len; ++i) { + fprintf(stdout, " | name: %.*s type: %d ", (int)msg->fields[i].name.len, + msg->fields[i].name.s, msg->fields[i].value_type); + output_cs("representation", &msg->fields[i].representation, false); + fprintf(stdout, " value: "); + const char *p = msg->fields[i].value.s; + const char *e = msg->fields[i].value.s + msg->fields[i].value.len; + switch (msg->fields[i].value_type) { + case LSB_PB_STRING: + { + lsb_const_string cs; + int tag = 0; + int wiretype = 0; + while (p && p < e) { + p = lsb_pb_read_key(p, &tag, &wiretype); + p = read_string(wiretype, p, e, &cs); + if (p) { + fprintf(stdout, "%.*s", (int)cs.len, cs.s); + if (p < e) fprintf(stdout, "|"); + } + } + } + break; + case LSB_PB_BYTES: + { + lsb_const_string cs; + int tag = 0; + int wiretype = 0; + while (p && p < e) { + p = lsb_pb_read_key(p, &tag, &wiretype); + p = read_string(wiretype, p, e, &cs); + if (p) { + for (size_t i = 0; i < cs.len; ++i) { + if (isprint(cs.s[i])) { + if (cs.s[i] == '\\') { + fwrite("\\\\", 2, 1, stdout); + } else { + putchar(cs.s[i]); + } + } else { + fprintf(stdout, "\\x%02hhx", (unsigned char)cs.s[i]); + } + } + if (p < e) fprintf(stdout, "|"); + } + } + } + break; + case LSB_PB_INTEGER: + { + long long ll = 0; + while (p && p < e) { + p = lsb_pb_read_varint(p, e, &ll); + if (p) { + fprintf(stdout, "%lld", ll); + if (p < e) fprintf(stdout, "|"); + } + } + } + break; + case LSB_PB_DOUBLE: + { + double d; + for (int i = 0; p <= (e - sizeof(double)); p += sizeof(double), ++i) { + memcpy(&d, p, sizeof(double)); + if (i > 0) fprintf(stdout, "|"); + fprintf(stdout, "%.17g", d); + } + } + break; + case LSB_PB_BOOL: + { + long long ll = 0; + while (p && p < e) { + p = lsb_pb_read_varint(p, e, &ll); + if (p) { + fprintf(stdout, "%s", ll == 0 ? "false" : "true"); + if (p < e) fprintf(stdout, "|"); + } + } + } + break; + } + fprintf(stdout, "\n"); + } + fprintf(stdout, "\n"); + return; +} + + +static void output_heka(lsb_heka_message *msg) +{ + static char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, msg->raw.len); + if (fwrite(header, hlen, 1, stdout) != 1) { + log_cb(NULL, NULL, 0, "error outputting header"); + exit(1); + } + if (fwrite(msg->raw.s, msg->raw.len, 1, stdout) != 1) { + log_cb(NULL, NULL, 0, "error outputting message"); + exit(1); + } + return; +} + + +static size_t read_file(FILE *fh, lsb_input_buffer *ib) +{ + size_t need; + if (ib->msglen) { + need = ib->msglen + (size_t)ib->buf[ib->scanpos + 1] + LSB_HDR_FRAME_SIZE + - (ib->readpos - ib->scanpos); + } else { + need = ib->scanpos + ib->size - ib->readpos; + } + + if (lsb_expand_input_buffer(ib, need)) { + log_cb(NULL, NULL, 0, "buffer reallocation failed"); + exit(EXIT_FAILURE); + } + size_t cnt = ib->size - ib->readpos; + size_t nread = fread(ib->buf + ib->readpos, + 1, + cnt, + fh); + ib->readpos += nread; + if (cnt != nread) { + clearerr(fh); + } + return nread; +} + + +static int find_header(const char *cur, int clen, const char *prev, int num) +{ + static int cnt = 0; + for (int i = clen - 1; i >= 0; --i) { + if (cur[i] == 0x1e) { + unsigned char hlen; + if (i + 1 < clen) { + hlen = (unsigned char)cur[i + 1]; + } else { + hlen = (unsigned char)prev[0]; + } + + char flag; + int hend = i + 2; + if (hend < clen) { + flag = cur[hend]; + } else { + flag = prev[hend - clen]; + } + + if (flag != 0x08) { + continue; + } + + hend += hlen; + if (hend < clen) { + flag = cur[hend]; + } else { + flag = prev[hend - clen]; + } + + if (flag == 0x1f) { + if (++cnt == num) { + return i; + } + } + } + } + return -1; +} + + +static void move_to_offset(FILE *fh, int num) +{ + char buf[2][BUFSIZ]; + memset(buf, 0, sizeof(buf)); + char *cur = buf[0]; + char *prev = buf[1]; + char *tmp; + + fseek(fh, 0, SEEK_END); + long len = ftell(fh); + if (len < 0) { + log_cb(NULL, NULL, 0, "ftell failed"); + } + size_t consume = len > BUFSIZ ? BUFSIZ : len; + size_t pos = len - consume; + while (consume > 0) { + if (fseek(fh, pos, SEEK_SET)) { + log_cb(NULL, NULL, 0, "fseek failed (reading)"); + break; + } + if (fread(cur, consume, 1, fh) != 1) { + log_cb(NULL, NULL, 0, "fread failed"); + break; + } + + int loc = find_header(cur, consume, prev, num); + if (loc >= 0) { + if (fseek(fh, -(consume - loc), SEEK_CUR)) { + log_cb(NULL, NULL, 0, "fseek failed (find position)"); + } + return; + } + + if (pos >= consume) { + pos -= consume; + } else { + consume = pos; + pos = 0; + } + + tmp = cur; + cur = prev; + prev = tmp; + } + fseek(fh, 0, SEEK_SET); +} + + +int main(int argc, char **argv) +{ + bool argerr = false; + bool follow = false; + bool use_stdin = false; + long num = -1; + char *matcher = "TRUE"; + char *filename = NULL; + + output_function ofn = output_text; + + int c; + while ((c = getopt(argc, argv, "tchfn:m:")) != -1) { + switch (c) { + case 't': + ofn = output_text; + break; + case 'c': + ofn = NULL; + break; + case 'h': + ofn = output_heka; + break; + case 'f': + follow = true; + break; + case 'n': + num = strtol(optarg, NULL, 10); + if (num < 0) argerr = true; + break; + case 'm': + matcher = optarg; + break; + default: + argerr = true; + break; + } + } + + if (argc - optind == 1) { + filename = argv[optind]; + use_stdin = strcmp("-", filename) == 0; + } else { + argerr = true; + } + + if (argerr) { + log_cb(NULL, NULL, 0, + "usage: %s [-t|-c|-h] [-m message_matcher] [-f] [-n #] \n" + "description:\n" + " -t output the messages in text format (default)\n" + " -c only output the message count\n" + " -h output the messages as a Heka protobuf stream\n" + " -f output appended data as the file grows\n" + " -n output the last # of messages (simple header check so not " + "100%% accurate)\n" + " -m message_matcher expression (default \"TRUE\")\n" + " FILE name of the file to cat or '-' for stdin\n" + "notes:\n" + " All output is written to stdout and all log/error messages are " + "written to stderr.\n", + argv[0]); + return EXIT_FAILURE; + } + + char ms[strlen(matcher) + 1]; + size_t len = sizeof(ms); + lsb_message_matcher *mm = lsb_create_message_matcher( + lsb_lua_string_unescape(ms, matcher, &len)); + if (!mm) { + log_cb(NULL, NULL, 0, "invalid message matcher: %s", matcher); + return EXIT_FAILURE; + } + + FILE *fh = stdin; + if (!use_stdin) { + fh = fopen(filename, "r"); + if (!fh) { + log_cb(NULL, NULL, 0, "error opening: %s", filename); + return EXIT_FAILURE; + } + if (num >= 0) { + move_to_offset(fh, num); + } + } + + size_t discarded_bytes; + size_t bytes_read = 0; + size_t pcnt = 0; + size_t mcnt = 0; + + lsb_input_buffer ib; + lsb_init_input_buffer(&ib, 1024 * 1024 * 1024); + lsb_heka_message msg; + lsb_init_heka_message(&msg, 8); + + do { + if (lsb_find_heka_message(&msg, &ib, true, &discarded_bytes, &logger)) { + if (lsb_eval_message_matcher(mm, &msg)) { + if (ofn) { + ofn(&msg); + } + ++mcnt; + } + ++pcnt; + } else { + bytes_read = read_file(fh, &ib); + } + if (bytes_read == 0 && follow && !use_stdin) { + sleep(1); + } + } while (bytes_read > 0 || follow); + + lsb_free_heka_message(&msg); + lsb_free_input_buffer(&ib); + lsb_destroy_message_matcher(mm); + if (!use_stdin) { + fclose(fh); + } + + if (ofn) { + log_cb(NULL, NULL, 0, "Processed: %zu, matched: %zu messages\n", pcnt, + mcnt); + } else { + printf("Processed: %zu, matched: %zu messages\n", pcnt, mcnt); + } +} diff --git a/src/heka/CMakeLists.txt b/src/heka/CMakeLists.txt new file mode 100644 index 0000000..74af8b6 --- /dev/null +++ b/src/heka/CMakeLists.txt @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set(HEKA_SRC +message.c +read_message_zc.c +sandbox.c +stream_reader.c +) + +add_library(luasandboxheka SHARED ${HEKA_SRC}) +set_target_properties(luasandboxheka PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_compile_definitions(luasandboxheka PRIVATE -Dluasandboxheka_EXPORTS) +target_link_libraries(luasandboxheka luasandbox) +if(WIN32) + target_link_libraries(luasandboxheka ws2_32) +endif() + +if(LIBM_LIBRARY) + target_link_libraries(luasandboxheka ${LIBM_LIBRARY}) +endif() + +install(TARGETS luasandboxheka DESTINATION ${CMAKE_INSTALL_LIBDIR}) +add_subdirectory(test) diff --git a/src/heka/message.c b/src/heka/message.c new file mode 100644 index 0000000..38fd99f --- /dev/null +++ b/src/heka/message.c @@ -0,0 +1,1060 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox Heka protobuf serialization/deserialization @file */ + +#include "message_impl.h" + +#include +#include +#include +#include + +#include "../luasandbox_impl.h" // todo the API should change so this doesn't +// need access to the impl + +#include "luasandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox_output.h" +#include "sandbox_impl.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox/util/protobuf.h" +#include "../luasandbox_defines.h" + + +static void set_missing_headers(lua_State *lua, int idx, lsb_heka_sandbox *hsb) +{ + lua_getfield(lua, idx, LSB_LOGGER); + int t = lua_type(lua, -1); + lua_pop(lua, 1); + if (t == LUA_TNIL && hsb->name) { + lua_pushstring(lua, hsb->name); + lua_setfield(lua, idx, LSB_LOGGER); + } + + lua_getfield(lua, idx, LSB_HOSTNAME); + t = lua_type(lua, -1); + lua_pop(lua, 1); + if (t == LUA_TNIL && hsb->hostname) { + lua_pushstring(lua, hsb->hostname); + lua_setfield(lua, idx, LSB_HOSTNAME); + } + + lua_getfield(lua, idx, LSB_PID); + t = lua_type(lua, -1); + lua_pop(lua, 1); + if (t == LUA_TNIL) { + lua_pushinteger(lua, hsb->pid); + lua_setfield(lua, idx, LSB_PID); + } +} + + +static const char* read_string(lua_State *lua, + int wiretype, + const char *p, + const char *e) +{ + if (wiretype != LSB_PB_WT_LENGTH) { + return NULL; + } + + long long len = 0; + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || len > e - p) { + return NULL; + } + lua_pushlstring(lua, p, (size_t)len); + p += len; + return p; +} + + +static const char* process_varint(lua_State *lua, + const char *name, + int wiretype, + int stack_index, + const char *p, + const char *e) +{ + if (wiretype != LSB_PB_WT_VARINT) { + return NULL; + } + long long val = 0; + p = lsb_pb_read_varint(p, e, &val); + if (!p) { + return NULL; + } + lua_pushnumber(lua, (lua_Number)val); + lua_setfield(lua, stack_index, name); + return p; +} + + +static const char* process_fields(lua_State *lua, const char *p, const char *e) +{ + int tag = 0; + int wiretype = 0; + int has_name = 0; + int value_count = 0; + long long len = 0; + + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || len > e - p) { + return NULL; + } + e = p + len; // only process to the end of the current field record + + lua_newtable(lua); // Table to be added to the Fields array index 4 + lua_newtable(lua); // Table to hold the value(s) index 5 + do { + p = lsb_pb_read_key(p, &tag, &wiretype); + + switch (tag) { + case 1: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 4, "name"); + has_name = 1; + } + break; + + case 2: + p = process_varint(lua, "value_type", wiretype, 4, p, e); + break; + + case 3: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 4, "representation"); + } + break; + + case 4: // value_string + case 5: // value_bytes + p = read_string(lua, wiretype, p, e); + if (p) { + lua_rawseti(lua, 5, ++value_count); + } + break; + + case 6: // value_integer + { + long long val = 0; + switch (wiretype) { + case 0: + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushnumber(lua, (lua_Number)val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || len > e - p) { + p = NULL; + break; + } + do { + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushnumber(lua, (lua_Number)val); + lua_rawseti(lua, 5, ++value_count); + } while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + + case 7: // value_double + { + double val = 0; + switch (wiretype) { + case 1: + if (p + sizeof(double) > e) { + p = NULL; + break; + } + memcpy(&val, p, sizeof(double)); + p += sizeof(double); + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || len > e - p || len % sizeof(double) != 0) { + p = NULL; + break; + } + do { + memcpy(&val, p, sizeof(double)); + p += sizeof(double); + lua_pushnumber(lua, val); + lua_rawseti(lua, 5, ++value_count); + } while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + + case 8: // value_bool + { + long long val = 0; + switch (wiretype) { + case 0: + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushboolean(lua, (int)val); + lua_rawseti(lua, 5, ++value_count); + break; + case 2: + p = lsb_pb_read_varint(p, e, &len); + if (!p || len < 0 || len > e - p) { + p = NULL; + break; + } + do { + p = lsb_pb_read_varint(p, p + len, &val); + if (!p) break; + lua_pushboolean(lua, (int)val); + lua_rawseti(lua, 5, ++value_count); + } while (p < e); + break; + default: + p = NULL; + break; + } + } + break; + default: + p = NULL; // don't allow unknown tags + break; + } + } while (p && p < e); + + lua_setfield(lua, 4, "value"); + + return has_name ? p : NULL; +} + + +/** + * Retrieve the string value for a Lua table entry (the table should be on top + * of the stack). If the entry is not found or not a string nothing is encoded. + * + * @param lua Pointer to the lua_State. + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +encode_string(lua_State *lua, lsb_output_buffer *ob, char tag, const char *name, + int index) +{ + lsb_err_value ret = NULL; + lua_getfield(lua, index, name); + if (lua_isstring(lua, -1)) { + size_t len; + const char *s = lua_tolstring(lua, -1, &len); + ret = lsb_pb_write_string(ob, tag, s, len); + } + lua_pop(lua, 1); + return ret; +} + + +/** + * Retrieve the numeric value for a Lua table entry (the table should be on top + * of the stack). If the entry is not found or not a number nothing is encoded, + * otherwise the number is varint encoded. + * + * @param lua Pointer to the lua_State. + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +encode_int(lua_State *lua, lsb_output_buffer *ob, char tag, const char *name, + int index) +{ + lsb_err_value ret = NULL; + lua_getfield(lua, index, name); + if (lua_isnumber(lua, -1)) { + unsigned long long i = (unsigned long long)lua_tonumber(lua, -1); + ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, i); + } + lua_pop(lua, 1); + return ret; +} + + +/** + * Encodes the field value. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. + * @param ob Pointer to the output data buffer. + * @param first Flag set on the first field value to add + * additional protobuf data in the correct order. + * In the case of arrays the value should contain + * the number of items in the array. + * @param representation String representation of the field + * i.e., "ms" + * @param value_type Protobuf value type + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +encode_field_value(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + int first, const char *representation, int value_type); + + +/** + * Encodes a field that has an array of values. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. + * @param ob Pointer to the output data buffer. + * @param t Lua type of the array values. + * @param representation String representation of the field i.e., "ms" + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +encode_field_array(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + int t, const char *representation, int value_type) +{ + int alen = (int)lua_objlen(lua, -2); + lsb_err_value ret = encode_field_value(lsb, lua, ob, alen, representation, + value_type); + size_t len_pos = ob->pos; + lua_pop(lua, 1); + for (int idx = 2; !ret && idx <= alen; ++idx) { + lua_rawgeti(lua, -1, idx); + if (lua_type(lua, -1) != t) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); + return LSB_ERR_HEKA_INPUT; + } + ret = encode_field_value(lsb, lua, ob, 0, representation, value_type); + lua_pop(lua, 1); + } + if (!ret && alen > 1 && value_type == LSB_PB_INTEGER) { + // fix up the varint packed length + size_t i = len_pos - 2; + int y = 0; + // find the length byte + while (ob->buf[i] != 0 && y < LSB_MAX_VARINT_BYTES) { + --i; + ++y; + } + if (y == LSB_MAX_VARINT_BYTES) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "unable set the length of the packed integer array"); + return LSB_ERR_LUA; + } + ret = lsb_pb_update_field_length(ob, i); + } + return ret; +} + + +/** + * Encodes a field that contains metadata in addition to its value. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. + * @param ob Pointer to the output data buffer. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +encode_field_object(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob) +{ + const char *representation = NULL; + lua_getfield(lua, -1, "representation"); + if (lua_isstring(lua, -1)) { + representation = lua_tostring(lua, -1); + } + + int value_type = -1; + lua_getfield(lua, -2, "value_type"); + if (lua_isnumber(lua, -1)) { + value_type = (int)lua_tointeger(lua, -1); + } + + lua_getfield(lua, -3, "value"); + lsb_err_value ret = encode_field_value(lsb, lua, ob, 1, representation, + value_type); + lua_pop(lua, 3); // remove representation, value_type and value + return ret; +} + + +static lsb_err_value +encode_field_value(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + int first, const char *representation, int value_type) +{ + lsb_err_value ret = NULL; + size_t len; + const char *s; + + int t = lua_type(lua, -1); + switch (t) { + case LUA_TSTRING: + switch (value_type) { + case -1: // not specified defaults to string + value_type = 0; + case 0: + case 1: + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid string value_type: %d", value_type); + return LSB_ERR_HEKA_INPUT; + } + if (first) { // this uglyness keeps the protobuf fields in order without + // additional lookups + if (value_type == LSB_PB_BYTES) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, value_type); + if (ret) return ret; + } + if (representation) { + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; + } + } + s = lua_tolstring(lua, -1, &len); + if (value_type == LSB_PB_BYTES) { + ret = lsb_pb_write_string(ob, LSB_PB_VALUE_BYTES, s, len); + if (ret) return ret; + } else { + ret = lsb_pb_write_string(ob, LSB_PB_VALUE_STRING, s, len); + if (ret) return ret; + } + break; + case LUA_TNUMBER: + switch (value_type) { + case -1: // not specified defaults to double + value_type = 3; + case 2: + case 3: + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid numeric value_type: %d", value_type); + return LSB_ERR_HEKA_INPUT; + } + if (first) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, value_type); + if (ret) return ret; + + if (representation) { + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; + } + if (1 == first) { + if (value_type == LSB_PB_INTEGER) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_VARINT); + } else { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_FIXED64); + } + if (ret) return ret; + } else { // pack array + if (value_type == LSB_PB_INTEGER) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_INTEGER, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, 0); // length tbd later + } else { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_DOUBLE, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, first * sizeof(double)); + } + if (ret) return ret; + } + } + if (value_type == LSB_PB_INTEGER) { + ret = lsb_pb_write_varint(ob, lua_tointeger(lua, -1)); + } else { + ret = lsb_pb_write_double(ob, lua_tonumber(lua, -1)); + } + if (ret) return ret; + break; + + case LUA_TBOOLEAN: + if (value_type != -1 && value_type != LSB_PB_BOOL) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "invalid boolean value_type: %d", value_type); + return LSB_ERR_HEKA_INPUT; + } + if (first) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, LSB_PB_BOOL); + if (ret) return ret; + + if (representation) { + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; + } + if (1 == first) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_VARINT); + } else { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_BOOL, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, first); + } + if (ret) return ret; + } + ret = lsb_pb_write_bool(ob, lua_toboolean(lua, -1)); + break; + + case LUA_TTABLE: + { + lua_rawgeti(lua, -1, 1); + int t = lua_type(lua, -1); + switch (t) { + case LUA_TNIL: + lua_pop(lua, 1); // remove the array test value + ret = encode_field_object(lsb, lua, ob); + break; + case LUA_TNUMBER: + case LUA_TSTRING: + case LUA_TBOOLEAN: + ret = encode_field_array(lsb, lua, ob, t, representation, value_type); + break; + default: + lua_pop(lua, 1); // remove the array test value + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "unsupported array type: %s", lua_typename(lua, t)); + return LSB_ERR_LUA; + } + } + break; + + case LUA_TLIGHTUSERDATA: + lua_getfield(lua, -4, "userdata"); + if (lua_type(lua, -1) != LUA_TUSERDATA) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "a lightuserdata output must also specify a userdata value"); + return LSB_ERR_LUA; + } + // fall thru + + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_output_function(lua, -1); + size_t len_pos = 0; + if (!fp) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "userdata object does not implement lsb_output"); + return LSB_ERR_LUA; + } + if (first) { + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_TYPE, LSB_PB_WT_VARINT); + if (ret) return ret; + + // encode userdata as a byte array + ret = lsb_pb_write_varint(ob, LSB_PB_BYTES); + if (ret) return ret; + + if (representation) { + ret = lsb_pb_write_string(ob, LSB_PB_REPRESENTATION, representation, + strlen(representation)); + if (ret) return ret; + } + } + + ret = lsb_pb_write_key(ob, LSB_PB_VALUE_BYTES, LSB_PB_WT_LENGTH); + if (ret) return ret; + + len_pos = ob->pos; + ret = lsb_pb_write_varint(ob, 0); // length tbd later + if (ret) return ret; + + lua_pushlightuserdata(lua, ob); + int result = fp(lua); + lua_pop(lua, 1); // remove output function + if (result) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "userdata output callback failed: %d", result); + return LSB_ERR_LUA; + } + ret = lsb_pb_update_field_length(ob, len_pos); + } + if (t == LUA_TLIGHTUSERDATA) lua_pop(lua, 1); // remove the userdata + break; + + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", + lua_typename(lua, t)); + return LSB_ERR_LUA; + } + return ret; +} + + +/** + * Iterates over the specified Lua table encoding the contents as user defined + * message fields. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State + * @param ob Pointer to the output data buffer. + * @param tag Field identifier. + * @param name Key used for the Lua table entry lookup. + * @param index Lua stack index of the table. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +encode_fields(lsb_lua_sandbox *lsb, lua_State *lua, lsb_output_buffer *ob, + char tag, const char *name, int index) +{ + lsb_err_value ret = NULL; + lua_getfield(lua, index, name); + if (!lua_istable(lua, -1)) { + return ret; + } + + lua_rawgeti(lua, -1, 1); // test for the array notation + size_t len_pos, len; + if (lua_istable(lua, -1)) { + int i = 1; + do { + ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (ret) return ret; + + len_pos = ob->pos; + ret = lsb_pb_write_varint(ob, 0); // length tbd later + if (ret) return ret; + + lua_getfield(lua, -1, "name"); + if (lua_isstring(lua, -1)) { + const char *s = lua_tolstring(lua, -1, &len); + ret = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "field name must be a string"); + ret = LSB_ERR_HEKA_INPUT; + } + lua_pop(lua, 1); // remove the name + if (ret) return ret; + + ret = encode_field_object(lsb, lua, ob); + if (!ret) ret = lsb_pb_update_field_length(ob, len_pos); + if (ret) return ret; + + lua_pop(lua, 1); // remove the current field object + lua_rawgeti(lua, -1, ++i); // grab the next field object + } while (!ret && !lua_isnil(lua, -1)); + } else { + lua_pop(lua, 1); // remove the array test value + lua_checkstack(lua, 2); + lua_pushnil(lua); + while (lua_next(lua, -2) != 0) { + ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (ret) return ret; + + len_pos = ob->pos; + ret = lsb_pb_write_varint(ob, 0); // length tbd later + if (ret) return ret; + + if (lua_isstring(lua, -2)) { + const char *s = lua_tolstring(lua, -2, &len); + ret = lsb_pb_write_string(ob, LSB_PB_NAME, s, len); + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "field name must be a string"); + ret = LSB_ERR_HEKA_INPUT; + } + if (ret) return ret; + + ret = encode_field_value(lsb, lua, ob, 1, NULL, -1); + if (!ret) ret = lsb_pb_update_field_length(ob, len_pos); + if (ret) return ret; + + lua_pop(lua, 1); // Remove the value leaving the key on top for + // the next interation. + } + } + lua_pop(lua, 1); // remove the fields table + return ret; +} + + +int heka_decode_message(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n == 1, n, "incorrect number of arguments"); + + size_t len; + const char *pbstr; + int t = lua_type(lua, 1); + if (t == LUA_TSTRING) { + pbstr = lua_tolstring(lua, 1, &len); + } else if (t == LUA_TUSERDATA) { + lua_CFunction fp = lsb_get_zero_copy_function(lua, 1); + if (!fp) { + return luaL_argerror(lua, 1, "no zero copy support"); + } + int results = fp(lua); + if (results != 2 || lua_type(lua, 2) != LUA_TLIGHTUSERDATA) { + return luaL_error(lua, "invalid zero copy return"); + } + pbstr = lua_touserdata(lua, 2); + len = (size_t)lua_tointeger(lua, 3); + lua_pop(lua, results); + } else { + return luaL_typerror(lua, 1, "string or userdata"); + } + + if (!pbstr || len < 20) { + return luaL_error(lua, "invalid message, too short"); + } + + const char *p = pbstr; + const char *lp = p; + const char *e = pbstr + len; + int wiretype = 0; + int tag = 0; + int has_uuid = 0; + int has_timestamp = 0; + int field_count = 0; + + lua_newtable(lua); // message table index 2 + do { + p = lsb_pb_read_key(p, &tag, &wiretype); + + switch (tag) { + case 1: + p = read_string(lua, wiretype, p, e); + if (p && p - lp == 18) { + lua_setfield(lua, 2, "Uuid"); + has_uuid = 1; + } else { + p = NULL; + } + break; + + case 2: + p = process_varint(lua, "Timestamp", wiretype, 2, p, e); + if (p) { + has_timestamp = 1; + } + break; + + case 3: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Type"); + } + break; + + case 4: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Logger"); + } + break; + + case 5: + p = process_varint(lua, "Severity", wiretype, 2, p, e); + break; + + case 6: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Payload"); + } + break; + + case 7: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "EnvVersion"); + } + break; + + case 8: + p = process_varint(lua, "Pid", wiretype, 2, p, e); + break; + + case 9: + p = read_string(lua, wiretype, p, e); + if (p) { + lua_setfield(lua, 2, "Hostname"); + } + break; + + case 10: + if (wiretype != 2) { + p = NULL; + break; + } + if (field_count == 0) { + lua_newtable(lua); // Fields table index 3 + } + p = process_fields(lua, p, e); + if (p) { + lua_rawseti(lua, 3, ++field_count); + } + break; + + default: + p = NULL; // don't allow unknown tags + break; + } + if (p) lp = p; + } while (p && p < e); + + if (!p) { + return luaL_error(lua, "error in tag: %d wiretype: %d offset: %d", tag, + wiretype, (const char *)lp - pbstr); + } + + if (!has_uuid || !has_timestamp) { + return luaL_error(lua, "missing required field uuid: %s timestamp: %s", + has_uuid ? "found" : "not found", + has_timestamp ? "found" : "not found"); + } + + if (field_count) { + lua_setfield(lua, 2, "Fields"); + } + + return 1; +} + + +int heka_encode_message(lua_State *lua) +{ + int n = lua_gettop(lua); + bool framed = false; + + switch (n) { + case 2: + luaL_checktype(lua, 2, LUA_TBOOLEAN); + framed = lua_toboolean(lua, 2); + // fall thru + case 1: + luaL_checktype(lua, 1, LUA_TTABLE); + break; + default: + return luaL_argerror(lua, n, "incorrect number of arguments"); + } + + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "encode_message() invalid " LSB_THIS_PTR); + + lsb->output.pos = 0; + lsb_err_value ret = heka_encode_message_table(lsb, lua, 1); + if (ret) { + const char *err = lsb_get_error(lsb); + if (strlen(err) == 0) err = ret; + return luaL_error(lua, "encode_message() failed: %s", err); + } + + size_t len = 0; + const char *output = lsb_get_output(lsb, &len); + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len; + + if (framed) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, len); + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = len + hlen; + luaL_Buffer b; + luaL_buffinit(lua, &b); + luaL_addlstring(&b, header, hlen); + luaL_addlstring(&b, output, len); + luaL_pushresult(&b); + } else { + lua_pushlstring(lua, output, len); + } + + if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] + > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + } + return 1; +} + + +lsb_err_value +heka_encode_message_table(lsb_lua_sandbox *lsb, lua_State *lua, int idx) +{ + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + lsb_err_value ret = NULL; + lsb_output_buffer *ob = &lsb->output; + ob->pos = 0; + + long long ts; + if (hsb->restricted_headers) { + ret = lsb_write_heka_uuid(ob, NULL, 0); + if (ret) return ret; + ts = lsb_get_timestamp(); + lua_pushstring(lua, hsb->name); + lua_setfield(lua, idx, LSB_LOGGER); + lua_pushstring(lua, hsb->hostname); + lua_setfield(lua, idx, LSB_HOSTNAME); + lua_pushinteger(lua, hsb->pid); + lua_setfield(lua, idx, LSB_PID); + } else { + lua_getfield(lua, idx, LSB_UUID); + size_t len; + const char *uuid = lua_tolstring(lua, -1, &len); + ret = lsb_write_heka_uuid(ob, uuid, len); + lua_pop(lua, 1); // remove uuid + + lua_getfield(lua, idx, LSB_TIMESTAMP); + if (lua_isnumber(lua, -1)) { + ts = (long long)lua_tonumber(lua, -1); + } else { + ts = lsb_get_timestamp(); + } + lua_pop(lua, 1); // remove timestamp + set_missing_headers(lua, idx, hsb); + } + + ret = lsb_pb_write_key(ob, LSB_PB_TIMESTAMP, LSB_PB_WT_VARINT); + if (!ret) ret = lsb_pb_write_varint(ob, ts); + if (!ret) ret = encode_string(lua, ob, LSB_PB_TYPE, LSB_TYPE, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_LOGGER, LSB_LOGGER, idx); + if (!ret) ret = encode_int(lua, ob, LSB_PB_SEVERITY, LSB_SEVERITY, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_PAYLOAD, LSB_PAYLOAD, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_ENV_VERSION, LSB_ENV_VERSION, + idx); + if (!ret) ret = encode_int(lua, ob, LSB_PB_PID, LSB_PID, idx); + if (!ret) ret = encode_string(lua, ob, LSB_PB_HOSTNAME, LSB_HOSTNAME, idx); + if (!ret) ret = encode_fields(lsb, lua, ob, LSB_PB_FIELDS, LSB_FIELDS, idx); + if (!ret) ret = lsb_expand_output_buffer(ob, 1); + ob->buf[ob->pos] = 0; // prevent possible overrun if treated as a string + return ret; +} + + +int heka_read_message(lua_State *lua, lsb_heka_message *m) +{ + int n = lua_gettop(lua); + if (n < 1 || n > 3) { + return luaL_error(lua, "%s() incorrect number of arguments", __func__); + } + size_t field_len; + const char *field = luaL_checklstring(lua, 1, &field_len); + int fi = luaL_optint(lua, 2, 0); + luaL_argcheck(lua, fi >= 0, 2, "field index must be >= 0"); + int ai = luaL_optint(lua, 3, 0); + luaL_argcheck(lua, ai >= 0, 3, "array index must be >= 0"); + + if (!m || !m->raw.s) { + lua_pushnil(lua); + return 1; + } + + if (strcmp(field, LSB_UUID) == 0) { + if (m->uuid.s) { + lua_pushlstring(lua, m->uuid.s, m->uuid.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_TIMESTAMP) == 0) { + lua_pushnumber(lua, (lua_Number)m->timestamp); + } else if (strcmp(field, LSB_TYPE) == 0) { + if (m->type.s) { + lua_pushlstring(lua, m->type.s, m->type.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_LOGGER) == 0) { + if (m->logger.s) { + lua_pushlstring(lua, m->logger.s, m->logger.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_SEVERITY) == 0) { + lua_pushinteger(lua, m->severity); + } else if (strcmp(field, LSB_PAYLOAD) == 0) { + if (m->payload.s) { + lua_pushlstring(lua, m->payload.s, m->payload.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_ENV_VERSION) == 0) { + if (m->env_version.s) { + lua_pushlstring(lua, m->env_version.s, m->env_version.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, LSB_PID) == 0) { + if (m->pid == INT_MIN) { + lua_pushnil(lua); + } else { + lua_pushinteger(lua, m->pid); + } + } else if (strcmp(field, LSB_HOSTNAME) == 0) { + if (m->hostname.s) { + lua_pushlstring(lua, m->hostname.s, m->hostname.len); + } else { + lua_pushnil(lua); + } + } else if (strcmp(field, "raw") == 0) { + lua_pushlstring(lua, m->raw.s, m->raw.len); + } else if (strcmp(field, "framed") == 0) { + { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, m->raw.len); + luaL_Buffer b; + luaL_buffinit(lua, &b); + luaL_addlstring(&b, header, hlen); + luaL_addlstring(&b, m->raw.s, m->raw.len); + luaL_pushresult(&b); + } + } else if (strcmp(field, "size") == 0) { + lua_pushnumber(lua, (lua_Number)m->raw.len); + } else { + if (field_len >= 8 + && memcmp(field, LSB_FIELDS "[", 7) == 0 + && field[field_len - 1] == ']') { + lsb_read_value v; + lsb_const_string f = { .s = field + 7, .len = field_len - 8 }; + lsb_read_heka_field(m, &f, fi, ai, &v); + switch (v.type) { + case LSB_READ_STRING: + lua_pushlstring(lua, v.u.s.s, v.u.s.len); + break; + case LSB_READ_NUMERIC: + lua_pushnumber(lua, v.u.d); + break; + case LSB_READ_BOOL: + lua_pushboolean(lua, v.u.d ? 1 : 0); + break; + default: + lua_pushnil(lua); + break; + } + } else { + luaL_error(lua, "%s() field: '%s' not supported/recognized", __func__, + field); + } + } + return 1; +} diff --git a/src/heka/message_impl.h b/src/heka/message_impl.h new file mode 100644 index 0000000..7c27434 --- /dev/null +++ b/src/heka/message_impl.h @@ -0,0 +1,66 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher @file */ + +#ifndef luasandbox_heka_sandbox_message_impl_h_ +#define luasandbox_heka_sandbox_message_impl_h_ + +#include + +#include "luasandbox.h" +#include "luasandbox/util/heka_message.h" + +// these functions are intentionally not exported + +/** + * Deserialize a Heka message protobuf string into a Lua table structure. + * + * @param lua Pointer the Lua state. + * + * @return int Number of items on the stack (1 table) or throws an error on + * failure + */ +int heka_decode_message(lua_State *lua); + +/** + * Serialize a Lua table structure into a Heka message protobuf string. + * + * @param lua Pointer the Lua state. + * + * @return int Number of items on the stack (1 string) or throws an error on + * failure + */ +int heka_encode_message(lua_State *lua); + +/** + * Serialize a Lua table structure into a Heka message protobuf (using the + * sandbox output buffer, Called indirectly from inject_message so the output + * buffer can be used without round tripping the resulting data back to the + * sandbox with heka_encode_message. + * + * @param lsb Pointer to the sandbox. + * @param lua Pointer to the lua_State. + * @param idx Lua stack index of the message table. + * + * @return lsb_err_value NULL on success error message on failure + */ +lsb_err_value +heka_encode_message_table(lsb_lua_sandbox *lsb, lua_State *lua, int idx); + + +/** + * Breakout of the common code for the read_message API + * + * @param lua Pointer to the lua_State + * @param m Heka message to extract the data from + * + * @return int Number of items on the stack (1 value) or throws an error on + * failure + */ +int heka_read_message(lua_State *lua, lsb_heka_message *m); + +#endif diff --git a/src/heka/read_message_zc.c b/src/heka/read_message_zc.c new file mode 100644 index 0000000..18f5017 --- /dev/null +++ b/src/heka/read_message_zc.c @@ -0,0 +1,200 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua Heka read message zero copy references @file */ + +#include +#include + +#include "../luasandbox_defines.h" +#include "lauxlib.h" +#include "lua.h" +#include "lualib.h" +#include "luasandbox/heka/sandbox.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox_output.h" +#include "luasandbox_serialize.h" + +static const char *metatable_name = "lsb.read_message_zc"; + +typedef struct read_message_zc +{ + lsb_const_string name; + int fi; + int ai; + char field[]; +} read_message_zc; + + +static const lsb_heka_message* get_heka_message(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + luaL_error(lua, "invalid " LSB_HEKA_THIS_PTR); + } + return lsb_heka_get_message(hsb); +} + + +static lsb_const_string read_message(lua_State *lua, read_message_zc *zc) +{ + lsb_const_string ret = { NULL, 0 }; + const lsb_heka_message *m = get_heka_message(lua); + if (!m || !m->raw.s) { + return ret; + } + + if (strcmp(zc->name.s, LSB_RAW) == 0) { + if (m->raw.s) ret = m->raw; + } else if (strcmp(zc->name.s, LSB_PAYLOAD) == 0) { + if (m->payload.s) ret = m->payload; + } else if (strcmp(zc->name.s, LSB_LOGGER) == 0) { + if (m->logger.s) ret = m->logger; + } else if (strcmp(zc->name.s, LSB_TYPE) == 0) { + if (m->type.s) ret = m->type; + } else if (strcmp(zc->name.s, LSB_ENV_VERSION) == 0) { + if (m->env_version.s) ret = m->env_version; + } else if (strcmp(zc->name.s, LSB_HOSTNAME) == 0) { + if (m->hostname.s) ret = m->hostname; + } else if (strcmp(zc->name.s, LSB_UUID) == 0) { + if (m->uuid.s) ret = m->uuid; + } else if (zc->name.len >= 8 + && memcmp(zc->name.s, LSB_FIELDS "[", 7) == 0 + && zc->name.s[zc->name.len - 1] == ']') { + lsb_read_value v; + lsb_const_string f = { zc->name.s + 7, zc->name.len - 8 }; + lsb_read_heka_field(m, &f, zc->fi, zc->ai, &v); + if (v.type == LSB_READ_STRING) { + ret = v.u.s; + } else if (v.type != LSB_READ_NIL) { + luaL_error(lua, "%s() zc->name.s: '%s' contains an unsupported type", + __func__, zc->name.s); + } + } else { + luaL_error(lua, "%s() zc->name.s: '%s' not supported/recognized", __func__, + zc->name.s); + } + return ret; +} + + +static int zc_output(lua_State *lua) +{ + lsb_output_buffer *ob = lua_touserdata(lua, -1); + if (!ob) {return 1;} + read_message_zc *zc = luaL_checkudata(lua, -2, metatable_name); + lsb_const_string f = read_message(lua, zc); + if (!f.s) return 0; + + if (strcmp(zc->field, LSB_FRAMED) == 0) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, f.len); + if (lsb_outputs(ob, header, hlen)) return 1; + } + if (lsb_outputs(ob, f.s, f.len)) return 1; + return 0; +} + + +static int zc_return(lua_State *lua) +{ + read_message_zc *zc = luaL_checkudata(lua, -1, metatable_name); + lua_checkstack(lua, 3); + int cnt = 2; + lsb_const_string f = read_message(lua, zc); + if (strcmp(zc->field, LSB_FRAMED) == 0) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, f.len); + lua_pushlstring(lua, header, hlen); + ++cnt; + } + lua_pushlightuserdata(lua, (void *)f.s); + lua_pushinteger(lua, (int)f.len); + return cnt; +} + + +static int zc_tostring(lua_State *lua) +{ + read_message_zc *zc = luaL_checkudata(lua, -1, metatable_name); + lsb_const_string f = read_message(lua, zc); + if (f.s) { + if (strcmp(zc->field, LSB_FRAMED) == 0) { + char header[LSB_MIN_HDR_SIZE]; + size_t hlen = lsb_write_heka_header(header, f.len); + lua_pushlstring(lua, header, hlen); + lua_pushlstring(lua, f.s, f.len); + lua_concat(lua, 2); + } else { + lua_pushlstring(lua, f.s, f.len); + } + } else { + lua_pushnil(lua); + } + return 1; +} + + +static const struct luaL_reg zclib_m[] = +{ + { "__tostring", zc_tostring }, + { NULL, NULL } +}; + + +int heka_create_read_message_zc(lua_State *lua) +{ + size_t len; + const char *field = luaL_checklstring(lua, 1, &len); + int fi = luaL_optint(lua, 2, 0); + luaL_argcheck(lua, fi >= 0, 2, "field index must be >= 0"); + int ai = luaL_optint(lua, 3, 0); + luaL_argcheck(lua, ai >= 0, 3, "array index must be >= 0"); + + if (!(strcmp(field, LSB_UUID) == 0 + || strcmp(field, LSB_TYPE) == 0 + || strcmp(field, LSB_LOGGER) == 0 + || strcmp(field, LSB_PAYLOAD) == 0 + || strcmp(field, LSB_ENV_VERSION) == 0 + || strcmp(field, LSB_HOSTNAME) == 0 + || strcmp(field, LSB_RAW) == 0 + || strcmp(field, LSB_FRAMED) == 0 + || (len >= 8 + && memcmp(field, LSB_FIELDS "[", 7) == 0 + && field[len - 1] == ']'))) { + luaL_error(lua, "%s() field: '%s' not supported/recognized", __func__, + field); + } + + if (luaL_newmetatable(lua, metatable_name) == 1) { + lua_newtable(lua); + lsb_add_output_function(lua, zc_output); + lsb_add_zero_copy_function(lua, zc_return); + lua_replace(lua, LUA_ENVIRONINDEX); + + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, zclib_m); + } + + read_message_zc *zc = lua_newuserdata(lua, sizeof(read_message_zc) + len + 1); + zc->fi = fi; + zc->ai = ai; + memcpy(zc->field, field, len + 1); + if (strcmp(field, LSB_FRAMED) == 0) { + zc->name.s = LSB_RAW; + zc->name.len = sizeof(LSB_RAW) - 1; + } else { + zc->name.s = zc->field; + zc->name.len = len; + } + + lua_pushvalue(lua, -2); + lua_setmetatable(lua, -2); + return 1; +} diff --git a/src/heka/sandbox.c b/src/heka/sandbox.c new file mode 100644 index 0000000..17cd176 --- /dev/null +++ b/src/heka/sandbox.c @@ -0,0 +1,1129 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Heka sandbox implementation @file */ + +#include "luasandbox/heka/sandbox.h" + +#include +#include +#include +#include + +#include "../luasandbox_defines.h" +#include "../luasandbox_impl.h" +#include "luasandbox.h" +#include "luasandbox/lualib.h" +#include "luasandbox/heka/stream_reader.h" +#include "luasandbox/util/heka_message_matcher.h" +#include "luasandbox/util/protobuf.h" +#include "luasandbox/util/running_stats.h" +#include "luasandbox_output.h" +#include "message_impl.h" +#include "sandbox_impl.h" + +#ifdef _WIN32 +#include +#else +#include +#endif + +lsb_err_id LSB_ERR_HEKA_INPUT = "invalid input"; + +static const char *pm_func_name = "process_message"; +static const char *im_func_name = "inject_message"; +static const char *lsb_heka_message_matcher = "lsb.heka_message_matcher"; + +int heka_create_stream_reader(lua_State *lua); +int heka_create_read_message_zc(lua_State *lua); + + +static int is_running(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); + } + // call inject_message with a NULL message/checkpoint (special case + // synchronization point) + if (hsb->cb.iim(hsb->parent, NULL, 0, NAN, NULL) != 0) { + return luaL_error(lua, "%s() failed: rejected by the callback", __func__); + } + lua_pushboolean(lua, lsb_heka_is_running(hsb)); + return 1; +} + + +static int read_message(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); + } + if (lua_gettop(lua) == 4) { + luaL_checktype(lua, 4, LUA_TBOOLEAN); + if (lua_toboolean(lua, 4)) { + return heka_create_read_message_zc(lua); + } + lua_pop(lua, 1); // remove the zc flag + } + return heka_read_message(lua, hsb->msg); +} + + +static lsb_message_matcher* mm_check(lua_State *lua) +{ + lsb_message_matcher **ppmm = luaL_checkudata(lua, 1, + lsb_heka_message_matcher); + return *ppmm; +} + + +static int mm_gc(lua_State *lua) +{ + lsb_message_matcher *mm = mm_check(lua); + lsb_destroy_message_matcher(mm); + return 0; +} + + +static int mm_eval(lua_State *lua) +{ + lsb_message_matcher *mm = mm_check(lua); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); + } + if (!hsb->msg || !hsb->msg->raw.s) { + return luaL_error(lua, "no active message"); + } + lua_pushboolean(lua, lsb_eval_message_matcher(mm, hsb->msg)); + return 1; +} + + +static int mm_create(lua_State *lua) +{ + const char *exp = luaL_checkstring(lua, 1); + lsb_message_matcher **ppmm = lua_newuserdata(lua, sizeof*ppmm); + if (luaL_newmetatable(lua, lsb_heka_message_matcher) == 1) { + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + lua_pushcfunction(lua, mm_gc); + lua_setfield(lua, -2, "__gc"); + lua_pushcfunction(lua, mm_eval); + lua_setfield(lua, -2, "eval"); + } + lua_setmetatable(lua, -2); + + *ppmm = lsb_create_message_matcher(exp); + if (!*ppmm) { + return luaL_error(lua, "invalid message matcher expression"); + } + return 1; +} + + +static int inject_message_input(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) { + return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); + } + + const char *scp = NULL; + double ncp = NAN; + int t = lua_type(lua, 2); + switch (t) { + case LUA_TNUMBER: + ncp = lua_tonumber(lua, 2); + break; + case LUA_TSTRING: + scp = lua_tostring(lua, 2); + break; + case LUA_TNONE: + case LUA_TNIL: + break; + default: + return luaL_error(lua, "%s() unsupported checkpoint type: %s", im_func_name, + lua_typename(lua, t)); + } + + lsb_const_string output; + t = lua_type(lua, 1); + switch (t) { + case LUA_TUSERDATA: + { + heka_stream_reader *hsr = luaL_checkudata(lua, 1, + LSB_HEKA_STREAM_READER); + if (hsr->msg.raw.s) { + output.len = hsr->msg.raw.len; + output.s = hsr->msg.raw.s; + } else { + return luaL_error(lua, "%s() attempted to inject a nil message", + im_func_name); + } + } + break; + case LUA_TSTRING: + { + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + output.s = lua_tolstring(lua, 1, &output.len); + bool ok = lsb_decode_heka_message(&m, output.s, output.len, NULL); + lsb_free_heka_message(&m); + if (!ok) { + return luaL_error(lua, "%s() attempted to inject a invalid protobuf " + "string", im_func_name); + } + } + break; + case LUA_TTABLE: + if (heka_encode_message_table(lsb, lua, 1)) { + const char *err = lsb_get_error(lsb); + if (strlen(err) == 0) err = "exceeded output_limit"; + return luaL_error(lua, "%s() failed: %s", im_func_name, err); + } + output.len = 0; + output.s = lsb_get_output(lsb, &output.len); + break; + case LUA_TNIL: + if (lua_isnoneornil(lua, 2)) { + return luaL_error(lua, "%s() message cannot be nil without a checkpoint" + " update", im_func_name); + } + output.len = 0; + output.s = NULL; + break; + default: + return luaL_error(lua, "%s() unsupported message type: %s", + im_func_name, lua_typename(lua, t)); + } + + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + int rv = hsb->cb.iim(hsb->parent, output.s, output.len, ncp, scp); + switch (rv) { + case LSB_HEKA_IM_SUCCESS: + break; + case LSB_HEKA_IM_CHECKPOINT: + return luaL_error(lua, "%s() failed: checkpoint update", + im_func_name); + case LSB_HEKA_IM_ERROR: + // fall through + default: + return luaL_error(lua, "%s() failed: rejected by the callback rv: %d", + im_func_name, rv); + } + if (output.s) { + ++hsb->stats.im_cnt; + hsb->stats.im_bytes += output.len; + } + return 0; +} + + +static int inject_message_analysis(lua_State *lua) +{ + luaL_checktype(lua, 1, LUA_TTABLE); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, im_func_name); + + if (heka_encode_message_table(lsb, lua, 1)) { + return luaL_error(lua, "%s() failed: %s", im_func_name, lsb_get_error(lsb)); + } + + size_t output_len = 0; + const char *output = lsb_get_output(lsb, &output_len); + lsb_heka_sandbox *hsb = lsb_get_parent(lsb); + int rv = hsb->cb.aim(hsb->parent, output, output_len); + switch (rv) { + case LSB_HEKA_IM_SUCCESS: + break; + case LSB_HEKA_IM_LIMIT: + return luaL_error(lua, "%s() failed: injection limit exceeded", + im_func_name); + case LSB_HEKA_IM_ERROR: + // fall through + default: + return luaL_error(lua, "%s() failed: rejected by the callback rv: %d", + im_func_name, rv); + } + ++hsb->stats.im_cnt; + hsb->stats.im_bytes += output_len; + return 0; +} + + +static int inject_payload(lua_State *lua) +{ + static const char *default_type = "txt"; + + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, __func__); + + int n = lua_gettop(lua); + + if (n > 0) { + if (lua_type(lua, 1) != LUA_TSTRING) { + return luaL_error(lua, "%s() payload_type argument must be a string", + __func__); + } + } + + if (n > 1) { + if (lua_type(lua, 2) != LUA_TSTRING) { + return luaL_error(lua, "%s() payload_name argument must be a string", + __func__); + } + } + + if (n > 2) { + lsb_output_coroutine(lsb, lua, 3, n, 1); + lua_pop(lua, n - 2); + } + size_t len = 0; + const char *output = lsb_get_output(lsb, &len); + if (!len) return 0; + + // build up a heka message table + lua_createtable(lua, 0, 2); // message + lua_createtable(lua, 0, 2); // Fields + if (n > 0) { + lua_pushvalue(lua, 1); + } else { + lua_pushstring(lua, default_type); + } + lua_setfield(lua, -2, "payload_type"); + + if (n > 1) { + lua_pushvalue(lua, 2); + lua_setfield(lua, -2, "payload_name"); + } + lua_setfield(lua, -2, LSB_FIELDS); + lua_pushstring(lua, "inject_payload"); + lua_setfield(lua, -2, LSB_TYPE); + lua_pushlstring(lua, output, len); + lua_setfield(lua, -2, LSB_PAYLOAD); + if (lua_gettop(lua) > 1) lua_replace(lua, 1); + + inject_message_analysis(lua); + return 0; +} + + +static int update_checkpoint(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + lsb_heka_sandbox *hsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!hsb) { + return luaL_error(lua, "%s() invalid " LSB_HEKA_THIS_PTR, __func__); + } + + int result = 0; + int n = lua_gettop(lua); + switch (n) { + case 2: // async case + luaL_checktype(lua, 2, LUA_TNUMBER); + hsb->stats.pm_failures += (unsigned long long)lua_tonumber(lua, 2); + // fall thru + case 1: + luaL_checktype(lua, 1, LUA_TLIGHTUSERDATA); + result = hsb->ucp(hsb->parent, lua_touserdata(lua, 1)); + break; + case 0: // batch case + result = hsb->ucp(hsb->parent, NULL); + break; + default: + return luaL_error(lua, "%s() invalid number of args: %d", __func__, n); + } + if (result) { + return luaL_error(lua, "%s() failed: rejected by the callback", __func__); + } + return result; +} + + +static void set_restrictions(lua_State *lua, lsb_heka_sandbox *hsb) +{ + static const char *io[] = { + NULL, "", "dofile", "load", "loadfile", "loadstring", "newproxy", + NULL, "os", "exit", "setlocale", + NULL, "string", "dump" + }; + + static const char *analysis[] = { + NULL, "", "collectgarbage", "dofile", "load", "loadfile", "loadstring", + "newproxy", "print", + NULL, "os", "getenv", "execute", "exit", "remove", "rename", "setlocale", + "tmpname", + NULL, "string", "dump" + }; + + const char **list; + size_t list_size; + + lua_pushlightuserdata(lua, hsb); + lua_setfield(lua, LUA_REGISTRYINDEX, LSB_HEKA_THIS_PTR); + + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + switch (hsb->type) { + case 'i': + case 'o': + list_size = sizeof(io) / sizeof(*io); + list = io; + break; + default: + list_size = sizeof(analysis) / sizeof(*analysis); + list = analysis; + lua_newtable(lua); + lua_pushboolean(lua, true); + lua_setfield(lua, -2, LUA_IOLIBNAME); + lua_pushboolean(lua, true); + lua_setfield(lua, -2, LUA_COLIBNAME); + lua_setfield(lua, 1, "disable_modules"); + break; + } + + lua_newtable(lua); + for (unsigned i = 0, j = 1; i < list_size; ++i) { + if (list[i]) { + lua_pushstring(lua, list[i]); + lua_rawseti(lua, -2, j++); + } else { + if (lua_gettop(lua) == 3) lua_pop(lua, 1); // remove the previous list + lua_newtable(lua); + lua_pushvalue(lua, -1); + lua_setfield(lua, -3, list[++i]); + j = 1; + } + } + lua_pop(lua, 1); // remove the last list + lua_setfield(lua, 1, "remove_entries"); + + lua_getfield(lua, 1, LSB_HEKA_MAX_MESSAGE_SIZE); + if (lua_type(lua, -1) != LUA_TNUMBER || lua_tointeger(lua, -1) <= 0) { + lua_pushnumber(lua, 64 * 1024); + lua_setfield(lua, 1, LSB_HEKA_MAX_MESSAGE_SIZE); + } + lua_pop(lua, 1); // remove max_message_size + + lua_getfield(lua, 1, LSB_LOGGER); + size_t len; + const char *tmp = lua_tolstring(lua, -1, &len); + if (tmp) { + hsb->name = malloc(len + 1); + if (hsb->name) strcpy(hsb->name, tmp); + } + lua_pop(lua, 1); // remove the Logger + + lua_getfield(lua, 1, LSB_HOSTNAME); + tmp = lua_tolstring(lua, -1, &len); + if (tmp) { + hsb->hostname = malloc(len + 1); + if (hsb->hostname) strcpy(hsb->hostname, tmp); + } else { + char hostname[256] = { 0 }; + if (gethostname(hostname, sizeof hostname)) { + hostname[sizeof hostname - 1] = 0; + } + len = strlen(hostname); + hsb->hostname = malloc(len + 1); + if (hsb->hostname) strcpy(hsb->hostname, hostname); + } + lua_pop(lua, 1); // remove the Hostname + + lua_getfield(lua, 1, LSB_PID); + hsb->pid = (int)lua_tointeger(lua, -1); + lua_pop(lua, 1); + + lua_getfield(lua, 1, "restricted_headers"); + if (lua_type(lua, -1) == LUA_TBOOLEAN) { + hsb->restricted_headers = lua_toboolean(lua, -1); + } + lua_pop(lua, 1); // remove the restricted_headers boolean + + lua_pop(lua, 1); // remove the lsb_config table +} + + +lsb_heka_sandbox* lsb_heka_create_input(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_im_input im) +{ + if (!lua_file) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } + return NULL; + } + + if (!im) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "inject_message callback must " + "be specified"); + } + return NULL; + } + + lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); + if (!hsb) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } + return NULL; + } + + hsb->type = 'i'; + hsb->parent = parent; + hsb->msg = NULL; + hsb->cb.iim = im; + hsb->name = NULL; + hsb->hostname = NULL; + + hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); + if (!hsb->lsb) { + free(hsb); + return NULL; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + set_restrictions(lua, hsb); + + lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); + lsb_add_function(hsb->lsb, inject_message_input, "inject_message"); +// inject_payload is intentionally excluded from input plugins +// you can construct whatever you need with inject_message + lsb_add_function(hsb->lsb, heka_create_stream_reader, "create_stream_reader"); + lsb_add_function(hsb->lsb, is_running, "is_running"); + + if (lsb_init(hsb->lsb, state_file)) { + if (logger && logger->cb) { + logger->cb(logger->context, hsb->name, 3, "%s", lsb_get_error(hsb->lsb)); + } + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return NULL; + } + +// remove output function + lua_pushnil(lua); + lua_setglobal(lua, "output"); + + return hsb; +} + + +static int process_message(lsb_heka_sandbox *hsb, lsb_heka_message *msg, + lua_State *lua, int nargs, bool profile) +{ + unsigned long long start, end; + + hsb->msg = msg; + if (profile) { + start = lsb_get_time(); + } + if (lua_pcall(lua, nargs, 2, 0) != 0) { + char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); + if (hsb->type == 'i' && em && strcmp(em, LSB_SHUTTING_DOWN) == 0) { + return 0; + } + size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", pm_func_name, + em ? em : LSB_NIL_ERROR); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + if (profile) { + end = lsb_get_time(); + lsb_update_running_stats(&hsb->stats.pm, (double)(end - start)); + } + hsb->msg = NULL; + + if (lua_type(lua, 1) != LUA_TNUMBER) { + char err[LSB_ERROR_SIZE]; + size_t len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a numeric status code", + pm_func_name); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + + int status = (int)lua_tointeger(lua, 1); + switch (lua_type(lua, 2)) { + case LUA_TNIL: + lsb_set_error(hsb->lsb, NULL); + break; + case LUA_TSTRING: + lsb_set_error(hsb->lsb, lua_tostring(lua, 2)); + break; + default: + { + char err[LSB_ERROR_SIZE]; + int len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a nil or string error message", + pm_func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + break; + } + lua_pop(lua, 2); + lsb_pcall_teardown(hsb->lsb); + + if (status > 0) { + char err[LSB_ERROR_SIZE]; + size_t len = snprintf(err, LSB_ERROR_SIZE, + "%s() received a termination status code", + pm_func_name); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } else if (status == LSB_HEKA_PM_FAIL) { + ++hsb->stats.pm_cnt; + ++hsb->stats.pm_failures; + } else if (hsb->type != 'o' || status != LSB_HEKA_PM_RETRY) { + ++hsb->stats.pm_cnt; + } + return status; +} + + +int lsb_heka_pm_input(lsb_heka_sandbox *hsb, + double cp_numeric, + const char *cp_string, + bool profile) +{ + if (!hsb || hsb->type != 'i') { + return 1; + } + + lsb_err_value ret = lsb_pcall_setup(hsb->lsb, pm_func_name); + if (ret) { + if (ret != LSB_ERR_TERMINATED) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", + pm_func_name); + lsb_terminate(hsb->lsb, err); + } + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + + if (!isnan(cp_numeric)) { + lua_pushnumber(lua, cp_numeric); + } else if (cp_string) { + lua_pushstring(lua, cp_string); + } else { + lua_pushnil(lua); + } + return process_message(hsb, NULL, lua, 1, profile); +} + + +lsb_heka_sandbox* lsb_heka_create_analysis(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_im_analysis im) +{ + if (!lua_file) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } + return NULL; + } + + if (!im) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "inject_message callback must " + "be specified"); + } + return NULL; + } + + + lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); + if (!hsb) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } + return NULL; + } + + hsb->type = 'a'; + hsb->parent = parent; + hsb->msg = NULL; + hsb->cb.aim = im; + hsb->name = NULL; + hsb->hostname = NULL; + hsb->restricted_headers = true; + + hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); + if (!hsb->lsb) { + free(hsb); + return NULL; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + set_restrictions(lua, hsb); + + lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); + lsb_add_function(hsb->lsb, read_message, "read_message"); + lsb_add_function(hsb->lsb, inject_message_analysis, "inject_message"); + lsb_add_function(hsb->lsb, inject_payload, "inject_payload"); + // rename output to add_to_payload + lua_getglobal(lua, "output"); + lua_setglobal(lua, "add_to_payload"); + lua_pushnil(lua); + lua_setglobal(lua, "output"); + + if (lsb_init(hsb->lsb, state_file)) { + if (logger && logger->cb) { + logger->cb(logger->context, hsb->name, 3, "%s", lsb_get_error(hsb->lsb)); + } + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return NULL; + } + return hsb; +} + + +int lsb_heka_pm_analysis(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + bool profile) +{ + if (!hsb || !msg || hsb->type != 'a') return 1; + + if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", pm_func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + return process_message(hsb, msg, lua, 0, profile); +} + + +// IO write zero copy replacement +static int pushresult(lua_State *lua, int i, const char *filename) +{ + int en = errno; /* calls to Lua API may change this value */ + if (i) { + lua_pushboolean(lua, 1); + return 1; + } else { + lua_pushnil(lua); + if (filename) lua_pushfstring(lua, "%s: %s", filename, strerror(en)); + else lua_pushfstring(lua, "%s", strerror(en)); + lua_pushinteger(lua, en); + return 3; + } +} + + +static int zc_write(lua_State *lua, FILE *f, int arg) +{ + int n = lua_gettop(lua); + int nargs = n - 1; + int status = 1; + for (; nargs--; arg++) { + switch (lua_type(lua, arg)) { + case LUA_TNUMBER: + /* optimization: could be done exactly as for strings */ + status = status && + fprintf(f, LUA_NUMBER_FMT, lua_tonumber(lua, arg)) > 0; + break; + case LUA_TSTRING: + { + size_t l; + const char *s = luaL_checklstring(lua, arg, &l); + status = status && (fwrite(s, sizeof(char), l, f) == l); + } + break; + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_zero_copy_function(lua, arg); + if (!fp) { + return luaL_argerror(lua, arg, "no zero copy support"); + } + lua_pushvalue(lua, arg); + int results = fp(lua); + int start = n + 2; + int end = start + results; + size_t len; + const char *s; + for (int i = start; i < end; ++i) { + switch (lua_type(lua, i)) { + case LUA_TSTRING: + s = lua_tolstring(lua, i, &len); + if (s && len > 0) { + status = status && (fwrite(s, sizeof(char), len, f) == len); + } + break; + case LUA_TLIGHTUSERDATA: + s = lua_touserdata(lua, i++); + len = (size_t)lua_tointeger(lua, i); + if (s && len > 0) { + status = status && (fwrite(s, sizeof(char), len, f) == len); + } + break; + default: + return luaL_error(lua, "invalid zero copy return"); + } + } + lua_pop(lua, results + 1); // remove the returns values and + // the copy of the userdata + } + break; + default: + return luaL_typerror(lua, arg, "number, string or userdata"); + } + } + return pushresult(lua, status, NULL); +} + +static FILE* getiofile(lua_State *lua, int findex) +{ + static const char *const fnames[] = { "input", "output" }; + FILE *f; + lua_rawgeti(lua, LUA_ENVIRONINDEX, findex); + f = *(FILE **)lua_touserdata(lua, -1); + if (f == NULL) luaL_error(lua, "standard %s file is closed", + fnames[findex - 1]); + return f; +} + + +static int zc_io_write(lua_State *lua) +{ + return zc_write(lua, getiofile(lua, 2), 1); +} + + +static FILE* tofile(lua_State *lua) +{ + FILE **f = (FILE **)luaL_checkudata(lua, 1, LUA_FILEHANDLE); + if (*f == NULL) luaL_error(lua, "attempt to use a closed file"); + return *f; +} + + +static int zc_f_write(lua_State *lua) +{ + return zc_write(lua, tofile(lua), 2); +} + + +static int zc_luaopen_io(lua_State *lua) +{ + luaopen_io(lua); + lua_pushcclosure(lua, zc_io_write, 0); + lua_setfield(lua, -2, "write"); + + luaL_getmetatable(lua, LUA_FILEHANDLE); + lua_pushcclosure(lua, zc_f_write, 0); + lua_setfield(lua, -2, "write"); + lua_pop(lua, 1); // remove the metatable + return 1; +} +// End IO write zero copy replacement + + +lsb_heka_sandbox* lsb_heka_create_output(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_update_checkpoint ucp) +{ + return lsb_heka_create_output_im(parent, lua_file, state_file, lsb_cfg, + logger, ucp, NULL); +} + + +lsb_heka_sandbox* lsb_heka_create_output_im(void *parent, + const char *lua_file, + const char *state_file, + const char *lsb_cfg, + lsb_logger *logger, + lsb_heka_update_checkpoint ucp, + lsb_heka_im_analysis im) +{ + if (!lua_file) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } + return NULL; + } + + if (!ucp) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, LSB_HEKA_UPDATE_CHECKPOINT + " callback must be specified"); + } + return NULL; + } + + lsb_heka_sandbox *hsb = calloc(1, sizeof(lsb_heka_sandbox)); + if (!hsb) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } + return NULL; + } + + hsb->type = 'o'; + hsb->parent = parent; + hsb->msg = NULL; + hsb->ucp = ucp; + hsb->cb.aim = im; + hsb->name = NULL; + hsb->hostname = NULL; + + hsb->lsb = lsb_create(hsb, lua_file, lsb_cfg, logger); + if (!hsb->lsb) { + free(hsb); + return NULL; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + set_restrictions(lua, hsb); + + lsb_add_function(hsb->lsb, read_message, "read_message"); + lsb_add_function(hsb->lsb, heka_decode_message, "decode_message"); + lsb_add_function(hsb->lsb, heka_encode_message, "encode_message"); + lsb_add_function(hsb->lsb, update_checkpoint, LSB_HEKA_UPDATE_CHECKPOINT); + lsb_add_function(hsb->lsb, mm_create, "create_message_matcher"); + if (im) { + lsb_add_function(hsb->lsb, inject_message_analysis, "inject_message"); + // inject_payload is intentionally excluded from output plugins + // you can construct whatever you need with inject_message + } + + // start io.write override with zero copy functionality + lua_getfield(lua, LUA_REGISTRYINDEX, "_PRELOADED"); + lua_pushstring(lua, LUA_IOLIBNAME); + lua_pushcfunction(lua, zc_luaopen_io); + lua_rawset(lua, -3); + lua_pop(lua, 1); + // end io.write override + + if (lsb_init(hsb->lsb, state_file)) { + if (logger && logger->cb) { + logger->cb(logger->context, hsb->name, 3, "%s", lsb_get_error(hsb->lsb)); + } + lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return NULL; + } + +// remove output function + lua_pushnil(lua); + lua_setglobal(lua, "output"); + + return hsb; +} + + + +void lsb_heka_stop_sandbox_clean(lsb_heka_sandbox *hsb) +{ + lsb_stop_sandbox_clean(hsb->lsb); +} + + +void lsb_heka_stop_sandbox(lsb_heka_sandbox *hsb) +{ + lsb_stop_sandbox(hsb->lsb); +} + + + +void lsb_heka_terminate_sandbox(lsb_heka_sandbox *hsb, const char *err) +{ + lsb_terminate(hsb->lsb, err); +} + + +char* lsb_heka_destroy_sandbox(lsb_heka_sandbox *hsb) +{ + if (!hsb) return NULL; + + char *msg = lsb_destroy(hsb->lsb); + free(hsb->hostname); + free(hsb->name); + free(hsb); + return msg; +} + + +int lsb_heka_pm_output(lsb_heka_sandbox *hsb, + lsb_heka_message *msg, + void *sequence_id, + bool profile) +{ + if (!hsb || !msg || hsb->type != 'o') return 1; + + if (lsb_pcall_setup(hsb->lsb, pm_func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", pm_func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + + int nargs = 0; + if (sequence_id) { + nargs = 1; + lua_pushlightuserdata(lua, sequence_id); + } + return process_message(hsb, msg, lua, nargs, profile); +} + + +int lsb_heka_timer_event(lsb_heka_sandbox *hsb, time_t t, bool shutdown) +{ + static const char *func_name = "timer_event"; + + if (!hsb || (hsb->type != 'o' && hsb->type != 'a')) { + return 1; + } + + lua_State *lua = lsb_get_lua(hsb->lsb); + if (!lua) return 1; + + if (lsb_pcall_setup(hsb->lsb, func_name)) { + char err[LSB_ERROR_SIZE]; + snprintf(err, LSB_ERROR_SIZE, "%s() function was not found", func_name); + lsb_terminate(hsb->lsb, err); + return 1; + } + lua_pushnumber(lua, t * 1e9); + lua_pushboolean(lua, shutdown); + + unsigned long long start, end; + start = lsb_get_time(); + if (lua_pcall(lua, 2, 0, 0) != 0) { + char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); + size_t len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, + em ? em : LSB_NIL_ERROR); + if (len >= LSB_ERROR_SIZE) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(hsb->lsb, err); + return 1; + } + end = lsb_get_time(); + lsb_update_running_stats(&hsb->stats.te, (double)(end - start)); + lsb_pcall_teardown(hsb->lsb); + lua_gc(lua, LUA_GCCOLLECT, 0); + return 0; +} + + +const char* lsb_heka_get_error(lsb_heka_sandbox *hsb) +{ + return hsb ? lsb_get_error(hsb->lsb) : ""; +} + + +const char* lsb_heka_get_lua_file(lsb_heka_sandbox *hsb) +{ + return hsb ? lsb_get_lua_file(hsb->lsb) : NULL; +} + + +lsb_heka_stats lsb_heka_get_stats(lsb_heka_sandbox *hsb) +{ + if (!hsb) return (struct lsb_heka_stats){ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + + return (struct lsb_heka_stats){ + .mem_cur = lsb_usage(hsb->lsb, LSB_UT_MEMORY, LSB_US_CURRENT), + .mem_max = lsb_usage(hsb->lsb, LSB_UT_MEMORY, LSB_US_MAXIMUM), + .out_max = lsb_usage(hsb->lsb, LSB_UT_OUTPUT, LSB_US_MAXIMUM), + .ins_max = lsb_usage(hsb->lsb, LSB_UT_INSTRUCTION, LSB_US_MAXIMUM), + .im_cnt = hsb->stats.im_cnt, + .im_bytes = hsb->stats.im_bytes, + .pm_cnt = hsb->stats.pm_cnt, + .pm_failures = hsb->stats.pm_failures, + .pm_avg = hsb->stats.pm.mean, + .pm_sd = lsb_sd_running_stats(&hsb->stats.pm), + .te_avg = hsb->stats.te.mean, + .te_sd = lsb_sd_running_stats(&hsb->stats.te) + }; +} + + +bool lsb_heka_is_running(lsb_heka_sandbox *hsb) +{ + if (!hsb) return false; + if (lsb_get_state(hsb->lsb) == LSB_RUNNING) return true; + return false; +} + + +lsb_state lsb_heka_get_state(lsb_heka_sandbox *hsb) +{ + if (!hsb) return LSB_UNKNOWN; + return lsb_get_state(hsb->lsb); +} + + +const lsb_heka_message* lsb_heka_get_message(lsb_heka_sandbox *hsb) +{ + if (!hsb) return NULL; + return hsb->msg; +} + + +char lsb_heka_get_type(lsb_heka_sandbox *hsb) +{ + if (!hsb) return '\0'; + return hsb->type; +} diff --git a/src/heka/sandbox_impl.h b/src/heka/sandbox_impl.h new file mode 100644 index 0000000..977516c --- /dev/null +++ b/src/heka/sandbox_impl.h @@ -0,0 +1,46 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight Heka sandbox private implementation @file */ + +#ifndef luasandbox_heka_sandbox_impl_h_ +#define luasandbox_heka_sandbox_impl_h_ + +#include "luasandbox.h" +#include "luasandbox/heka/sandbox.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/running_stats.h" + +struct heka_stats { + unsigned long long im_cnt; + unsigned long long im_bytes; + + unsigned long long pm_cnt; + unsigned long long pm_failures; + + lsb_running_stats pm; + lsb_running_stats te; +}; + + +struct lsb_heka_sandbox { + void *parent; + lsb_lua_sandbox *lsb; + lsb_heka_message *msg; + char *name; + char *hostname; + union { + lsb_heka_im_input iim; // used in input plugins only + lsb_heka_im_analysis aim; // used in analysis and output plugins + } cb; + struct heka_stats stats; + char type; + bool restricted_headers; + int pid; + lsb_heka_update_checkpoint ucp; // used in output plugins only +}; + +#endif diff --git a/src/heka/stream_reader.c b/src/heka/stream_reader.c new file mode 100644 index 0000000..f6ee2a4 --- /dev/null +++ b/src/heka/stream_reader.c @@ -0,0 +1,222 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight Heka stream reader implementation @file */ + +#include "luasandbox/heka/stream_reader.h" + +#include +#include + +#include "luasandbox/heka/sandbox.h" +#include "message_impl.h" +#include "luasandbox/lauxlib.h" + +static heka_stream_reader* check_hsr(lua_State *lua, int args) +{ + heka_stream_reader *hsr = luaL_checkudata(lua, 1, LSB_HEKA_STREAM_READER); + luaL_argcheck(lua, args == lua_gettop(lua), 0, + "incorrect number of arguments"); + return hsr; +} + + +static int hsr_decode_message(lua_State *lua) +{ + heka_stream_reader *hsr = check_hsr(lua, 2); + lsb_input_buffer *b = &hsr->buf; + b->readpos = b->scanpos = b->msglen = 0; + + size_t len; + if (lua_type(lua, 2) == LUA_TSTRING) { + const char *s = lua_tolstring(lua, 2, &len); + if (len > 0) { + if (lsb_expand_input_buffer(b, len)) { + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); + } + memcpy(b->buf, s, len); + } else { + return luaL_error(lua, "empty protobuf string"); + } + } else { + return luaL_error(lua, "buffer must be string"); + } + + if (!lsb_decode_heka_message(&hsr->msg, b->buf, len, NULL)) { + return luaL_error(lua, "invalid protobuf string"); + } + return 0; +} + + +static int hsr_find_message(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n > 1 && n < 4, 0, "incorrect number of arguments"); + heka_stream_reader *hsr = luaL_checkudata(lua, 1, LSB_HEKA_STREAM_READER); + lsb_input_buffer *b = &hsr->buf; + + FILE *fh = NULL; + switch (lua_type(lua, 2)) { + case LUA_TNIL: // scan the existing buffer + break; + case LUA_TSTRING: // add data to the buffer + { + size_t len; + const char *s = lua_tolstring(lua, 2, &len); + if (len > 0) { + if (lsb_expand_input_buffer(b, len)) { + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); + } + memcpy(b->buf + b->readpos, s, len); + b->readpos += len; + } + } + break; + case LUA_TUSERDATA: // add data from the provided file handle to the buffer + fh = *(FILE **)luaL_checkudata(lua, 2, "FILE*"); + if (!fh) luaL_error(lua, "attempt to use a closed file"); + break; + default: + return luaL_error(lua, "buffer must be a nil, string, userdata (FILE*)"); + } + + bool decode = true; + if (n == 3) { + luaL_checktype(lua, 3, LUA_TBOOLEAN); + decode = (bool)lua_toboolean(lua, 3); + } + + size_t pos_r = b->readpos; + size_t pos_s = b->scanpos; + size_t discarded = 0; + bool found = lsb_find_heka_message(&hsr->msg, b, decode, &discarded, NULL); + + size_t need = b->size; + if (found) { + lua_pushboolean(lua, 1); // found + lua_pushinteger(lua, b->scanpos - pos_s); // consumed + } else { + lua_pushboolean(lua, 0); // not found + if (b->readpos == 0) { + lua_pushinteger(lua, pos_r - pos_s); // consumed everything in the buf + } else { + lua_pushinteger(lua, b->scanpos - pos_s); + } + if (b->msglen) { + need = b->msglen + (size_t)b->buf[b->scanpos + 1] + LSB_HDR_FRAME_SIZE - + (b->readpos - b->scanpos); + } else { + need = b->scanpos + b->size - b->readpos; + } + } + + if (fh) { // update bytes read + if (found) { + lua_pushinteger(lua, 0); + } else { + if (lsb_expand_input_buffer(b, need)) { + return luaL_error(lua, "buffer reallocation failed\tname:%s", + hsr->name); + } + size_t nread = fread(b->buf + b->readpos, + 1, + b->size - b->readpos, + fh); + b->readpos += nread; + lua_pushnumber(lua, (lua_Number)nread); + } + } else { // update bytes needed + if (found && b->scanpos != b->readpos) need = 0; + lua_pushinteger(lua, need); + } + return 3; +} + + +static int hsr_read_message(lua_State *lua) +{ + int n = lua_gettop(lua); + if (n < 1 || n > 4) { + return luaL_error(lua, "read_message() incorrect number of arguments"); + } + heka_stream_reader *hsr = check_hsr(lua, n); + lua_remove(lua, 1); // remove the hsr user data + return heka_read_message(lua, &hsr->msg); +} + + +static int hsr_gc(lua_State *lua) +{ + heka_stream_reader *hsr = check_hsr(lua, 1); + free(hsr->name); + lsb_free_heka_message(&hsr->msg); + lsb_free_input_buffer(&hsr->buf); + return 0; +} + + +static const struct luaL_reg heka_stream_readerlib_m[] = +{ + { "find_message", hsr_find_message }, + { "decode_message", hsr_decode_message }, + { "read_message", hsr_read_message }, + { "__gc", hsr_gc }, + { NULL, NULL } +}; + + +int heka_create_stream_reader(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n == 1, 0, "incorrect number of arguments"); + size_t len; + const char *name = luaL_checklstring(lua, 1, &len); + luaL_argcheck(lua, len < 255, 1, "name is too long"); + + size_t nbytes = sizeof(heka_stream_reader); + heka_stream_reader *hsr = lua_newuserdata(lua, nbytes); + + if (luaL_newmetatable(lua, LSB_HEKA_STREAM_READER) == 1) { + lua_pushvalue(lua, -1); + lua_setfield(lua, -2, "__index"); + luaL_register(lua, NULL, heka_stream_readerlib_m); + } + lua_setmetatable(lua, -2); + + size_t mms = 0; + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + lua_getfield(lua, -1, LSB_HEKA_MAX_MESSAGE_SIZE); + mms = (size_t)lua_tointeger(lua, -1); + lua_pop(lua, 1); // remove limit + } else { + free(hsr); + return luaL_error(lua, LSB_CONFIG_TABLE " is missing"); + } + lua_pop(lua, 1); // remove config + + if (lsb_init_heka_message(&hsr->msg, 8)) { + free(hsr); + return luaL_error(lua, "failed to init the message struct"); + } + if (lsb_init_input_buffer(&hsr->buf, mms)) { + lsb_free_heka_message(&hsr->msg); + free(hsr); + return luaL_error(lua, "failed to init the input buffer"); + } + hsr->name = malloc(len + 1); + if (!hsr->name) { + lsb_free_input_buffer(&hsr->buf); + lsb_free_heka_message(&hsr->msg); + free(hsr); + return luaL_error(lua, "memory allocation failed"); + } + strcpy(hsr->name, name); + return 1; +} diff --git a/src/heka/test/CMakeLists.txt b/src/heka/test/CMakeLists.txt new file mode 100644 index 0000000..e38530e --- /dev/null +++ b/src/heka/test/CMakeLists.txt @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +configure_file(test.h.in test.h ESCAPE_QUOTES) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/heka") + +add_test(NAME test_move_heka_sandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) + +add_executable(test_heka_sandbox test_heka_sandbox.c) +target_link_libraries(test_heka_sandbox luasandboxheka) +add_test(NAME test_heka_sandbox COMMAND test_heka_sandbox) + +if(WIN32) + STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") + set_tests_properties(test_heka_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) +endif() diff --git a/src/heka/test/lua/aim.lua b/src/heka/test/lua/aim.lua new file mode 100644 index 0000000..e8a3b46 --- /dev/null +++ b/src/heka/test/lua/aim.lua @@ -0,0 +1,69 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +-- Table tests +local msgs = { + {Pid = 0, Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {Pid = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "ignore", Hostname = "spoof"}, +} + +local err_msgs = { + {err = "bad argument #1 to '?' (table expected, got nil)"}, +} + +for i, v in ipairs(msgs) do + inject_message(v) +end + +for i, v in ipairs(err_msgs) do + local ok, err = pcall(inject_message, v.msg) + if ok then error(string.format("test: %d should have failed", i)) end + assert(v.err == err, string.format("test: %d expected: %s received: %s", i, v.err, err)) +end + +add_to_payload("foo bar") +inject_payload() + +add_to_payload("foo") +inject_payload("dat", "test", " bar") + +local ok, err = pcall(add_to_payload, add_to_payload) +if ok then error("cannot output functions") end +local eerr = "bad argument #1 to '?' (unsupported type)" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_payload, "txt", "name", add_to_payload) +if ok then error("cannot output functions") end +eerr = "bad argument #3 to '?' (unsupported type)" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_payload, true, "name", "data") +assert(not ok) +assert("inject_payload() payload_type argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_payload, "txt", true, "data") +assert(not ok) +assert("inject_payload() payload_name argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_payload, nil) +assert(not ok) +assert("inject_payload() payload_type argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_payload, "", nil) +assert(not ok) +assert("inject_payload() payload_name argument must be a string" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_message, {Fields = {foo = {value = {"s", true}}}}) +assert(not ok) +assert("inject_message() failed: array has mixed types" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_message, {}) +assert(not ok) +assert("inject_message() failed: rejected by the callback rv: 99" == err, string.format("received: %s", err)) + +ok, err = pcall(inject_message, {}) +assert(not ok) +assert("inject_message() failed: injection limit exceeded" == err, string.format("received: %s", err)) diff --git a/src/heka/test/lua/analysis.lua b/src/heka/test/lua/analysis.lua new file mode 100644 index 0000000..f862d90 --- /dev/null +++ b/src/heka/test/lua/analysis.lua @@ -0,0 +1,38 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +assert(read_config) +assert(read_message) +assert(decode_message) +assert(inject_message) +assert(add_to_payload) +assert(inject_payload) +assert(not encode_message) +assert(not update_checkpoint) + +function process_message() + return 0 +end + +local work_cnt = 1 +function timer_event(ns, shutdown) + local cnt = 0 + for i=1, work_cnt * 1000 do + cnt = cnt + 1 + end + work_cnt = work_cnt + 1 + + if ns == 1e9 then + assert(shutdown, "not shutting down") + return + end + + local uuid = read_message("Uuid") + assert(not uuid) + + if ns == 2e9 then error("boom") end + + if shutdown then error("should not have a shutdown signal") end +end + diff --git a/src/heka/test/lua/decode_message.lua b/src/heka/test/lua/decode_message.lua new file mode 100644 index 0000000..037a563 --- /dev/null +++ b/src/heka/test/lua/decode_message.lua @@ -0,0 +1,218 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- minimal message, required fields only +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\000" +local msg = decode_message(test) +assert(msg.Timestamp == 0) + +-- message headers only +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\003\026\004type\034\006logger\040\009\050\007payload\058\011env_version\074\008hostname" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Type == "type") +assert(msg.Logger == "logger") +assert(msg.Payload == "payload") +assert(msg.EnvVersion == "env_version") +assert(msg.Hostname == "hostname") +assert(msg.Severity == 9) + +-- repeated unpacked doubles +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\027\010\005count\016\003\057\000\000\000\000\000\000\240\063\057\000\000\000\000\000\000\240\063" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 3) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated packed doubles +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\027\010\005count\016\003\058\016\000\000\000\000\000\000\240\063\000\000\000\000\000\000\240\063" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 3) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated unpacked ints +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\048\001\048\001" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 2) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated packed ints +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\050\002\001\001" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 2) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == 1) +assert(msg.Fields[1].value[2] == 1) + +-- repeated unpacked bools +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\004\064\001\064\000" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 4) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == true) +assert(msg.Fields[1].value[2] == false) + +-- repeated packed bools +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\004\066\002\001\000" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "count") +assert(msg.Fields[1].value_type == 4) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == true) +assert(msg.Fields[1].value[2] == false) + +-- repeated strings +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\023\010\005names\016\000\034\002s1\034\002s2\026\004keys" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "names") +assert(msg.Fields[1].representation == "keys") +assert(msg.Fields[1].value_type == 0) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == "s1") +assert(msg.Fields[1].value[2] == "s2") + +-- repeated bytes +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(msg.Fields[1].name == "names") +assert(msg.Fields[1].representation == "keys") +assert(msg.Fields[1].value_type == 1) +assert(#msg.Fields[1].value == 2) +assert(msg.Fields[1].value[1] == "s1") +assert(msg.Fields[1].value[2] == "s2") + +-- recent timestamp +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\152\141\135\236\222\200\233\19" +msg = decode_message(test) +assert(msg.Timestamp == 1428523950000000000) + +-- benchmark message +test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" +msg = decode_message(test) +assert(msg.Timestamp == 1e9) +assert(#msg.Fields == 6) + +assert(#msg.Fields[1].value == 1) +assert(msg.Fields[1].name == "number") +assert(msg.Fields[1].value_type == 3) +assert(msg.Fields[1].value[1] == 1) + +assert(msg.Fields[2].name == "numbers") +assert(msg.Fields[2].value_type == 3) +assert(#msg.Fields[2].value == 3) +assert(msg.Fields[2].value[1] == 1) +assert(msg.Fields[2].value[2] == 2) +assert(msg.Fields[2].value[3] == 3) +assert(msg.Fields[2].representation == "count") + +assert(#msg.Fields[3].value == 3) +assert(msg.Fields[3].name == "bools") +assert(msg.Fields[3].value[1]) +assert(msg.Fields[3].value[2] == false) +assert(msg.Fields[3].value[3] == false) + +assert(#msg.Fields[4].value == 1) +assert(msg.Fields[4].name == "bool") +assert(msg.Fields[4].value[1]) + +assert(#msg.Fields[5].value == 1) +assert(msg.Fields[5].name == "string") +assert(msg.Fields[5].value[1] == "string") + +assert(msg.Fields[6].name == "strings") +assert(#msg.Fields[6].value == 3) +assert(msg.Fields[6].value[1] =="s1") +assert(msg.Fields[6].value[2] =="s2") +assert(msg.Fields[6].value[3] =="s3") + +-- test negative varint headers +test = "\010\016\243\083\052\234\016\052\066\236\160\084\236\003\227\231\170\203\016\255\255\255\255\255\255\255\255\255\001\040\255\255\255\255\255\255\255\255\255\001\064\255\255\255\255\255\255\255\255\255\001" +msg = decode_message(test) +assert(msg.Timestamp == -1) +assert(msg.Pid == -1) +assert(msg.Severity == -1) + +-- multi byte length +test = "\010\016\104\183\062\106\120\227\073\248\173\168\122\019\057\236\124\025\016\128\148\235\220\003\082\141\001\010\006\115\116\114\105\110\103\034\130\001\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057\048\049\050\051\052\053\054\055\056\057" +msg = decode_message(test) +assert(msg.Fields[1].value[1] == "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789") + + +--[[ +Parse Errors +--]] + +-- too short +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "invalid message, too short", msg) + +-- no uuid +test = "\016\128\148\235\220\003\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "missing required field uuid: not found timestamp: found", msg) + +-- no timestamp +local test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\082\023\010\005names\016\001\042\002s1\042\002s2\026\004keys" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "missing required field uuid: found timestamp: not found", msg) + +-- missing field name +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\020\016\003\057\000\000\000\000\000\000\240\063\057\000\000\000\000\000\000\240\063" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 10 wiretype: 2 offset: 24", msg) + +-- random string +test = "this is a test item over twenty bytes long" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 14 wiretype: 4 offset: 0", msg) + +-- repeated packed with ints invalid length +test = "\010\016\111\021\235\034\090\107\077\120\169\175\058\232\153\002\231\132\016\128\148\235\220\003\082\013\010\005count\016\002\050\003\001\001" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 10 wiretype: 2 offset: 24", msg) + +-- invalid timestamp varint encoding +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\220\220\220\220\220\220" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 2 wiretype: 0 offset: 18", msg) + +-- invalid length encoding +local test = "\010\016\233\213\137\149\106\254\064\066\175\098\058\163\017\067\202\068\016\128\148\235\220\003\026\128\148\235\220\220\220\220\220\220\220type" +ok, msg = pcall(decode_message, test) +assert(not ok) +assert(msg == "error in tag: 3 wiretype: 2 offset: 24", msg) + + +function process_message() + msg = decode_message(read_message("raw", nil, nil, true)) + assert(msg.Timestamp == 1e9) + assert(#msg.Fields == 6) + return 0 +end diff --git a/src/heka/test/lua/decode_message_benchmark.lua b/src/heka/test/lua/decode_message_benchmark.lua new file mode 100644 index 0000000..c69da71 --- /dev/null +++ b/src/heka/test/lua/decode_message_benchmark.lua @@ -0,0 +1,10 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local test = "\010\016\096\006\214\155\119\188\078\023\172\076\081\127\129\143\250\040\016\128\148\235\220\003\082\019\010\006\110\117\109\098\101\114\016\003\057\000\000\000\000\000\000\240\063\082\044\010\007\110\117\109\098\101\114\115\016\003\026\005\099\111\117\110\116\058\024\000\000\000\000\000\000\240\063\000\000\000\000\000\000\000\064\000\000\000\000\000\000\008\064\082\014\010\005\098\111\111\108\115\016\004\066\003\001\000\000\082\010\010\004\098\111\111\108\016\004\064\001\082\016\010\006\115\116\114\105\110\103\034\006\115\116\114\105\110\103\082\021\010\007\115\116\114\105\110\103\115\034\002\115\049\034\002\115\050\034\002\115\051" + +function process_message() + local msg = decode_message(test) + return 0 +end diff --git a/src/heka/test/lua/encode_message.lua b/src/heka/test/lua/encode_message.lua new file mode 100644 index 0000000..b116249 --- /dev/null +++ b/src/heka/test/lua/encode_message.lua @@ -0,0 +1,102 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" +require "table" + +local msgs = { + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl@\000\074\002\sh" + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h"}, + rv = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\001l@\000\074\001\h" + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + rv = "\030\002\008\030\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\000\034\002sl@\000\074\002\sh", + framed = true, + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h", + Fields = { + number = 1, + numbers = {value = {1,2,3}, representation = "count"}, + string = "string", + strings = {"s1","s2","s3"}, + bool = true, + bools = {true,false,false}}}, + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108@\0\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0\82\10\10\4\98\111\111\108\16\4\64\1\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51" + }, + { + msg = {Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "l", Hostname = "h", + Fields = { + {name = "number" ,value = 1}, + {name = "numbers" ,value = {1,2,3}, representation="count"}, + {name = "string" ,value = "string"}, + {name = "strings" ,value = {"s1","s2","s3"}}, + {name = "bool" ,value = true}, + {name = "bools" ,value = {true,false,false}}} + }, + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\1\108@\0\74\1\104\82\19\10\6\110\117\109\98\101\114\16\3\57\0\0\0\0\0\0\240\63\82\44\10\7\110\117\109\98\101\114\115\16\3\26\5\99\111\117\110\116\58\24\0\0\0\0\0\0\240\63\0\0\0\0\0\0\0\64\0\0\0\0\0\0\8\64\82\16\10\6\115\116\114\105\110\103\34\6\115\116\114\105\110\103\82\21\10\7\115\116\114\105\110\103\115\34\2\115\49\34\2\115\50\34\2\115\51\82\10\10\4\98\111\111\108\16\4\64\1\82\14\10\5\98\111\111\108\115\16\4\66\3\1\0\0" + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = "value", value_type = 0, representation = "widget"}}}, -- string + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\20\10\3\107\101\121\26\6\119\105\100\103\101\116\34\5\118\97\108\117\101", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = "value", value_type = 1, representation = "widget"}}}, -- bytes + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\22\10\3\107\101\121\16\1\26\6\119\105\100\103\101\116\42\5\118\97\108\117\101", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = 34, value_type = 2, representation = "widget"}}}, -- int + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\17\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\48\34", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = 34, value_type = 3, representation = "widget"}}}, -- double + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\24\10\3\107\101\121\16\3\26\6\119\105\100\103\101\116\57\0\0\0\0\0\0\65\64", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = true, value_type = 4, representation = "widget"}}}, -- bool + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\17\10\3\107\101\121\16\4\26\6\119\105\100\103\101\116\64\1", + }, + { + msg = {Timestamp = 0, Uuid = string.rep("\0", 16), Fields = {key = {value = {1,2,3}, value_type = 2, representation = "widget"}}}, -- int + rv = "\10\16\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\16\0\34\2\115\108@\0\74\2\115\104\82\20\10\3\107\101\121\16\2\26\6\119\105\100\103\101\116\50\3\1\2\3", + }, +} + +for i, v in ipairs(msgs) do + local rv + if v.framed then + rv = encode_message(v.msg, v.framed) + else + rv = encode_message(v.msg) + end + + if v.rv ~= rv then + --local et = {string.byte(v.rv, 1, -1)} + local rt = {string.byte(rv, 1, -1)} + assert(v.rv == rv, string.format("test: %d received: %s", i, table.concat(rt, " "))) + end + +end + +local ok, err = pcall(encode_message, "foo") +if ok then error("encode_message should not accept a string") end +local eerr = "bad argument #1 to '?' (table expected, got string)" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(encode_message, "", true, "extra") +assert(not ok) +assert("bad argument #3 to '?' (incorrect number of arguments)" == err, string.format("received: %s", err)) + +ok, err = pcall(encode_message, {Fields = {foo = {value = {"s", true}}}}) +assert(not ok) +assert("encode_message() failed: array has mixed types" == err, string.format("received: %s", err)) + +ok, err = pcall(encode_message, {Fields = { {noname = "foo", value = 1}} }) +assert(not ok) +assert("encode_message() failed: field name must be a string" == err, string.format("received: %s", err)) diff --git a/src/heka/test/lua/iim.lua b/src/heka/test/lua/iim.lua new file mode 100644 index 0000000..5264787 --- /dev/null +++ b/src/heka/test/lua/iim.lua @@ -0,0 +1,138 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" +require "io" + +-- Table tests +local msgs = { + {{Timestamp = 0, Uuid = "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000"}, nil}, + {{Timestamp = 1, Uuid = "00000000-0000-0000-0000-000000000000"}, nil}, + {{Timestamp = 2, Uuid = "00000000-0000-0000-0000-000000000000", Logger = "logger", Hostname = "hostname", Type = "type", Payload = "payload", EnvVersion = "envversion", Pid = 99, Severity = 5}, 99}, + {{Timestamp = 3, Uuid = "00000000-0000-0000-0000-000000000000", Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}}, "foo.log:123"}, + {nil, "foo.log:123"}, + {nil, 123}, +} + +local err_msgs = { + {msg = {Timestamp = 1e9, Fields = {counts={2,"ten",4}}}, err = "inject_message() failed: array has mixed types"}, + {msg = {Timestamp = 1e9, Fields = {counts={}}}, err = "inject_message() failed: unsupported type: nil"}, + {err = "inject_message() message cannot be nil without a checkpoint update"}, + {msg = {Timestamp = 1e9, Fields = {counts={{1},{2}}}}, err = "inject_message() failed: unsupported array type: table"}, + {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=2}}}, err = "inject_message() failed: invalid string value_type: 2"}, + {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=3}}}, err = "inject_message() failed: invalid string value_type: 3"}, + {msg = {Timestamp = 1e9, Fields = {counts={value="s", value_type=4}}}, err = "inject_message() failed: invalid string value_type: 4"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=1, value_type=0}}}, err = "inject_message() failed: invalid numeric value_type: 0"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=1, value_type=1}}}, err = "inject_message() failed: invalid numeric value_type: 1"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=1, value_type=4}}}, err = "inject_message() failed: invalid numeric value_type: 4"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=0}}}, err = "inject_message() failed: invalid boolean value_type: 0"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=1}}}, err = "inject_message() failed: invalid boolean value_type: 1"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, + {msg = {Timestamp = 1e9, Fields = {counts={value=true, value_type=2}}}, err = "inject_message() failed: invalid boolean value_type: 2"}, + {msg = {Timestamp = 1e9, Fields = {bf = {value=io.stdin, representation="bf"}}}, err = "inject_message() failed: userdata object does not implement lsb_output"}, +} + +for i, v in ipairs(msgs) do + local ok, err = pcall(inject_message, v[1], v[2]) + assert(ok, string.format("test: %d err: %s", i, tostring(err))) +end + +for i, v in ipairs(err_msgs) do + local ok, err = pcall(inject_message, v.msg) + if ok then error(string.format("test: %d should have failed", i)) end + assert(v.err == err, string.format("test: %d expected: %s received: %s", i, v.err, err)) +end + +-- Stream Reader tests +require "io" +local hsr = create_stream_reader("test") +ok, err = pcall(inject_message, hsr) +local eerr = "inject_message() attempted to inject a nil message" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) +ok, err = pcall(hsr.decode_message, hsr, true) +assert(not ok) +assert("buffer must be string" == err, string.format("received: %s", err)) +ok, err = pcall(hsr.decode_message, hsr, "") +assert(not ok) +assert("empty protobuf string" == err, string.format("received: %s", err)) + +hsr:decode_message("\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\004") +inject_message(hsr) +local ts = hsr:read_message("Timestamp") +assert(4 == ts, string.format("received: %g", ts)) + +local framed = "\030\002\008\020\031\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\006" +local fh = assert(io.open("hekamsg.pb", "w+")) +fh:write(framed) +fh:seek("set") +local found, consumed, read = hsr:find_message(fh) +assert(not found) +assert(0 == consumed, string.format("expected: 0 received %d", consumed)) +assert(25 == read, string.format("expected: 25 received: %d", read)) +found, consumed, read = hsr:find_message(fh) +assert(found) +assert(25 == consumed, string.format("expected: 25 received %d", consumed)) +assert(0 == read, string.format("expected: >0 received: %d", read)) +fh:close() + +local found, consumed, need = hsr:find_message(framed) +assert(true == found) +assert(25 == consumed, string.format("expected: 25 received %d", consumed)) +assert(0 < need, string.format("expected: >0 received: %d", need)) + +found, consumed, need = hsr:find_message("\0\0\0\0\0\030\002\008\020\031") +assert(false == found) +assert(5 == consumed, string.format("expected: 5 received %d", consumed)) +assert(20 == need, string.format("expected: 20 received: %d", need)) + +local found, consumed, need = hsr:find_message("") +assert(not found) + +ok, err = pcall(hsr.find_message, hsr, assert) +assert(not ok, "accepted function as second arg") + +ok, err = pcall(hsr.find_message, hsr, nil, "str") +assert(not ok, "accepted string as third arg") + +ok, err = pcall(hsr.find_message, hsr, hsr) +assert(not ok, "accepted non FILE userdata") + +fh = assert(io.open("lua/iim.lua")) +found, consumed, read = hsr:find_message(fh) +fh:close() +assert(false == found) +assert(0 == consumed, string.format("expected: 0 received %d", consumed)) +assert(0 < need, string.format("expected: >0 received: %d", need)) + +ok, err = pcall(hsr.read_message) +assert(not ok) +assert("read_message() incorrect number of arguments" == err, string.format("received: %s", err)) + +ok, err = pcall(hsr.read_message, hsr, 1, 2, 3, 4) +assert(not ok) +assert("read_message() incorrect number of arguments" == err, string.format("received: %s", err)) + +ok, err = pcall(hsr.read_message, 1, 2) +assert(not ok) +assert("bad argument #1 to '?' (lsb.heka_stream_reader expected, got number)" == err, string.format("received: %s", err)) + + +-- String tests +local pbstr = "\010\016\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\016\005" +inject_message(pbstr) + +ok, err = pcall(inject_message, pbstr, 2) +if ok then error(string.format("test should have failed")) end +eerr = "inject_message() failed: checkpoint update" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_message, "\000") +if ok then error(string.format("string test should have failed")) end +local eerr = "inject_message() attempted to inject a invalid protobuf string" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) + +ok, err = pcall(inject_message, {}) +if ok then error(string.format("test should have failed")) end +eerr = "inject_message() failed: rejected by the callback rv: 1" +assert(eerr == err, string.format("expected: %s received: %s", eerr, err)) diff --git a/src/heka/test/lua/input.lua b/src/heka/test/lua/input.lua new file mode 100644 index 0000000..8a8aa7e --- /dev/null +++ b/src/heka/test/lua/input.lua @@ -0,0 +1,58 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +assert(read_config) +assert(decode_message) +assert(inject_message) +assert(create_stream_reader) +assert(is_running) +assert(not read_message) +assert(not encode_message) +assert(not update_checkpoint) +assert(not add_to_payload) +assert(not inject_payload) + +require "string" +msg = {Timestamp = 8} + +local work_cnt = 1 +local function add_work() + local cnt = 0 + for i = 1, work_cnt * 1000 do + cnt = cnt + 1 + end + work_cnt = work_cnt + 1 +end + +function process_message(cp) + if cp == 0 then + add_work() + return -2, "host specific failure" + elseif cp == 1 then + add_work() + return -1, "failed" + elseif cp == 2 then + add_work() + return 0, "ok" + elseif cp == 3 then + error("boom") + elseif cp == 4 then + return 0, msg + elseif cp == 5 then + return msg + elseif cp == 6 then + error(string.rep("a", 255)) + elseif cp == "string" then + add_work() + return 0, "string" + elseif cp == 7 then + error(nil) + elseif cp == 8 then + assert(is_running(), "running") + elseif cp == 9 then + assert(not is_running(), "not running") + end + add_work() + return 0, "no cp" +end diff --git a/src/heka/test/lua/oim.lua b/src/heka/test/lua/oim.lua new file mode 100644 index 0000000..68435a9 --- /dev/null +++ b/src/heka/test/lua/oim.lua @@ -0,0 +1,25 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +-- Table tests +local msgs = { + {Timestamp = 1, Uuid = "\001\002\003\000\000\000\000\000\000\000\000\000\000\000\000\000"}, + {Pid = 2, Timestamp = 3, Uuid = "\004\005\006\000\000\000\000\000\000\000\000\000\000\000\000\000", Logger = "ignore", Hostname = "spoof"}, +} + +local err_msgs = { + {err = "bad argument #1 to '?' (table expected, got nil)"}, +} + +for i, v in ipairs(msgs) do + inject_message(v) +end + +for i, v in ipairs(err_msgs) do + local ok, err = pcall(inject_message, v.msg) + if ok then error(string.format("test: %d should have failed", i)) end + assert(v.err == err, string.format("test: %d expected: %s received: %s", i, v.err, err)) +end diff --git a/src/heka/test/lua/output.lua b/src/heka/test/lua/output.lua new file mode 100644 index 0000000..deb8869 --- /dev/null +++ b/src/heka/test/lua/output.lua @@ -0,0 +1,59 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +local sid + +assert(read_config) +assert(read_message) +assert(decode_message) +assert(encode_message) +assert(update_checkpoint) +assert(create_message_matcher) +assert(not inject_message) +assert(not add_to_payload) +assert(not inject_payload) + +function process_message(sequence_id) + if not sequence_id then + update_checkpoint() + local ok, err = pcall(update_checkpoint, 1, 2, 3) + assert(not ok) + return 0 + else + if not sid then + update_checkpoint(sequence_id, 7) + update_checkpoint(sequence_id) + local ok, err = pcall(update_checkpoint, true) + assert(not ok) + ok, err = pcall(update_checkpoint, sequence_id, "foo") + assert(not ok) + sid = sequence_id + return -5 + else + local ok, mm = pcall(create_message_matcher, "Type == 'type'") + assert(ok, mm) + assert(mm:eval()) + + ok, mm = pcall(create_message_matcher, "Type == 'foo'") + assert(ok, mm) + assert(not mm:eval()) + + ok, err = pcall(create_message_matcher, "Widget == 'foo'") + assert(not ok) + + ok, err = pcall(update_checkpoint, sequence_id) + assert(not ok) + + return -3 -- retry + end + end + return 0 +end + +function timer_event(ns, shutdown) + local ok, mm = pcall(create_message_matcher, "Type == 'foo'"); + assert(ok, mm) + ok, err = pcall(mm.eval, mm) + assert(err == "no active message") +end diff --git a/src/test/lua/serialize_noglobal.lua b/src/heka/test/lua/pm_no_return.lua similarity index 86% rename from src/test/lua/serialize_noglobal.lua rename to src/heka/test/lua/pm_no_return.lua index 37c87b4..aa5c3dc 100644 --- a/src/test/lua/serialize_noglobal.lua +++ b/src/heka/test/lua/pm_no_return.lua @@ -2,4 +2,6 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -_G = nil +function process_message() +end + diff --git a/src/heka/test/lua/read_message.lua b/src/heka/test/lua/read_message.lua new file mode 100644 index 0000000..a205593 --- /dev/null +++ b/src/heka/test/lua/read_message.lua @@ -0,0 +1,72 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +local tests = { + {"Timestamp", 1e9}, + {"Uuid", "abcdefghijklmnop"}, + {"Type", "type"}, + {"Logger", "logger"}, + {"Payload", "payload"}, + {"EnvVersion", "env_version"}, + {"Hostname", "hostname"}, + {"Severity", 9}, + {"Pid", nil}, + {"raw", 208}, + {"size", 208}, + {"framed", 214}, + {"Fields[string]", "string"}, + {"Fields[notfound]", nil}, +} + +local fields = { + {{"Fields[number]" , 0 , 0 }, 1}, + {{"Fields[number]" , 0 , 0, false }, 1}, + {{"Fields[numbers]" , 0 , 2 }, 3}, + {{"Fields[bool]" , 0 , 0 }, true}, + {{"Fields[bools]" , 0 , 1 }, false}, + {{"Fields[strings]" , 0 , 0 }, "s1"}, + {{"Fields[strings]" , 0 , 2 }, "s3"}, + {{"Fields[strings]" , 10, 0 }, nil}, + {{"Fields[strings]" , 0 , 10}, nil}, + {{"Fields[notfound]" , 0 , 0 }, nil}, +} + +local errors = { + {"Fields[string"}, + {"morethan8"}, + {"lt8"}, + {"Fields[string]", -1, 0}, + {"Fields[string]", 0, -1}, + {"Fields[string]", "a", 0}, + {"Fields[string]", 0, "a"}, + {"Fields[string]", 0, 0, "str"}, +} + + +function process_message() + for i, v in ipairs(tests) do + local r = read_message(v[1]) + if v[1] == "raw" or v[1] == "framed" then + assert(v[2] == string.len(r), string.format("test: %d expected: %d received: %d", i, string.len(v[2]), string.len(r))) + elseif v[1] == "size" then + assert(v[2] == r, string.format("test: %d expected: %d received: %d", i, v[2], r)) + else + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + end + + for i, v in ipairs(fields) do + local r = read_message(unpack(v[1])) + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + + for i, v in ipairs(errors) do + local ok, r = pcall(read_message, unpack(v)) + assert(not ok, string.format("test: %d should have errored", i)) + end + + return 0 +end diff --git a/src/heka/test/lua/read_message_zc.lua b/src/heka/test/lua/read_message_zc.lua new file mode 100644 index 0000000..2b5b14a --- /dev/null +++ b/src/heka/test/lua/read_message_zc.lua @@ -0,0 +1,83 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +require "string" + +local tests = { + {read_message("Uuid", 0, 0, true), "abcdefghijklmnop"}, + {read_message("Type", nil, nil, true), "type"}, + {read_message("Logger", 0, 0, true), "logger"}, + {read_message("Payload", 0, 0, true), "payload"}, + {read_message("EnvVersion", 0, 0, true), "env_version"}, + {read_message("Hostname", 0, 0, true), "hostname"}, + {read_message("Fields[string]" , 0 , 0 , true), "string"}, + {read_message("Fields[notfound]", 0 , 0 , true), nil}, + {read_message("Fields[strings]" , 0 , 0 , true), "s1"}, + {read_message("Fields[strings]" , 0 , 2 , true), "s3"}, + {read_message("Fields[strings]" , 10, 0 , true), nil}, + {read_message("Fields[strings]" , 0 , 10, true), nil}, + {read_message("Fields[notfound]", 0 , 0 , true), nil}, +} + +local tests_size = { + {read_message("raw", 0 , 0 , true), 208}, + {read_message("framed", 0 , 0 , true), 214}, +} + +local creation_errors = { + {"Timestamp", 0, 0, true}, + {"Severity", 0, 0, true}, + {"Pid", 0, 0, true}, + {"size", 0, 0, true}, + {"Fields[string", 0, 0, true}, + {"morethan8", 0, 0, true}, + {"lt8", 0, 0, true}, + {"Fields[string]", -1, 0, true}, + {"Fields[string]", 0, -1, true}, + {"Fields[string]", "a", 0, true}, + {"Fields[string]", 0, "a", true}, + {"Fields[string]", 0, 0, true, 0}, +} + +local fetch_errors = { + read_message("Fields[bool]", 0, 0, true), + read_message("Fields[number]", 0, 0, true), +} + +function process_message() + for i, v in ipairs(tests) do + local r = read_message_zc(v[1]) + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + + for i, v in ipairs(tests) do + local r = tostring(v[1]) + assert(v[2] == r, string.format("test: %d expected: %s received: %s", i, tostring(v[2]), tostring(r))) + end + + add_to_payload(read_message("Uuid", 0, 0, true)) + add_to_payload(read_message("framed", 0, 0, true)) + inject_payload() + + for i, v in ipairs(tests_size) do + local r = read_message_zc(v[1]) + assert(v[2] == #r, string.format("test: %d expected: %d received: %d", i, tostring(v[2]), tostring(#r))) + end + + for i, v in ipairs(tests_size) do + local r = tostring(v[1]) + assert(v[2] == #r, string.format("test: %d expected: %d received: %d", i, tostring(v[2]), tostring(#r))) + end + + for i, v in ipairs(creation_errors) do + local ok, r = pcall(read_message, unpack(v)) + assert(not ok, string.format("test: %d creation should have errored", i)) + end + + for i, v in ipairs(fetch_errors) do + local ok, r = pcall(read_message_zc, v) + assert(not ok, string.format("test: %d fetch should have errored", i)) + end + return 0 +end diff --git a/src/heka/test/test.h.in b/src/heka/test/test.h.in new file mode 100644 index 0000000..688064e --- /dev/null +++ b/src/heka/test/test.h.in @@ -0,0 +1,28 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Unit test setup @file */ + +#ifndef test_h_ +#define test_h_ + +#include "luasandbox/test/mu_test.h" + +#include +#include +#include +#include + +#define TEST_LUA_PATH "${CMAKE_SOURCE_DIR}/modules/?.lua" +#define TEST_BASE_CPATH "${CMAKE_BINARY_DIR}/ep_base/inst/lib/luasandbox/" + +#ifdef _WIN32 +#define TEST_LUA_CPATH TEST_BASE_CPATH "modules/?.dll;" TEST_BASE_CPATH "io_modules/?.dll" +#else +#define TEST_LUA_CPATH TEST_BASE_CPATH "modules/?.so;" TEST_BASE_CPATH "io_modules/?.so" +#endif + +#endif diff --git a/src/heka/test/test_heka_sandbox.c b/src/heka/test/test_heka_sandbox.c new file mode 100644 index 0000000..a69f363 --- /dev/null +++ b/src/heka/test/test_heka_sandbox.c @@ -0,0 +1,894 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Heka sandbox unit tests @file */ + +#include "test.h" + +#include +#include +#include +#include +#include +#include + +#include "../sandbox_impl.h" +#include "luasandbox/heka/sandbox.h" +#include "luasandbox_output.h" + +static unsigned long long clockres = 1; + +// {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} +static char pb[] = "\x0a\x10" "abcdefghijklmnop" "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; + +char *e = NULL; + +void dlog(void *context, const char *component, int level, const char *fmt, ...) +{ + (void)context; + va_list args; + va_start(args, fmt); + fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, + component ? component : "unnamed"); + vfprintf(stderr, fmt, args); + fwrite("\n", 1, 1, stderr); + va_end(args); +} +static lsb_logger logger = { .context = NULL, .cb = dlog }; + + +static int iim(void *parent, const char *pb, size_t pb_len, double cp_numeric, + const char *cp_string) +{ + if (!pb) { + return LSB_HEKA_IM_SUCCESS; + } + + static int cnt = 0; + struct im_result { + const char *pb; + size_t pb_len; + double cp_numeric; + const char *cp_string; + }; + + struct im_result results[] = { + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x00\x22\x03iim\x40\x00\x4a\x02hn", .pb_len = 31, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03iim\x40\x00\x4a\x02hn", .pb_len = 31, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x02\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x05\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0a\x65\x6e\x76\x76\x65\x72\x73\x69\x6f\x6e\x40\x63\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65", .pb_len = 69, .cp_numeric = 99, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x03iim\x40\x00\x4a\x02hn\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33", .pb_len = 167, .cp_numeric = NAN, .cp_string = "foo.log:123" }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x04", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x05", .pb_len = 20, .cp_numeric = NAN, .cp_string = NULL }, + }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return LSB_HEKA_IM_ERROR; + } + + if (parent) { + fprintf(stderr, "test: %d parent set\n", cnt); + } + + if (pb_len != results[cnt].pb_len) { + fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" + PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); + return 1; + } + + if (memcmp(pb, results[cnt].pb, pb_len)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = 0; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); + } + fprintf(stderr, "\nreceived: "); + for (size_t i = 0; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; + } + + bool ncp_failed = false; + if (isnan(results[cnt].cp_numeric)) { + if (!isnan(cp_numeric)) { + ncp_failed = true; + } + } else if (results[cnt].cp_numeric != cp_numeric) { + ncp_failed = true; + } + if (ncp_failed) { + fprintf(stderr, "test: %d cp_numeric expected: %g received: %g\n", cnt, + results[cnt].cp_numeric, cp_numeric); + return LSB_HEKA_IM_CHECKPOINT; + } + + bool ncs_failed = false; + if (!results[cnt].cp_string) { + if (cp_string) { + ncs_failed = true; + } + } else if (!cp_string || strcmp(results[cnt].cp_string, cp_string)) { + ncs_failed = true; + } + if (ncs_failed) { + fprintf(stderr, "test: %d cp_string expected: %s received: %s\n", cnt, + results[cnt].cp_string ? results[cnt].cp_string : "NULL", + cp_string ? cp_string : "NULL"); + return LSB_HEKA_IM_CHECKPOINT; + } + cnt++; + return LSB_HEKA_IM_SUCCESS; +} + + +static int aim(void *parent, const char *pb, size_t pb_len) +{ + static int offset = 28; // skip Uuid and Timestamp + static int cnt = 0; + struct im_result { + const char *pb; + size_t pb_len; + }; + + struct im_result results[] = { + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x8e\xa8\xf3\xde\x88\xb5\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 44}, + { .pb = "\x0a\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\xbf\x97\x9c\xcc\xbe\xc2\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x00\x00\x00\x00\x0a\x10\x00\x00", .pb_len = 44}, + { .pb = "\x0a\x10\xea\x95\xd4\xfc\x7c\x10\x40\x95\xa8\x17\xcb\x56\x26\x91\x8c\x47\x10\xba\xf6\xd5\xf9\xfd\xce\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x74\x78\x74", .pb_len = 90}, + { .pb = "\x0a\x10\xfd\x49\x92\x77\x02\x37\x4b\xf0\xaf\x86\x6f\x9b\x80\x26\xf4\x35\x10\xaf\xec\x9e\xa4\xd8\xcf\xb7\x93\x15\x1a\x0e\x69\x6e\x6a\x65\x63\x74\x5f\x70\x61\x79\x6c\x6f\x61\x64\x22\x03\x61\x69\x6d\x32\x07\x66\x6f\x6f\x20\x62\x61\x72\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d\x52\x13\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x74\x79\x70\x65\x22\x03\x64\x61\x74\x52\x14\x0a\x0c\x70\x61\x79\x6c\x6f\x61\x64\x5f\x6e\x61\x6d\x65\x22\x04\x74\x65\x73\x74", .pb_len = 112}, + { .pb = "\x0a\x10\x7c\x32\xd6\x23\x98\xe8\x49\x9e\xa2\xe8\x0d\x78\x84\x8e\x75\xb2\x10\xf7\xf5\xdb\x89\x88\xe4\xb7\x93\x15\x22\x03\x61\x69\x6d\x40\x00\x4a\x07\x66\x6f\x6f\x2e\x63\x6f\x6d", .pb_len = 0}, }; // intentionally fail on size to to test the custom return value + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return LSB_HEKA_IM_LIMIT; + } + + if (parent) { + fprintf(stderr, "test: %d parent set\n", cnt); + } + + if (pb_len != results[cnt].pb_len) { + fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" + PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); + cnt++; + return 99; + } + + if (memcmp(pb + offset, results[cnt].pb + offset, pb_len - offset)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = offset; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); + } + fprintf(stderr, "\nreceived: "); + for (size_t i = offset; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; + } + cnt++; + return LSB_HEKA_IM_SUCCESS; +} + +static int aim1(void *parent, const char *pb, size_t pb_len) +{ + if (parent) { + fprintf(stderr, "parent set\n"); + return 1; + } + if (!pb) { + fprintf(stderr, "pb null set\n"); + return 1; + } + size_t expected = 308; + if (pb_len != expected) { + fprintf(stderr, "pb_len expected: %" PRIuSIZE " received: %" PRIuSIZE "\n", + expected, pb_len); + + for (size_t i = 0; i < pb_len; ++i) { + if (isprint(pb[i])) { + fprintf(stderr, "%c", pb[i]); + } else { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + } + fprintf(stderr, "\n"); + return 1; + } + return LSB_HEKA_IM_SUCCESS; +} + + +static int ucp(void *parent, void *sequence_id) +{ + static int cnt = 0; + if (parent) return 1; + void *results[] = { NULL, (void *)99, (void *)99 }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return 1; + } + + if (results[cnt] != sequence_id) { + fprintf(stderr, "expected: %p received: %p\n", results[cnt], sequence_id); + return 1; + } + cnt++; + return 0; +} + + +static int oim(void *parent, const char *pb, size_t pb_len) +{ + static int cnt = 0; + struct im_result { + const char *pb; + size_t pb_len; + }; + + struct im_result results[] = { + { .pb = "\x0a\x10\x01\x02\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x01\x22\x03\x6f\x69\x6d\x40\x00\x4a\x04\x74\x65\x73\x74", .pb_len = 33 }, + { .pb = "\x0a\x10\x04\x05\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10\x03\x22\x06\x69\x67\x6e\x6f\x72\x65\x40\x02\x4a\x05\x73\x70\x6f\x6f\x66", .pb_len = 37 }, }; + + if (cnt >= (int)(sizeof results / sizeof results[0])) { + fprintf(stderr, "tests and results are mis-matched\n"); + return LSB_HEKA_IM_LIMIT; + } + + if (parent) { + fprintf(stderr, "test: %d parent set\n", cnt); + } + + if (pb_len != results[cnt].pb_len) { + fprintf(stderr, "test: %d pb len expected: %" PRIuSIZE " received: %" + PRIuSIZE "\n", cnt, results[cnt].pb_len, pb_len); + cnt++; + return 99; + } + + if (memcmp(pb, results[cnt].pb, pb_len)) { + fprintf(stderr, "test: %d\nexpected: ", cnt); + for (size_t i = 0; i < results[cnt].pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", results[cnt].pb[i]); + } + fprintf(stderr, "\nreceived: "); + for (size_t i = 0; i < pb_len; ++i) { + fprintf(stderr, "\\x%02hhx", pb[i]); + } + fprintf(stderr, "\n"); + return 1; + } + cnt++; + return LSB_HEKA_IM_SUCCESS; +} + + +static char* test_api_assertion() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + + lsb_heka_sandbox * isb,*asb,*osb; + isb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); + asb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); + osb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, &logger, ucp); + + mu_assert(isb, "lsb_heka_create_input failed"); + mu_assert_rv(1, lsb_heka_pm_input(NULL, 0, NULL, false)); + mu_assert_rv(1, lsb_heka_timer_event(isb, 0, false)); + + mu_assert(isb, "lsb_heka_create_analysis failed"); + mu_assert_rv(1, lsb_heka_pm_analysis(NULL, NULL, false)); + mu_assert_rv(1, lsb_heka_pm_analysis(asb, NULL, false)); + mu_assert_rv(1, lsb_heka_timer_event(NULL, 0, false)); + mu_assert_rv(1, lsb_heka_pm_analysis(isb, &m, false)); + + mu_assert(isb, "lsb_heka_create_output failed"); + mu_assert_rv(1, lsb_heka_pm_output(NULL, NULL, NULL, false)); + mu_assert_rv(1, lsb_heka_pm_output(osb, NULL, NULL, false)); + mu_assert_rv(1, lsb_heka_pm_output(asb, &m, NULL, false)); + + lsb_heka_destroy_sandbox(isb); + lsb_heka_destroy_sandbox(asb); + lsb_heka_destroy_sandbox(osb); + lsb_free_heka_message(&m); + + mu_assert(strcmp(lsb_heka_get_error(NULL), "") == 0, "not empty"); + mu_assert(lsb_heka_get_lua_file(NULL) == NULL, "not null"); + lsb_heka_stats stats = lsb_heka_get_stats(NULL); + mu_assert(0 == stats.mem_cur, "received: %llu", stats.mem_cur); + mu_assert(!lsb_heka_is_running(NULL), "running is true"); + int state = lsb_heka_get_state(NULL); + mu_assert(LSB_UNKNOWN == state, "received: %d", state); + return NULL; +} + + +static char* test_create_input_sandbox() +{ + static const char *lua_file = "lua/input.lua"; + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + const char *lfn = lsb_heka_get_lua_file(hsb); + mu_assert(strcmp(lua_file, lfn) == 0, "expected %s received %s", lua_file, + lfn); + e = lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_input(NULL, "notfourd.lua", NULL, NULL, NULL, + iim); + mu_assert(!hsb, "lsb_heka_create_input succeeded"); + + hsb = lsb_heka_create_input(NULL, NULL, NULL, NULL, NULL, iim); + mu_assert(!hsb, "lsb_heka_create_input succeeded"); + + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, NULL, NULL); + mu_assert(!hsb, "lsb_heka_create_input succeeded"); + e = lsb_heka_destroy_sandbox(hsb); // test NULL + + hsb = lsb_heka_create_input(NULL, lua_file, NULL, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + lsb_heka_terminate_sandbox(hsb, "boom"); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp("boom", err) == 0, "received: %s", err); + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "received %s", e); + return NULL; +} + + +static char* test_create_analysis_sandbox() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + e = lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_analysis(NULL, "notfound.lua", NULL, NULL, NULL, aim); + mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); + + hsb = lsb_heka_create_analysis(NULL, NULL, NULL, NULL, NULL, aim); + mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); + + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, NULL, + NULL); + mu_assert(!hsb, "lsb_heka_create_analysis succeeded"); + return NULL; +} + + +static char* test_create_output_sandbox() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, &logger, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_output(NULL, "notfound.lua", NULL, NULL, NULL, ucp); + mu_assert(!hsb, "lsb_heka_create_output succeeded"); + + hsb = lsb_heka_create_output(NULL, NULL, NULL, NULL, NULL, ucp); + mu_assert(!hsb, "lsb_heka_create_output succeeded"); + + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, NULL, NULL, NULL); + mu_assert(!hsb, "lsb_heka_create_output succeeded"); + return NULL; +} + + +static char* test_timer_event() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(0 < stats.mem_cur, "received %llu", stats.mem_cur); + mu_assert(0 < stats.mem_max, "received %llu", stats.mem_max); + mu_assert(0 == stats.out_max, "received %llu", stats.out_max); + mu_assert(0 < stats.ins_max, "received %llu", stats.ins_max); + mu_assert(0 == stats.pm_cnt, "received %llu", stats.pm_cnt); + mu_assert(0 == stats.pm_failures, "received %llu", stats.pm_failures); + mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); + mu_assert(0 == stats.te_avg, "received %g", stats.te_avg); + mu_assert(0 == stats.te_sd, "received %g", stats.te_sd); + mu_assert(true == lsb_heka_is_running(hsb), "not running"); + int state = lsb_heka_get_state(hsb); + mu_assert(LSB_RUNNING == state, "received: %d", state); + + mu_assert(0 == lsb_heka_timer_event(hsb, 0, false), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(0 == lsb_heka_timer_event(hsb, 1, true), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(1 == lsb_heka_timer_event(hsb, 2, false), "err: %s", + lsb_heka_get_error(hsb)); + mu_assert(false == lsb_heka_is_running(hsb), "not running"); + state = lsb_heka_get_state(hsb); + mu_assert(LSB_TERMINATED == state, "received: %d", state); + + stats = lsb_heka_get_stats(hsb); + mu_assert(0 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(0 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(0 == stats.pm_cnt, "received %llu", stats.pm_cnt); + mu_assert(0 == stats.pm_failures, "received %llu", stats.pm_failures); + mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); + if (clockres <= 100) { + mu_assert(0 < stats.te_avg, "received %g res %llu", stats.te_avg, clockres); + mu_assert(0 < stats.te_sd, "received %g", stats.te_sd); + } + + e = lsb_heka_destroy_sandbox(hsb); + + hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, NULL, aim); + mu_assert(hsb, "lsb_heka_create_analysis succeeded"); + mu_assert_rv(1, lsb_heka_timer_event(hsb, 0, false)); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp("timer_event() function was not found", err) == 0, + "received: %s", err); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_clean_stop_input() +{ + static const char *state_file = "stop.data"; + remove(state_file); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + + mu_assert(true == lsb_heka_is_running(hsb), "running"); + int rv = lsb_heka_pm_input(hsb, 8, NULL, true); + const char *err = lsb_heka_get_error(hsb); + mu_assert(rv == 0, "error: %s expected: %d received: %d", err, 0, rv); + mu_assert(true == lsb_heka_is_running(hsb), "running"); + + lsb_heka_stop_sandbox_clean(hsb); + rv = lsb_heka_pm_input(hsb, 9, NULL, true); + err = lsb_heka_get_error(hsb); + mu_assert(rv == 0, "error: %s expected: %d received: %d", err, 0, rv); + mu_assert(false == lsb_heka_is_running(hsb), "not running"); + + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "received %s", e); + return NULL; +} + + +static char* test_stop_input() +{ + static const char *state_file = "stop.data"; + remove(state_file); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", state_file, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + lsb_heka_stop_sandbox(hsb); + e = lsb_heka_destroy_sandbox(hsb); + mu_assert(!e, "received %s", e); + return NULL; +} + + +static char* test_pm_input() +{ + struct pm_result { + double ncp; + const char *scp; + int rv; + const char *err; + }; + + struct pm_result results[] = { + { .ncp = 0, .scp = NULL, .rv = -2, .err = "host specific failure" }, + { .ncp = 1, .scp = NULL, .rv = -1, .err = "failed" }, + { .ncp = 2, .scp = NULL, .rv = 0, .err = "ok" }, + { .ncp = NAN, .scp = "string", .rv = 0, .err = "string" }, + { .ncp = NAN, .scp = NULL, .rv = 0, .err = "no cp" }, + }; + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + for (unsigned i = 0; i < sizeof results / sizeof results[0];++i){ + int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", + results[i].err, err); + mu_assert(results[i].rv == rv, "test: %u expected: %d received: %d", i, + results[i].rv, + rv); + } + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(5 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(1 == stats.pm_failures, "expected %llu", stats.pm_failures); + mu_assert(0 < stats.mem_cur, "expected %llu", stats.mem_cur); + if (clockres <= 100) { + mu_assert(0 < stats.pm_avg, "received %g res %llu", stats.pm_avg, clockres); + mu_assert(0 < stats.pm_sd, "received %g", stats.pm_sd); + } + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + +static char* test_pm_error() +{ + struct pm_result { + double ncp; + const char *scp; + int rv; + const char *err; + }; + + struct pm_result results[] = { + { .ncp = 3, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:39: boom" }, + { .ncp = 4, .scp = NULL, .rv = 1, .err = "process_message() must return a nil or string error message" }, + { .ncp = 5, .scp = NULL, .rv = 1, .err = "process_message() must return a numeric status code" }, + { .ncp = 6, .scp = NULL, .rv = 1, .err = "process_message() lua/input.lua:45: aaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, // >max error message + { .ncp = 7, .scp = NULL, .rv = 1, .err = "process_message() " }, + }; + + lsb_heka_sandbox *hsb; + for (unsigned i = 0; i < sizeof results / sizeof results[0];++i){ + hsb = lsb_heka_create_input(NULL, "lua/input.lua", NULL, NULL, &logger, iim); + mu_assert(hsb, "lsb_heka_create_input failed"); + int rv = lsb_heka_pm_input(hsb, results[i].ncp, results[i].scp, true); + const char *err = lsb_heka_get_error(hsb); + mu_assert(strcmp(results[i].err, err) == 0, "expected: %s received: %s", + results[i].err, err); + mu_assert(results[i].rv == rv, "test: %u expected: %d received: %d", i, + results[i].rv, + rv); + e = lsb_heka_destroy_sandbox(hsb); + } + return NULL; +} + + +static char* test_pm_analysis() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/analysis.lua", NULL, NULL, &logger, aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + mu_assert_rv(0, lsb_heka_pm_analysis(hsb, &m, false)); + const char *err = lsb_heka_get_error(hsb); + const char *eerr = ""; + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(1 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(0 == stats.pm_failures, "expected %llu", stats.pm_failures); + mu_assert(0 == stats.pm_avg, "received %g", stats.pm_avg); + mu_assert(0 == stats.pm_sd, "received %g", stats.pm_sd); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_pm_no_return() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/pm_no_return.lua", NULL, NULL, &logger, aim); + mu_assert_rv(1, lsb_heka_pm_analysis(hsb, &m, false)); + const char *err = lsb_heka_get_error(hsb); + const char *eerr = "process_message() must return a numeric status code"; + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_pm_output() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/output.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n", &logger, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + + mu_assert_rv(0, lsb_heka_pm_output(hsb, &m, NULL, false)); + const char *err = lsb_heka_get_error(hsb); + const char *eerr = ""; + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(0 == stats.pm_failures, "expected %llu", stats.pm_failures); + + mu_assert_rv(-5, lsb_heka_pm_output(hsb, &m, (void *)99, false)); + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + + stats = lsb_heka_get_stats(hsb); + mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); + + mu_assert_rv(-3, lsb_heka_pm_output(hsb, &m, (void *)100, false)); + mu_assert(strcmp(eerr, err) == 0, "expected: %s received: %s", eerr, err); + stats = lsb_heka_get_stats(hsb); + mu_assert(2 == stats.pm_cnt, "expected %llu", stats.pm_cnt); + mu_assert(7 == stats.pm_failures, "expected %llu", stats.pm_failures); + + mu_assert_rv(0, lsb_heka_timer_event(hsb, 0, false)); + + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_im_input() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_input(NULL, "lua/iim.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n" + "Hostname = 'hn'\n" + "Logger = 'iim'\n", + &logger, iim); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(6 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(338 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(hsb, "lsb_heka_create_input failed"); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_im_analysis() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/aim.lua", NULL, + "Hostname = 'foo.com';Logger = 'aim'", + &logger, aim); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(4 == stats.im_cnt, "expected %llu", stats.im_cnt); + mu_assert(290 == stats.im_bytes, "expected %llu", stats.im_bytes); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_im_output() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output_im(NULL, "lua/oim.lua", NULL, + "Hostname = 'test';Logger = 'oim'", + &logger, ucp, oim); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(2 == stats.im_cnt, "received %llu", stats.im_cnt); + mu_assert(70 == stats.im_bytes, "received %llu", stats.im_bytes); + mu_assert(hsb, "lsb_heka_create_input failed"); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_encode_message() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/encode_message.lua", NULL, + "path = [[" TEST_LUA_PATH "]]\n" + "cpath = [[" TEST_LUA_CPATH "]]\n" + "Hostname = 'sh'\n" + "Logger = 'sl'\n" + "Pid = 0\n", + &logger, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + lsb_heka_stats stats = lsb_heka_get_stats(hsb); + mu_assert(164 == stats.out_max, "received %llu", stats.out_max); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_decode_message() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/decode_message.lua", NULL, NULL, &logger, ucp); + mu_assert(hsb, "lsb_heka_create_output failed"); + int rv = lsb_heka_pm_output(hsb, &m, NULL, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_read_message() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + int rv = lsb_heka_pm_analysis(hsb, &m, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static int rm_zc(lua_State *lua) +{ + int n = lua_gettop(lua); + luaL_argcheck(lua, n == 1, n, "incorrect number of arguments"); + luaL_checktype(lua, 1, LUA_TUSERDATA); + lua_CFunction fp = lsb_get_zero_copy_function(lua, 1); + if (!fp) { + return luaL_argerror(lua, 1, "no zero copy support"); + } + int results = fp(lua); + int start = n + 1; + int end = start + results; + int cnt = 0; + for (int i = start; i < end; ++i) { + switch (lua_type(lua, i)) { + case LUA_TSTRING: + lua_pushvalue(lua, i); + ++cnt; + break; + case LUA_TLIGHTUSERDATA: + { + const char *s = lua_touserdata(lua, i++); + size_t len = (size_t)lua_tointeger(lua, i); + if (s && len > 0) { + lua_pushlstring(lua, s, len); + ++cnt; + } + } + break; + default: + return luaL_error(lua, "invalid zero copy return"); + } + } + if (cnt) { + lua_concat(lua, cnt); + } else { + lua_pushnil(lua); + } + return 1; +} + + +static char* test_read_message_zc() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof(pb) - 1, &logger), "failed"); + + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message_zc.lua", NULL, "Hostname = 'ubuntu'", &logger, aim1); + mu_assert(hsb, "lsb_heka_create_analysis failed"); + + lsb_add_function(hsb->lsb, rm_zc, "read_message_zc"); + int rv = lsb_heka_pm_analysis(hsb, &m, false); + mu_assert(0 == rv, "expected: %d received: %d %s", 0, rv, + lsb_heka_get_error(hsb)); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_get_message() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); + mu_assert(lsb_heka_get_message(NULL) == NULL, "non NULL"); + mu_assert(lsb_heka_get_message(hsb) == NULL, "non NULL"); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* test_get_type() +{ + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_analysis(NULL, "lua/read_message.lua", NULL, NULL, &logger, aim); + char t = lsb_heka_get_type(NULL); + mu_assert(t == '\0', "received %c", t); + t = lsb_heka_get_type(hsb); + mu_assert(t == 'a', "received %c", t); + e = lsb_heka_destroy_sandbox(hsb); + return NULL; +} + + +static char* benchmark_decode_message() +{ + int iter = 100000; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed to init message"); + lsb_heka_sandbox *hsb; + hsb = lsb_heka_create_output(NULL, "lua/decode_message_benchmark.lua", NULL, + NULL, &logger, ucp); + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(0 == lsb_heka_pm_output(hsb, &m, NULL, false), "%s", + lsb_heka_get_error(hsb)); + } + t = clock() - t; + printf("benchmark_decode_message() %g seconds\n", ((double)t) + / CLOCKS_PER_SEC / iter); + + mu_assert(hsb, "lsb_heka_create_output failed"); + e = lsb_heka_destroy_sandbox(hsb); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* all_tests() +{ +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; + clock_getres(CLOCK_MONOTONIC, &ts); + clockres = ts.tv_sec * 1000000000ULL + ts.tv_nsec; +#endif + + mu_run_test(test_api_assertion); + mu_run_test(test_create_input_sandbox); + mu_run_test(test_create_analysis_sandbox); + mu_run_test(test_create_output_sandbox); + mu_run_test(test_timer_event); + mu_run_test(test_clean_stop_input); + mu_run_test(test_stop_input); + mu_run_test(test_pm_input); + mu_run_test(test_pm_error); + mu_run_test(test_pm_analysis); + mu_run_test(test_pm_no_return); + mu_run_test(test_pm_output); + mu_run_test(test_im_input); + mu_run_test(test_im_analysis); + mu_run_test(test_im_output); + mu_run_test(test_encode_message); + mu_run_test(test_decode_message); + mu_run_test(test_read_message); + mu_run_test(test_read_message_zc); + mu_run_test(test_get_message); + mu_run_test(test_get_type); + + mu_run_test(benchmark_decode_message); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + free(e); + + return result != 0; +} diff --git a/src/lua/lapi.c b/src/lua/lapi.c new file mode 100644 index 0000000..b8dfde1 --- /dev/null +++ b/src/lua/lapi.c @@ -0,0 +1,1099 @@ +/* +** $Id: lapi.c,v 2.55.1.5 2008/07/04 18:41:18 roberto Exp $ +** Lua API +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include + +#define lapi_c +#define LUA_CORE + +#include "lua.h" + +#include "lapi.h" +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lundump.h" +#include "lvm.h" + + + +const char lua_ident[] = + "$Lua: " LUA_RELEASE " " LUA_COPYRIGHT " $\n" + "$Authors: " LUA_AUTHORS " $\n" + "$URL: www.lua.org $\n"; + + + +#define api_checknelems(L, n) api_check(L, (n) <= (L->top - L->base)) + +#define api_checkvalidindex(L, i) api_check(L, (i) != luaO_nilobject) + +#define api_incr_top(L) {api_check(L, L->top < L->ci->top); L->top++;} + + + +static TValue *index2adr (lua_State *L, int idx) { + if (idx > 0) { + TValue *o = L->base + (idx - 1); + api_check(L, idx <= L->ci->top - L->base); + if (o >= L->top) return cast(TValue *, luaO_nilobject); + else return o; + } + else if (idx > LUA_REGISTRYINDEX) { + api_check(L, idx != 0 && -idx <= L->top - L->base); + return L->top + idx; + } + else switch (idx) { /* pseudo-indices */ + case LUA_REGISTRYINDEX: return registry(L); + case LUA_ENVIRONINDEX: { + Closure *func = curr_func(L); + sethvalue(L, &L->env, func->c.env); + return &L->env; + } + case LUA_GLOBALSINDEX: return gt(L); + default: { + Closure *func = curr_func(L); + idx = LUA_GLOBALSINDEX - idx; + return (idx <= func->c.nupvalues) + ? &func->c.upvalue[idx-1] + : cast(TValue *, luaO_nilobject); + } + } +} + + +static Table *getcurrenv (lua_State *L) { + if (L->ci == L->base_ci) /* no enclosing function? */ + return hvalue(gt(L)); /* use global table as environment */ + else { + Closure *func = curr_func(L); + return func->c.env; + } +} + + +void luaA_pushobject (lua_State *L, const TValue *o) { + setobj2s(L, L->top, o); + api_incr_top(L); +} + + +LUA_API int lua_checkstack (lua_State *L, int size) { + int res = 1; + lua_lock(L); + if (size > LUAI_MAXCSTACK || (L->top - L->base + size) > LUAI_MAXCSTACK) + res = 0; /* stack overflow */ + else if (size > 0) { + luaD_checkstack(L, size); + if (L->ci->top < L->top + size) + L->ci->top = L->top + size; + } + lua_unlock(L); + return res; +} + + +LUA_API void lua_xmove (lua_State *from, lua_State *to, int n) { + int i; + if (from == to) return; + lua_lock(to); + api_checknelems(from, n); + api_check(from, G(from) == G(to)); + api_check(from, to->ci->top - to->top >= n); + from->top -= n; + for (i = 0; i < n; i++) { + setobj2s(to, to->top++, from->top + i); + } + lua_unlock(to); +} + + +LUA_API void lua_setlevel (lua_State *from, lua_State *to) { + to->nCcalls = from->nCcalls; +} + + +LUA_API lua_CFunction lua_atpanic (lua_State *L, lua_CFunction panicf) { + lua_CFunction old; + lua_lock(L); + old = G(L)->panic; + G(L)->panic = panicf; + lua_unlock(L); + return old; +} + + +LUA_API lua_State *lua_newthread (lua_State *L) { + lua_State *L1; + lua_lock(L); + luaC_checkGC(L); + L1 = luaE_newthread(L); + setthvalue(L, L->top, L1); + api_incr_top(L); + lua_unlock(L); + luai_userstatethread(L, L1); + return L1; +} + + + +/* +** basic stack manipulation +*/ + + +LUA_API int lua_gettop (lua_State *L) { + return cast_int(L->top - L->base); +} + + +LUA_API void lua_settop (lua_State *L, int idx) { + lua_lock(L); + if (idx >= 0) { + api_check(L, idx <= L->stack_last - L->base); + while (L->top < L->base + idx) + setnilvalue(L->top++); + L->top = L->base + idx; + } + else { + api_check(L, -(idx+1) <= (L->top - L->base)); + L->top += idx+1; /* `subtract' index (index is negative) */ + } + lua_unlock(L); +} + + +LUA_API void lua_remove (lua_State *L, int idx) { + StkId p; + lua_lock(L); + p = index2adr(L, idx); + api_checkvalidindex(L, p); + while (++p < L->top) setobjs2s(L, p-1, p); + L->top--; + lua_unlock(L); +} + + +LUA_API void lua_insert (lua_State *L, int idx) { + StkId p; + StkId q; + lua_lock(L); + p = index2adr(L, idx); + api_checkvalidindex(L, p); + for (q = L->top; q>p; q--) setobjs2s(L, q, q-1); + setobjs2s(L, p, L->top); + lua_unlock(L); +} + + +LUA_API void lua_replace (lua_State *L, int idx) { + StkId o; + lua_lock(L); + /* explicit test for incompatible code */ + if (idx == LUA_ENVIRONINDEX && L->ci == L->base_ci) + luaG_runerror(L, "no calling environment"); + api_checknelems(L, 1); + o = index2adr(L, idx); + api_checkvalidindex(L, o); + if (idx == LUA_ENVIRONINDEX) { + Closure *func = curr_func(L); + api_check(L, ttistable(L->top - 1)); + func->c.env = hvalue(L->top - 1); + luaC_barrier(L, func, L->top - 1); + } + else { + setobj(L, o, L->top - 1); + if (idx < LUA_GLOBALSINDEX) /* function upvalue? */ + luaC_barrier(L, curr_func(L), L->top - 1); + } + L->top--; + lua_unlock(L); +} + + +LUA_API void lua_pushvalue (lua_State *L, int idx) { + lua_lock(L); + setobj2s(L, L->top, index2adr(L, idx)); + api_incr_top(L); + lua_unlock(L); +} + + + +/* +** access functions (stack -> C) +*/ + + +LUA_API int lua_type (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return (o == luaO_nilobject) ? LUA_TNONE : ttype(o); +} + + +LUA_API const char *lua_typename (lua_State *L, int t) { + UNUSED(L); + return (t == LUA_TNONE) ? "no value" : luaT_typenames[t]; +} + + +LUA_API int lua_iscfunction (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return iscfunction(o); +} + + +LUA_API int lua_tabletype (lua_State *L, int idx) { + StkId t; + int tt; + lua_lock(L); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + tt = luaH_type(hvalue(t)); + lua_unlock(L); + return tt; +} + + +LUA_API int lua_isnumber (lua_State *L, int idx) { + TValue n; + const TValue *o = index2adr(L, idx); + return tonumber(o, &n); +} + + +LUA_API int lua_isstring (lua_State *L, int idx) { + int t = lua_type(L, idx); + return (t == LUA_TSTRING || t == LUA_TNUMBER); +} + + +LUA_API int lua_isuserdata (lua_State *L, int idx) { + const TValue *o = index2adr(L, idx); + return (ttisuserdata(o) || ttislightuserdata(o)); +} + + +LUA_API int lua_rawequal (lua_State *L, int index1, int index2) { + StkId o1 = index2adr(L, index1); + StkId o2 = index2adr(L, index2); + return (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 + : luaO_rawequalObj(o1, o2); +} + + +LUA_API int lua_equal (lua_State *L, int index1, int index2) { + StkId o1, o2; + int i; + lua_lock(L); /* may call tag method */ + o1 = index2adr(L, index1); + o2 = index2adr(L, index2); + i = (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 : equalobj(L, o1, o2); + lua_unlock(L); + return i; +} + + +LUA_API int lua_lessthan (lua_State *L, int index1, int index2) { + StkId o1, o2; + int i; + lua_lock(L); /* may call tag method */ + o1 = index2adr(L, index1); + o2 = index2adr(L, index2); + i = (o1 == luaO_nilobject || o2 == luaO_nilobject) ? 0 + : luaV_lessthan(L, o1, o2); + lua_unlock(L); + return i; +} + + + +LUA_API lua_Number lua_tonumber (lua_State *L, int idx) { + TValue n; + const TValue *o = index2adr(L, idx); + if (tonumber(o, &n)) + return nvalue(o); + else + return 0; +} + + +LUA_API lua_Integer lua_tointeger (lua_State *L, int idx) { + TValue n; + const TValue *o = index2adr(L, idx); + if (tonumber(o, &n)) { + lua_Integer res; + lua_Number num = nvalue(o); + lua_number2integer(res, num); + return res; + } + else + return 0; +} + + +LUA_API int lua_toboolean (lua_State *L, int idx) { + const TValue *o = index2adr(L, idx); + return !l_isfalse(o); +} + + +LUA_API const char *lua_tolstring (lua_State *L, int idx, size_t *len) { + StkId o = index2adr(L, idx); + if (!ttisstring(o)) { + lua_lock(L); /* `luaV_tostring' may create a new string */ + if (!luaV_tostring(L, o)) { /* conversion failed? */ + if (len != NULL) *len = 0; + lua_unlock(L); + return NULL; + } + luaC_checkGC(L); + o = index2adr(L, idx); /* previous call may reallocate the stack */ + lua_unlock(L); + } + if (len != NULL) *len = tsvalue(o)->len; + return svalue(o); +} + + +LUA_API size_t lua_objlen (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + switch (ttype(o)) { + case LUA_TSTRING: return tsvalue(o)->len; + case LUA_TUSERDATA: return uvalue(o)->len; + case LUA_TTABLE: return luaH_getn(hvalue(o)); + case LUA_TNUMBER: { + size_t l; + lua_lock(L); /* `luaV_tostring' may create a new string */ + l = (luaV_tostring(L, o) ? tsvalue(o)->len : 0); + lua_unlock(L); + return l; + } + default: return 0; + } +} + + +LUA_API lua_CFunction lua_tocfunction (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return (!iscfunction(o)) ? NULL : clvalue(o)->c.f; +} + + +LUA_API void *lua_touserdata (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + switch (ttype(o)) { + case LUA_TUSERDATA: return (rawuvalue(o) + 1); + case LUA_TLIGHTUSERDATA: return pvalue(o); + default: return NULL; + } +} + + +LUA_API lua_State *lua_tothread (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + return (!ttisthread(o)) ? NULL : thvalue(o); +} + + +LUA_API const void *lua_topointer (lua_State *L, int idx) { + StkId o = index2adr(L, idx); + switch (ttype(o)) { + case LUA_TTABLE: return hvalue(o); + case LUA_TFUNCTION: return clvalue(o); + case LUA_TTHREAD: return thvalue(o); + case LUA_TUSERDATA: + case LUA_TLIGHTUSERDATA: + return lua_touserdata(L, idx); + default: return NULL; + } +} + + + +/* +** push functions (C -> stack) +*/ + + +LUA_API void lua_pushnil (lua_State *L) { + lua_lock(L); + setnilvalue(L->top); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushnumber (lua_State *L, lua_Number n) { + lua_lock(L); + setnvalue(L->top, n); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushinteger (lua_State *L, lua_Integer n) { + lua_lock(L); + setnvalue(L->top, cast_num(n)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushlstring (lua_State *L, const char *s, size_t len) { + lua_lock(L); + luaC_checkGC(L); + setsvalue2s(L, L->top, luaS_newlstr(L, s, len)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushstring (lua_State *L, const char *s) { + if (s == NULL) + lua_pushnil(L); + else + lua_pushlstring(L, s, strlen(s)); +} + + +LUA_API const char *lua_pushvfstring (lua_State *L, const char *fmt, + va_list argp) { + const char *ret; + lua_lock(L); + luaC_checkGC(L); + ret = luaO_pushvfstring(L, fmt, argp); + lua_unlock(L); + return ret; +} + + +LUA_API const char *lua_pushfstring (lua_State *L, const char *fmt, ...) { + const char *ret; + va_list argp; + lua_lock(L); + luaC_checkGC(L); + va_start(argp, fmt); + ret = luaO_pushvfstring(L, fmt, argp); + va_end(argp); + lua_unlock(L); + return ret; +} + + +LUA_API void lua_pushcclosure (lua_State *L, lua_CFunction fn, int n) { + Closure *cl; + lua_lock(L); + luaC_checkGC(L); + api_checknelems(L, n); + cl = luaF_newCclosure(L, n, getcurrenv(L)); + cl->c.f = fn; + L->top -= n; + while (n--) + setobj2n(L, &cl->c.upvalue[n], L->top+n); + setclvalue(L, L->top, cl); + lua_assert(iswhite(obj2gco(cl))); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushboolean (lua_State *L, int b) { + lua_lock(L); + setbvalue(L->top, (b != 0)); /* ensure that true is 1 */ + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_pushlightuserdata (lua_State *L, void *p) { + lua_lock(L); + setpvalue(L->top, p); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API int lua_pushthread (lua_State *L) { + lua_lock(L); + setthvalue(L, L->top, L); + api_incr_top(L); + lua_unlock(L); + return (G(L)->mainthread == L); +} + + + +/* +** get functions (Lua -> stack) +*/ + + +LUA_API void lua_gettable (lua_State *L, int idx) { + StkId t; + lua_lock(L); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + luaV_gettable(L, t, L->top - 1, L->top - 1); + lua_unlock(L); +} + + +LUA_API void lua_getfield (lua_State *L, int idx, const char *k) { + StkId t; + TValue key; + lua_lock(L); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + setsvalue(L, &key, luaS_new(L, k)); + luaV_gettable(L, t, &key, L->top); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_rawget (lua_State *L, int idx) { + StkId t; + lua_lock(L); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + setobj2s(L, L->top - 1, luaH_get(hvalue(t), L->top - 1)); + lua_unlock(L); +} + + +LUA_API void lua_rawgeti (lua_State *L, int idx, int n) { + StkId o; + lua_lock(L); + o = index2adr(L, idx); + api_check(L, ttistable(o)); + setobj2s(L, L->top, luaH_getnum(hvalue(o), n)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API void lua_createtable (lua_State *L, int narray, int nrec) { + lua_lock(L); + luaC_checkGC(L); + sethvalue(L, L->top, luaH_new(L, narray, nrec)); + api_incr_top(L); + lua_unlock(L); +} + + +LUA_API int lua_getmetatable (lua_State *L, int objindex) { + const TValue *obj; + Table *mt = NULL; + int res; + lua_lock(L); + obj = index2adr(L, objindex); + switch (ttype(obj)) { + case LUA_TTABLE: + mt = hvalue(obj)->metatable; + break; + case LUA_TUSERDATA: + mt = uvalue(obj)->metatable; + break; + default: + mt = G(L)->mt[ttype(obj)]; + break; + } + if (mt == NULL) + res = 0; + else { + sethvalue(L, L->top, mt); + api_incr_top(L); + res = 1; + } + lua_unlock(L); + return res; +} + + +LUA_API void lua_getfenv (lua_State *L, int idx) { + StkId o; + lua_lock(L); + o = index2adr(L, idx); + api_checkvalidindex(L, o); + switch (ttype(o)) { + case LUA_TFUNCTION: + sethvalue(L, L->top, clvalue(o)->c.env); + break; + case LUA_TUSERDATA: + sethvalue(L, L->top, uvalue(o)->env); + break; + case LUA_TTHREAD: + setobj2s(L, L->top, gt(thvalue(o))); + break; + default: + setnilvalue(L->top); + break; + } + api_incr_top(L); + lua_unlock(L); +} + + +/* +** set functions (stack -> Lua) +*/ + + +LUA_API void lua_settable (lua_State *L, int idx) { + StkId t; + lua_lock(L); + api_checknelems(L, 2); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + luaV_settable(L, t, L->top - 2, L->top - 1); + L->top -= 2; /* pop index and value */ + lua_unlock(L); +} + + +LUA_API void lua_setfield (lua_State *L, int idx, const char *k) { + StkId t; + TValue key; + lua_lock(L); + api_checknelems(L, 1); + t = index2adr(L, idx); + api_checkvalidindex(L, t); + setsvalue(L, &key, luaS_new(L, k)); + luaV_settable(L, t, &key, L->top - 1); + L->top--; /* pop value */ + lua_unlock(L); +} + + +LUA_API void lua_rawset (lua_State *L, int idx) { + StkId t; + lua_lock(L); + api_checknelems(L, 2); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + setobj2t(L, luaH_set(L, hvalue(t), L->top-2), L->top-1); + luaC_barriert(L, hvalue(t), L->top-1); + L->top -= 2; + lua_unlock(L); +} + + +LUA_API void lua_rawseti (lua_State *L, int idx, int n) { + StkId o; + lua_lock(L); + api_checknelems(L, 1); + o = index2adr(L, idx); + api_check(L, ttistable(o)); + setobj2t(L, luaH_setnum(L, hvalue(o), n), L->top-1); + luaC_barriert(L, hvalue(o), L->top-1); + L->top--; + lua_unlock(L); +} + + +LUA_API int lua_setmetatable (lua_State *L, int objindex) { + TValue *obj; + Table *mt; + lua_lock(L); + api_checknelems(L, 1); + obj = index2adr(L, objindex); + api_checkvalidindex(L, obj); + if (ttisnil(L->top - 1)) + mt = NULL; + else { + api_check(L, ttistable(L->top - 1)); + mt = hvalue(L->top - 1); + } + switch (ttype(obj)) { + case LUA_TTABLE: { + hvalue(obj)->metatable = mt; + if (mt) + luaC_objbarriert(L, hvalue(obj), mt); + break; + } + case LUA_TUSERDATA: { + uvalue(obj)->metatable = mt; + if (mt) + luaC_objbarrier(L, rawuvalue(obj), mt); + break; + } + default: { + G(L)->mt[ttype(obj)] = mt; + break; + } + } + L->top--; + lua_unlock(L); + return 1; +} + + +LUA_API int lua_setfenv (lua_State *L, int idx) { + StkId o; + int res = 1; + lua_lock(L); + api_checknelems(L, 1); + o = index2adr(L, idx); + api_checkvalidindex(L, o); + api_check(L, ttistable(L->top - 1)); + switch (ttype(o)) { + case LUA_TFUNCTION: + clvalue(o)->c.env = hvalue(L->top - 1); + break; + case LUA_TUSERDATA: + uvalue(o)->env = hvalue(L->top - 1); + break; + case LUA_TTHREAD: + sethvalue(L, gt(thvalue(o)), hvalue(L->top - 1)); + break; + default: + res = 0; + break; + } + if (res) luaC_objbarrier(L, gcvalue(o), hvalue(L->top - 1)); + L->top--; + lua_unlock(L); + return res; +} + + +/* +** `load' and `call' functions (run Lua code) +*/ + + +#define adjustresults(L,nres) \ + { if (nres == LUA_MULTRET && L->top >= L->ci->top) L->ci->top = L->top; } + + +#define checkresults(L,na,nr) \ + api_check(L, (nr) == LUA_MULTRET || (L->ci->top - L->top >= (nr) - (na))) + + +LUA_API void lua_call (lua_State *L, int nargs, int nresults) { + StkId func; + lua_lock(L); + api_checknelems(L, nargs+1); + checkresults(L, nargs, nresults); + func = L->top - (nargs+1); + luaD_call(L, func, nresults); + adjustresults(L, nresults); + lua_unlock(L); +} + + + +/* +** Execute a protected call. +*/ +struct CallS { /* data to `f_call' */ + StkId func; + int nresults; +}; + + +static void f_call (lua_State *L, void *ud) { + struct CallS *c = cast(struct CallS *, ud); + luaD_call(L, c->func, c->nresults); +} + + + +LUA_API int lua_pcall (lua_State *L, int nargs, int nresults, int errfunc) { + struct CallS c; + int status; + ptrdiff_t func; + lua_lock(L); + api_checknelems(L, nargs+1); + checkresults(L, nargs, nresults); + if (errfunc == 0) + func = 0; + else { + StkId o = index2adr(L, errfunc); + api_checkvalidindex(L, o); + func = savestack(L, o); + } + c.func = L->top - (nargs+1); /* function to be called */ + c.nresults = nresults; + status = luaD_pcall(L, f_call, &c, savestack(L, c.func), func); + adjustresults(L, nresults); + lua_unlock(L); + return status; +} + + +/* +** Execute a protected C call. +*/ +struct CCallS { /* data to `f_Ccall' */ + lua_CFunction func; + void *ud; +}; + + +static void f_Ccall (lua_State *L, void *ud) { + struct CCallS *c = cast(struct CCallS *, ud); + Closure *cl; + cl = luaF_newCclosure(L, 0, getcurrenv(L)); + cl->c.f = c->func; + setclvalue(L, L->top, cl); /* push function */ + api_incr_top(L); + setpvalue(L->top, c->ud); /* push only argument */ + api_incr_top(L); + luaD_call(L, L->top - 2, 0); +} + + +LUA_API int lua_cpcall (lua_State *L, lua_CFunction func, void *ud) { + struct CCallS c; + int status; + lua_lock(L); + c.func = func; + c.ud = ud; + status = luaD_pcall(L, f_Ccall, &c, savestack(L, L->top), 0); + lua_unlock(L); + return status; +} + + +LUA_API int lua_load (lua_State *L, lua_Reader reader, void *data, + const char *chunkname) { + ZIO z; + int status; + lua_lock(L); + if (!chunkname) chunkname = "?"; + luaZ_init(L, &z, reader, data); + status = luaD_protectedparser(L, &z, chunkname); + lua_unlock(L); + return status; +} + + +LUA_API int lua_dump (lua_State *L, lua_Writer writer, void *data) { + int status; + TValue *o; + lua_lock(L); + api_checknelems(L, 1); + o = L->top - 1; + if (isLfunction(o)) + status = luaU_dump(L, clvalue(o)->l.p, writer, data, 0); + else + status = 1; + lua_unlock(L); + return status; +} + + +LUA_API int lua_status (lua_State *L) { + return L->status; +} + + +/* +** Garbage-collection function +*/ + +LUA_API int lua_gc (lua_State *L, int what, int data) { + int res = 0; + global_State *g; + lua_lock(L); + g = G(L); + switch (what) { + case LUA_GCSTOP: { + g->GCthreshold = MAX_LUMEM; + break; + } + case LUA_GCRESTART: { + g->GCthreshold = g->totalbytes; + break; + } + case LUA_GCCOLLECT: { + luaC_fullgc(L); + break; + } + case LUA_GCCOUNT: { + /* GC values are expressed in Kbytes: #bytes/2^10 */ + res = cast_int(g->totalbytes >> 10); + break; + } + case LUA_GCCOUNTB: { + res = cast_int(g->totalbytes & 0x3ff); + break; + } + case LUA_GCSTEP: { + lu_mem a = (cast(lu_mem, data) << 10); + if (a <= g->totalbytes) + g->GCthreshold = g->totalbytes - a; + else + g->GCthreshold = 0; + while (g->GCthreshold <= g->totalbytes) { + luaC_step(L); + if (g->gcstate == GCSpause) { /* end of cycle? */ + res = 1; /* signal it */ + break; + } + } + break; + } + case LUA_GCSETPAUSE: { + res = g->gcpause; + g->gcpause = data; + break; + } + case LUA_GCSETSTEPMUL: { + res = g->gcstepmul; + g->gcstepmul = data; + break; + } + default: res = -1; /* invalid option */ + } + lua_unlock(L); + return res; +} + + + +/* +** miscellaneous functions +*/ + + +LUA_API int lua_error (lua_State *L) { + lua_lock(L); + api_checknelems(L, 1); + luaG_errormsg(L); + lua_unlock(L); + return 0; /* to avoid warnings */ +} + + +LUA_API int lua_next (lua_State *L, int idx) { + StkId t; + int more; + lua_lock(L); + t = index2adr(L, idx); + api_check(L, ttistable(t)); + more = luaH_next(L, hvalue(t), L->top - 1); + if (more) { + api_incr_top(L); + } + else /* no more elements */ + L->top -= 1; /* remove key */ + lua_unlock(L); + return more; +} + + +LUA_API void lua_concat (lua_State *L, int n) { + lua_lock(L); + api_checknelems(L, n); + if (n >= 2) { + luaC_checkGC(L); + luaV_concat(L, n, cast_int(L->top - L->base) - 1); + L->top -= (n-1); + } + else if (n == 0) { /* push empty string */ + setsvalue2s(L, L->top, luaS_newlstr(L, "", 0)); + api_incr_top(L); + } + /* else n == 1; nothing to do */ + lua_unlock(L); +} + + +LUA_API lua_Alloc lua_getallocf (lua_State *L, void **ud) { + lua_Alloc f; + lua_lock(L); + if (ud) *ud = G(L)->ud; + f = G(L)->frealloc; + lua_unlock(L); + return f; +} + + +LUA_API void lua_setallocf (lua_State *L, lua_Alloc f, void *ud) { + lua_lock(L); + G(L)->ud = ud; + G(L)->frealloc = f; + lua_unlock(L); +} + + +LUA_API void *lua_newuserdata (lua_State *L, size_t size) { + Udata *u; + lua_lock(L); + luaC_checkGC(L); + u = luaS_newudata(L, size, getcurrenv(L)); + setuvalue(L, L->top, u); + api_incr_top(L); + lua_unlock(L); + return u + 1; +} + + + + +static const char *aux_upvalue (StkId fi, int n, TValue **val) { + Closure *f; + if (!ttisfunction(fi)) return NULL; + f = clvalue(fi); + if (f->c.isC) { + if (!(1 <= n && n <= f->c.nupvalues)) return NULL; + *val = &f->c.upvalue[n-1]; + return ""; + } + else { + Proto *p = f->l.p; + if (!(1 <= n && n <= p->sizeupvalues)) return NULL; + *val = f->l.upvals[n-1]->v; + return getstr(p->upvalues[n-1]); + } +} + + +LUA_API const char *lua_getupvalue (lua_State *L, int funcindex, int n) { + const char *name; + TValue *val; + lua_lock(L); + name = aux_upvalue(index2adr(L, funcindex), n, &val); + if (name) { + setobj2s(L, L->top, val); + api_incr_top(L); + } + lua_unlock(L); + return name; +} + + +LUA_API const char *lua_setupvalue (lua_State *L, int funcindex, int n) { + const char *name; + TValue *val; + StkId fi; + lua_lock(L); + fi = index2adr(L, funcindex); + api_checknelems(L, 1); + name = aux_upvalue(fi, n, &val); + if (name) { + L->top--; + setobj(L, val, L->top); + luaC_barrier(L, clvalue(fi), L->top); + } + lua_unlock(L); + return name; +} + diff --git a/src/lua/lapi.h b/src/lua/lapi.h new file mode 100644 index 0000000..2c3fab2 --- /dev/null +++ b/src/lua/lapi.h @@ -0,0 +1,16 @@ +/* +** $Id: lapi.h,v 2.2.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions from Lua API +** See Copyright Notice in lua.h +*/ + +#ifndef lapi_h +#define lapi_h + + +#include "lobject.h" + + +LUAI_FUNC void luaA_pushobject (lua_State *L, const TValue *o); + +#endif diff --git a/src/lua/lauxlib.c b/src/lua/lauxlib.c new file mode 100644 index 0000000..a1ca299 --- /dev/null +++ b/src/lua/lauxlib.c @@ -0,0 +1,653 @@ +/* +** $Id: lauxlib.c,v 1.159.1.3 2008/01/21 13:20:51 roberto Exp $ +** Auxiliary functions for building Lua libraries +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include +#include +#include + + +/* This file uses only the official API of Lua. +** Any function declared here could be written as an application function. +*/ + +#define lauxlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" + + +#define FREELIST_REF 0 /* free list of references */ + + +/* convert a stack index to positive */ +#define abs_index(L, i) ((i) > 0 || (i) <= LUA_REGISTRYINDEX ? (i) : \ + lua_gettop(L) + (i) + 1) + + +/* +** {====================================================== +** Error-report functions +** ======================================================= +*/ + + +LUALIB_API int luaL_argerror (lua_State *L, int narg, const char *extramsg) { + lua_Debug ar; + if (!lua_getstack(L, 0, &ar)) /* no stack frame? */ + return luaL_error(L, "bad argument #%d (%s)", narg, extramsg); + lua_getinfo(L, "n", &ar); + if (strcmp(ar.namewhat, "method") == 0) { + narg--; /* do not count `self' */ + if (narg == 0) /* error is in the self argument itself? */ + return luaL_error(L, "calling " LUA_QS " on bad self (%s)", + ar.name, extramsg); + } + if (ar.name == NULL) + ar.name = "?"; + return luaL_error(L, "bad argument #%d to " LUA_QS " (%s)", + narg, ar.name, extramsg); +} + + +LUALIB_API int luaL_typerror (lua_State *L, int narg, const char *tname) { + const char *msg = lua_pushfstring(L, "%s expected, got %s", + tname, luaL_typename(L, narg)); + return luaL_argerror(L, narg, msg); +} + + +static void tag_error (lua_State *L, int narg, int tag) { + luaL_typerror(L, narg, lua_typename(L, tag)); +} + + +LUALIB_API void luaL_where (lua_State *L, int level) { + lua_Debug ar; + if (lua_getstack(L, level, &ar)) { /* check function at level */ + lua_getinfo(L, "Sl", &ar); /* get info about it */ + if (ar.currentline > 0) { /* is there info? */ + lua_pushfstring(L, "%s:%d: ", ar.short_src, ar.currentline); + return; + } + } + lua_pushliteral(L, ""); /* else, no information available... */ +} + + +LUALIB_API int luaL_error (lua_State *L, const char *fmt, ...) { + va_list argp; + va_start(argp, fmt); + luaL_where(L, 1); + lua_pushvfstring(L, fmt, argp); + va_end(argp); + lua_concat(L, 2); + return lua_error(L); +} + +/* }====================================================== */ + + +LUALIB_API int luaL_checkoption (lua_State *L, int narg, const char *def, + const char *const lst[]) { + const char *name = (def) ? luaL_optstring(L, narg, def) : + luaL_checkstring(L, narg); + int i; + for (i=0; lst[i]; i++) + if (strcmp(lst[i], name) == 0) + return i; + return luaL_argerror(L, narg, + lua_pushfstring(L, "invalid option " LUA_QS, name)); +} + + +LUALIB_API int luaL_newmetatable (lua_State *L, const char *tname) { + lua_getfield(L, LUA_REGISTRYINDEX, tname); /* get registry.name */ + if (!lua_isnil(L, -1)) /* name already in use? */ + return 0; /* leave previous value on top, but return 0 */ + lua_pop(L, 1); + lua_newtable(L); /* create metatable */ + lua_pushvalue(L, -1); + lua_setfield(L, LUA_REGISTRYINDEX, tname); /* registry.name = metatable */ + return 1; +} + + +LUALIB_API void *luaL_checkudata (lua_State *L, int ud, const char *tname) { + void *p = lua_touserdata(L, ud); + if (p != NULL) { /* value is a userdata? */ + if (lua_getmetatable(L, ud)) { /* does it have a metatable? */ + lua_getfield(L, LUA_REGISTRYINDEX, tname); /* get correct metatable */ + if (lua_rawequal(L, -1, -2)) { /* does it have the correct mt? */ + lua_pop(L, 2); /* remove both metatables */ + return p; + } + } + } + luaL_typerror(L, ud, tname); /* else error */ + return NULL; /* to avoid warnings */ +} + + +LUALIB_API void luaL_checkstack (lua_State *L, int space, const char *mes) { + if (!lua_checkstack(L, space)) + luaL_error(L, "stack overflow (%s)", mes); +} + + +LUALIB_API void luaL_checktype (lua_State *L, int narg, int t) { + if (lua_type(L, narg) != t) + tag_error(L, narg, t); +} + + +LUALIB_API void luaL_checkany (lua_State *L, int narg) { + if (lua_type(L, narg) == LUA_TNONE) + luaL_argerror(L, narg, "value expected"); +} + + +LUALIB_API const char *luaL_checklstring (lua_State *L, int narg, size_t *len) { + const char *s = lua_tolstring(L, narg, len); + if (!s) tag_error(L, narg, LUA_TSTRING); + return s; +} + + +LUALIB_API const char *luaL_optlstring (lua_State *L, int narg, + const char *def, size_t *len) { + if (lua_isnoneornil(L, narg)) { + if (len) + *len = (def ? strlen(def) : 0); + return def; + } + else return luaL_checklstring(L, narg, len); +} + + +LUALIB_API lua_Number luaL_checknumber (lua_State *L, int narg) { + lua_Number d = lua_tonumber(L, narg); + if (d == 0 && !lua_isnumber(L, narg)) /* avoid extra test when d is not 0 */ + tag_error(L, narg, LUA_TNUMBER); + return d; +} + + +LUALIB_API lua_Number luaL_optnumber (lua_State *L, int narg, lua_Number def) { + return luaL_opt(L, luaL_checknumber, narg, def); +} + + +LUALIB_API lua_Integer luaL_checkinteger (lua_State *L, int narg) { + lua_Integer d = lua_tointeger(L, narg); + if (d == 0 && !lua_isnumber(L, narg)) /* avoid extra test when d is not 0 */ + tag_error(L, narg, LUA_TNUMBER); + return d; +} + + +LUALIB_API lua_Integer luaL_optinteger (lua_State *L, int narg, + lua_Integer def) { + return luaL_opt(L, luaL_checkinteger, narg, def); +} + + +LUALIB_API int luaL_getmetafield (lua_State *L, int obj, const char *event) { + if (!lua_getmetatable(L, obj)) /* no metatable? */ + return 0; + lua_pushstring(L, event); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { + lua_pop(L, 2); /* remove metatable and metafield */ + return 0; + } + else { + lua_remove(L, -2); /* remove only metatable */ + return 1; + } +} + + +LUALIB_API int luaL_callmeta (lua_State *L, int obj, const char *event) { + obj = abs_index(L, obj); + if (!luaL_getmetafield(L, obj, event)) /* no metafield? */ + return 0; + lua_pushvalue(L, obj); + lua_call(L, 1, 1); + return 1; +} + + +LUALIB_API void (luaL_register) (lua_State *L, const char *libname, + const luaL_Reg *l) { + luaI_openlib(L, libname, l, 0); +} + + +static int libsize (const luaL_Reg *l) { + int size = 0; + for (; l->name; l++) size++; + return size; +} + + +LUALIB_API void luaI_openlib (lua_State *L, const char *libname, + const luaL_Reg *l, int nup) { + if (libname) { + int size = libsize(l); + /* check whether lib already exists */ + luaL_findtable(L, LUA_REGISTRYINDEX, "_LOADED", 1); + lua_getfield(L, -1, libname); /* get _LOADED[libname] */ + if (!lua_istable(L, -1)) { /* not found? */ + lua_pop(L, 1); /* remove previous result */ + /* try global variable (and create one if it does not exist) */ + if (luaL_findtable(L, LUA_GLOBALSINDEX, libname, size) != NULL) + luaL_error(L, "name conflict for module " LUA_QS, libname); + lua_pushvalue(L, -1); + lua_setfield(L, -3, libname); /* _LOADED[libname] = new table */ + } + lua_remove(L, -2); /* remove _LOADED table */ + lua_insert(L, -(nup+1)); /* move library table to below upvalues */ + } + for (; l->name; l++) { + int i; + for (i=0; ifunc, nup); + lua_setfield(L, -(nup+2), l->name); + } + lua_pop(L, nup); /* remove upvalues */ +} + + + +/* +** {====================================================== +** getn-setn: size for arrays +** ======================================================= +*/ + +#if defined(LUA_COMPAT_GETN) + +static int checkint (lua_State *L, int topop) { + int n = (lua_type(L, -1) == LUA_TNUMBER) ? lua_tointeger(L, -1) : -1; + lua_pop(L, topop); + return n; +} + + +static void getsizes (lua_State *L) { + lua_getfield(L, LUA_REGISTRYINDEX, "LUA_SIZES"); + if (lua_isnil(L, -1)) { /* no `size' table? */ + lua_pop(L, 1); /* remove nil */ + lua_newtable(L); /* create it */ + lua_pushvalue(L, -1); /* `size' will be its own metatable */ + lua_setmetatable(L, -2); + lua_pushliteral(L, "kv"); + lua_setfield(L, -2, "__mode"); /* metatable(N).__mode = "kv" */ + lua_pushvalue(L, -1); + lua_setfield(L, LUA_REGISTRYINDEX, "LUA_SIZES"); /* store in register */ + } +} + + +LUALIB_API void luaL_setn (lua_State *L, int t, int n) { + t = abs_index(L, t); + lua_pushliteral(L, "n"); + lua_rawget(L, t); + if (checkint(L, 1) >= 0) { /* is there a numeric field `n'? */ + lua_pushliteral(L, "n"); /* use it */ + lua_pushinteger(L, n); + lua_rawset(L, t); + } + else { /* use `sizes' */ + getsizes(L); + lua_pushvalue(L, t); + lua_pushinteger(L, n); + lua_rawset(L, -3); /* sizes[t] = n */ + lua_pop(L, 1); /* remove `sizes' */ + } +} + + +LUALIB_API int luaL_getn (lua_State *L, int t) { + int n; + t = abs_index(L, t); + lua_pushliteral(L, "n"); /* try t.n */ + lua_rawget(L, t); + if ((n = checkint(L, 1)) >= 0) return n; + getsizes(L); /* else try sizes[t] */ + lua_pushvalue(L, t); + lua_rawget(L, -2); + if ((n = checkint(L, 2)) >= 0) return n; + return (int)lua_objlen(L, t); +} + +#endif + +/* }====================================================== */ + + + +LUALIB_API const char *luaL_gsub (lua_State *L, const char *s, const char *p, + const char *r) { + const char *wild; + size_t l = strlen(p); + luaL_Buffer b; + luaL_buffinit(L, &b); + while ((wild = strstr(s, p)) != NULL) { + luaL_addlstring(&b, s, wild - s); /* push prefix */ + luaL_addstring(&b, r); /* push replacement in place of pattern */ + s = wild + l; /* continue after `p' */ + } + luaL_addstring(&b, s); /* push last suffix */ + luaL_pushresult(&b); + return lua_tostring(L, -1); +} + + +LUALIB_API const char *luaL_findtable (lua_State *L, int idx, + const char *fname, int szhint) { + const char *e; + lua_pushvalue(L, idx); + do { + e = strchr(fname, '.'); + if (e == NULL) e = fname + strlen(fname); + lua_pushlstring(L, fname, e - fname); + lua_rawget(L, -2); + if (lua_isnil(L, -1)) { /* no such field? */ + lua_pop(L, 1); /* remove this nil */ + lua_createtable(L, 0, (*e == '.' ? 1 : szhint)); /* new table for field */ + lua_pushlstring(L, fname, e - fname); + lua_pushvalue(L, -2); + lua_settable(L, -4); /* set new table into field */ + } + else if (!lua_istable(L, -1)) { /* field has a non-table value? */ + lua_pop(L, 2); /* remove table and value */ + return fname; /* return problematic part of the name */ + } + lua_remove(L, -2); /* remove previous table */ + fname = e + 1; + } while (*e == '.'); + return NULL; +} + + + +/* +** {====================================================== +** Generic Buffer manipulation +** ======================================================= +*/ + + +#define bufflen(B) ((B)->p - (B)->buffer) +#define bufffree(B) ((size_t)(LUAL_BUFFERSIZE - bufflen(B))) + +#define LIMIT (LUA_MINSTACK/2) + + +static int emptybuffer (luaL_Buffer *B) { + size_t l = bufflen(B); + if (l == 0) return 0; /* put nothing on stack */ + else { + lua_pushlstring(B->L, B->buffer, l); + B->p = B->buffer; + B->lvl++; + return 1; + } +} + + +static void adjuststack (luaL_Buffer *B) { + if (B->lvl > 1) { + lua_State *L = B->L; + int toget = 1; /* number of levels to concat */ + size_t toplen = lua_strlen(L, -1); + do { + size_t l = lua_strlen(L, -(toget+1)); + if (B->lvl - toget + 1 >= LIMIT || toplen > l) { + toplen += l; + toget++; + } + else break; + } while (toget < B->lvl); + lua_concat(L, toget); + B->lvl = B->lvl - toget + 1; + } +} + + +LUALIB_API char *luaL_prepbuffer (luaL_Buffer *B) { + if (emptybuffer(B)) + adjuststack(B); + return B->buffer; +} + + +LUALIB_API void luaL_addlstring (luaL_Buffer *B, const char *s, size_t l) { + while (l--) + luaL_addchar(B, *s++); +} + + +LUALIB_API void luaL_addstring (luaL_Buffer *B, const char *s) { + luaL_addlstring(B, s, strlen(s)); +} + + +LUALIB_API void luaL_pushresult (luaL_Buffer *B) { + emptybuffer(B); + lua_concat(B->L, B->lvl); + B->lvl = 1; +} + + +LUALIB_API void luaL_addvalue (luaL_Buffer *B) { + lua_State *L = B->L; + size_t vl; + const char *s = lua_tolstring(L, -1, &vl); + if (vl <= bufffree(B)) { /* fit into buffer? */ + memcpy(B->p, s, vl); /* put it there */ + B->p += vl; + lua_pop(L, 1); /* remove from stack */ + } + else { + if (emptybuffer(B)) + lua_insert(L, -2); /* put buffer before new value */ + B->lvl++; /* add new value into B stack */ + adjuststack(B); + } +} + + +LUALIB_API void luaL_buffinit (lua_State *L, luaL_Buffer *B) { + B->L = L; + B->p = B->buffer; + B->lvl = 0; +} + +/* }====================================================== */ + + +LUALIB_API int luaL_ref (lua_State *L, int t) { + int ref; + t = abs_index(L, t); + if (lua_isnil(L, -1)) { + lua_pop(L, 1); /* remove from stack */ + return LUA_REFNIL; /* `nil' has a unique fixed reference */ + } + lua_rawgeti(L, t, FREELIST_REF); /* get first free element */ + ref = (int)lua_tointeger(L, -1); /* ref = t[FREELIST_REF] */ + lua_pop(L, 1); /* remove it from stack */ + if (ref != 0) { /* any free element? */ + lua_rawgeti(L, t, ref); /* remove it from list */ + lua_rawseti(L, t, FREELIST_REF); /* (t[FREELIST_REF] = t[ref]) */ + } + else { /* no free elements */ + ref = (int)lua_objlen(L, t); + ref++; /* create new reference */ + } + lua_rawseti(L, t, ref); + return ref; +} + + +LUALIB_API void luaL_unref (lua_State *L, int t, int ref) { + if (ref >= 0) { + t = abs_index(L, t); + lua_rawgeti(L, t, FREELIST_REF); + lua_rawseti(L, t, ref); /* t[ref] = t[FREELIST_REF] */ + lua_pushinteger(L, ref); + lua_rawseti(L, t, FREELIST_REF); /* t[FREELIST_REF] = ref */ + } +} + + + +/* +** {====================================================== +** Load functions +** ======================================================= +*/ + +typedef struct LoadF { + int extraline; + FILE *f; + char buff[LUAL_BUFFERSIZE]; +} LoadF; + + +static const char *getF (lua_State *L, void *ud, size_t *size) { + LoadF *lf = (LoadF *)ud; + (void)L; + if (lf->extraline) { + lf->extraline = 0; + *size = 1; + return "\n"; + } + if (feof(lf->f)) return NULL; + *size = fread(lf->buff, 1, sizeof(lf->buff), lf->f); + return (*size > 0) ? lf->buff : NULL; +} + + +static int errfile (lua_State *L, const char *what, int fnameindex) { + const char *serr = strerror(errno); + const char *filename = lua_tostring(L, fnameindex) + 1; + lua_pushfstring(L, "cannot %s %s: %s", what, filename, serr); + lua_remove(L, fnameindex); + return LUA_ERRFILE; +} + + +LUALIB_API int luaL_loadfile (lua_State *L, const char *filename) { + LoadF lf; + int status, readstatus; + int c; + int fnameindex = lua_gettop(L) + 1; /* index of filename on the stack */ + lf.extraline = 0; + if (filename == NULL) { + lua_pushliteral(L, "=stdin"); + lf.f = stdin; + } + else { + lua_pushfstring(L, "@%s", filename); + lf.f = fopen(filename, "r"); + if (lf.f == NULL) return errfile(L, "open", fnameindex); + } + c = getc(lf.f); + if (c == '#') { /* Unix exec. file? */ + lf.extraline = 1; + while ((c = getc(lf.f)) != EOF && c != '\n') ; /* skip first line */ + if (c == '\n') c = getc(lf.f); + } + if (c == LUA_SIGNATURE[0] && filename) { /* binary file? */ + lf.f = freopen(filename, "rb", lf.f); /* reopen in binary mode */ + if (lf.f == NULL) return errfile(L, "reopen", fnameindex); + /* skip eventual `#!...' */ + while ((c = getc(lf.f)) != EOF && c != LUA_SIGNATURE[0]) + ; + lf.extraline = 0; + } + ungetc(c, lf.f); + status = lua_load(L, getF, &lf, lua_tostring(L, -1)); + readstatus = ferror(lf.f); + if (filename) fclose(lf.f); /* close file (even in case of errors) */ + if (readstatus) { + lua_settop(L, fnameindex); /* ignore results from `lua_load' */ + return errfile(L, "read", fnameindex); + } + lua_remove(L, fnameindex); + return status; +} + + +typedef struct LoadS { + const char *s; + size_t size; +} LoadS; + + +static const char *getS (lua_State *L, void *ud, size_t *size) { + LoadS *ls = (LoadS *)ud; + (void)L; + if (ls->size == 0) return NULL; + *size = ls->size; + ls->size = 0; + return ls->s; +} + + +LUALIB_API int luaL_loadbuffer (lua_State *L, const char *buff, size_t size, + const char *name) { + LoadS ls; + ls.s = buff; + ls.size = size; + return lua_load(L, getS, &ls, name); +} + + +LUALIB_API int (luaL_loadstring) (lua_State *L, const char *s) { + return luaL_loadbuffer(L, s, strlen(s), s); +} + + + +/* }====================================================== */ + + +static void *l_alloc (void *ud, void *ptr, size_t osize, size_t nsize) { + (void)ud; + (void)osize; + if (nsize == 0) { + free(ptr); + return NULL; + } + else + return realloc(ptr, nsize); +} + + +static int panic (lua_State *L) { + (void)L; /* to avoid warnings */ + fprintf(stderr, "PANIC: unprotected error in call to Lua API (%s)\n", + lua_tostring(L, -1)); + return 0; +} + + +LUALIB_API lua_State *luaL_newstate (void) { + lua_State *L = lua_newstate(l_alloc, NULL); + if (L) lua_atpanic(L, &panic); + return L; +} + diff --git a/src/lua/lbaselib.c b/src/lua/lbaselib.c new file mode 100644 index 0000000..8552405 --- /dev/null +++ b/src/lua/lbaselib.c @@ -0,0 +1,657 @@ +/* +** $Id: lbaselib.c,v 1.191.1.6 2008/02/14 16:46:22 roberto Exp $ +** Basic library +** See Copyright Notice in lua.h +*/ + + + +#include +#include +#include +#include + +#define lbaselib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + + + +/* +** If your system does not support `stdout', you can just remove this function. +** If you need, you can define your own `print' function, following this +** model but changing `fputs' to put the strings at a proper place +** (a console window or a log file, for instance). +*/ +static int luaB_print (lua_State *L) { + int n = lua_gettop(L); /* number of arguments */ + int i; + lua_getglobal(L, "tostring"); + for (i=1; i<=n; i++) { + const char *s; + lua_pushvalue(L, -1); /* function to be called */ + lua_pushvalue(L, i); /* value to print */ + lua_call(L, 1, 1); + s = lua_tostring(L, -1); /* get result */ + if (s == NULL) + return luaL_error(L, LUA_QL("tostring") " must return a string to " + LUA_QL("print")); + if (i>1) fputs("\t", stdout); + fputs(s, stdout); + lua_pop(L, 1); /* pop result */ + } + fputs("\n", stdout); + return 0; +} + + +static int luaB_tonumber (lua_State *L) { + int base = luaL_optint(L, 2, 10); + if (base == 10) { /* standard conversion */ + luaL_checkany(L, 1); + if (lua_isnumber(L, 1)) { + lua_pushnumber(L, lua_tonumber(L, 1)); + return 1; + } + } + else { + const char *s1 = luaL_checkstring(L, 1); + char *s2; + unsigned long n; + luaL_argcheck(L, 2 <= base && base <= 36, 2, "base out of range"); + n = strtoul(s1, &s2, base); + if (s1 != s2) { /* at least one valid digit? */ + while (isspace((unsigned char)(*s2))) s2++; /* skip trailing spaces */ + if (*s2 == '\0') { /* no invalid trailing characters? */ + lua_pushnumber(L, (lua_Number)n); + return 1; + } + } + } + lua_pushnil(L); /* else not a number */ + return 1; +} + + +static int luaB_error (lua_State *L) { + int level = luaL_optint(L, 2, 1); + lua_settop(L, 1); + if (lua_isstring(L, 1) && level > 0) { /* add extra information? */ + luaL_where(L, level); + lua_pushvalue(L, 1); + lua_concat(L, 2); + } + return lua_error(L); +} + + +static int luaB_getmetatable (lua_State *L) { + luaL_checkany(L, 1); + if (!lua_getmetatable(L, 1)) { + lua_pushnil(L); + return 1; /* no metatable */ + } + luaL_getmetafield(L, 1, "__metatable"); + return 1; /* returns either __metatable field (if present) or metatable */ +} + + +static int luaB_setmetatable (lua_State *L) { + int t = lua_type(L, 2); + luaL_checktype(L, 1, LUA_TTABLE); + luaL_argcheck(L, t == LUA_TNIL || t == LUA_TTABLE, 2, + "nil or table expected"); + if (luaL_getmetafield(L, 1, "__metatable")) + luaL_error(L, "cannot change a protected metatable"); + lua_settop(L, 2); + lua_setmetatable(L, 1); + return 1; +} + + +static void getfunc (lua_State *L, int opt) { + if (lua_isfunction(L, 1)) lua_pushvalue(L, 1); + else { + lua_Debug ar; + int level = opt ? luaL_optint(L, 1, 1) : luaL_checkint(L, 1); + luaL_argcheck(L, level >= 0, 1, "level must be non-negative"); + if (lua_getstack(L, level, &ar) == 0) + luaL_argerror(L, 1, "invalid level"); + lua_getinfo(L, "f", &ar); + if (lua_isnil(L, -1)) + luaL_error(L, "no function environment for tail call at level %d", + level); + } +} + + +static int luaB_getfenv (lua_State *L) { + getfunc(L, 1); + if (lua_iscfunction(L, -1)) /* is a C function? */ + lua_pushvalue(L, LUA_GLOBALSINDEX); /* return the thread's global env. */ + else + lua_getfenv(L, -1); + return 1; +} + + +static int luaB_setfenv (lua_State *L) { + luaL_checktype(L, 2, LUA_TTABLE); + getfunc(L, 0); + lua_pushvalue(L, 2); + if (lua_isnumber(L, 1) && lua_tonumber(L, 1) == 0) { + /* change environment of current thread */ + lua_pushthread(L); + lua_insert(L, -2); + lua_setfenv(L, -2); + return 0; + } + else if (lua_iscfunction(L, -2) || lua_setfenv(L, -2) == 0) + luaL_error(L, + LUA_QL("setfenv") " cannot change environment of given object"); + return 1; +} + + +static int luaB_rawequal (lua_State *L) { + luaL_checkany(L, 1); + luaL_checkany(L, 2); + lua_pushboolean(L, lua_rawequal(L, 1, 2)); + return 1; +} + + +static int luaB_rawget (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + luaL_checkany(L, 2); + lua_settop(L, 2); + lua_rawget(L, 1); + return 1; +} + +static int luaB_rawset (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + luaL_checkany(L, 2); + luaL_checkany(L, 3); + lua_settop(L, 3); + lua_rawset(L, 1); + return 1; +} + + +static int luaB_gcinfo (lua_State *L) { + lua_pushinteger(L, lua_getgccount(L)); + return 1; +} + + +static int luaB_collectgarbage (lua_State *L) { + static const char *const opts[] = {"stop", "restart", "collect", + "count", "step", "setpause", "setstepmul", NULL}; + static const int optsnum[] = {LUA_GCSTOP, LUA_GCRESTART, LUA_GCCOLLECT, + LUA_GCCOUNT, LUA_GCSTEP, LUA_GCSETPAUSE, LUA_GCSETSTEPMUL}; + int o = luaL_checkoption(L, 1, "collect", opts); + int ex = luaL_optint(L, 2, 0); + int res = lua_gc(L, optsnum[o], ex); + switch (optsnum[o]) { + case LUA_GCCOUNT: { + int b = lua_gc(L, LUA_GCCOUNTB, 0); + lua_pushnumber(L, res + ((lua_Number)b/1024)); + return 1; + } + case LUA_GCSTEP: { + lua_pushboolean(L, res); + return 1; + } + default: { + lua_pushnumber(L, res); + return 1; + } + } +} + + +static int luaB_type (lua_State *L) { + luaL_checkany(L, 1); + lua_pushstring(L, luaL_typename(L, 1)); + return 1; +} + + +static int luaB_next (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_settop(L, 2); /* create a 2nd argument if there isn't one */ + if (lua_next(L, 1)) + return 2; + else { + lua_pushnil(L); + return 1; + } +} + + +static int luaB_pairs (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_pushvalue(L, lua_upvalueindex(1)); /* return generator, */ + lua_pushvalue(L, 1); /* state, */ + lua_pushnil(L); /* and initial value */ + return 3; +} + + +static int ipairsaux (lua_State *L) { + int i = luaL_checkint(L, 2); + luaL_checktype(L, 1, LUA_TTABLE); + i++; /* next value */ + lua_pushinteger(L, i); + lua_rawgeti(L, 1, i); + return (lua_isnil(L, -1)) ? 0 : 2; +} + + +static int luaB_ipairs (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + lua_pushvalue(L, lua_upvalueindex(1)); /* return generator, */ + lua_pushvalue(L, 1); /* state, */ + lua_pushinteger(L, 0); /* and initial value */ + return 3; +} + + +static int load_aux (lua_State *L, int status) { + if (status == 0) /* OK? */ + return 1; + else { + lua_pushnil(L); + lua_insert(L, -2); /* put before error message */ + return 2; /* return nil plus error message */ + } +} + + +static int luaB_loadstring (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + const char *chunkname = luaL_optstring(L, 2, s); + return load_aux(L, luaL_loadbuffer(L, s, l, chunkname)); +} + + +static int luaB_loadfile (lua_State *L) { + const char *fname = luaL_optstring(L, 1, NULL); + return load_aux(L, luaL_loadfile(L, fname)); +} + + +/* +** Reader for generic `load' function: `lua_load' uses the +** stack for internal stuff, so the reader cannot change the +** stack top. Instead, it keeps its resulting string in a +** reserved slot inside the stack. +*/ +static const char *generic_reader (lua_State *L, void *ud, size_t *size) { + (void)ud; /* to avoid warnings */ + luaL_checkstack(L, 2, "too many nested functions"); + lua_pushvalue(L, 1); /* get function */ + lua_call(L, 0, 1); /* call it */ + if (lua_isnil(L, -1)) { + *size = 0; + return NULL; + } + else if (lua_isstring(L, -1)) { + lua_replace(L, 3); /* save string in a reserved stack slot */ + return lua_tolstring(L, 3, size); + } + else luaL_error(L, "reader function must return a string"); + return NULL; /* to avoid warnings */ +} + + +static int luaB_load (lua_State *L) { + int status; + const char *cname = luaL_optstring(L, 2, "=(load)"); + luaL_checktype(L, 1, LUA_TFUNCTION); + lua_settop(L, 3); /* function, eventual name, plus one reserved slot */ + status = lua_load(L, generic_reader, NULL, cname); + return load_aux(L, status); +} + + +static int luaB_dofile (lua_State *L) { + const char *fname = luaL_optstring(L, 1, NULL); + int n = lua_gettop(L); + if (luaL_loadfile(L, fname) != 0) lua_error(L); + lua_call(L, 0, LUA_MULTRET); + return lua_gettop(L) - n; +} + + +static int luaB_assert (lua_State *L) { + luaL_checkany(L, 1); + if (!lua_toboolean(L, 1)) + return luaL_error(L, "%s", luaL_optstring(L, 2, "assertion failed!")); + return lua_gettop(L); +} + + +static int luaB_unpack (lua_State *L) { + int i, e, n; + luaL_checktype(L, 1, LUA_TTABLE); + i = luaL_optint(L, 2, 1); + e = luaL_opt(L, luaL_checkint, 3, luaL_getn(L, 1)); + if (i > e) return 0; /* empty range */ + n = e - i + 1; /* number of elements */ + if (n <= 0 || !lua_checkstack(L, n)) /* n <= 0 means arith. overflow */ + return luaL_error(L, "too many results to unpack"); + lua_rawgeti(L, 1, i); /* push arg[i] (avoiding overflow problems) */ + while (i++ < e) /* push arg[i + 1...e] */ + lua_rawgeti(L, 1, i); + return n; +} + + +static int luaB_select (lua_State *L) { + int n = lua_gettop(L); + if (lua_type(L, 1) == LUA_TSTRING && *lua_tostring(L, 1) == '#') { + lua_pushinteger(L, n-1); + return 1; + } + else { + int i = luaL_checkint(L, 1); + if (i < 0) i = n + i; + else if (i > n) i = n; + luaL_argcheck(L, 1 <= i, 1, "index out of range"); + return n - i; + } +} + + +static int luaB_pcall (lua_State *L) { + int status; + luaL_checkany(L, 1); + status = lua_pcall(L, lua_gettop(L) - 1, LUA_MULTRET, 0); + lua_pushboolean(L, (status == 0)); + lua_insert(L, 1); + return lua_gettop(L); /* return status + all results */ +} + + +static int luaB_xpcall (lua_State *L) { + int status; + luaL_checkany(L, 2); + lua_settop(L, 2); + lua_insert(L, 1); /* put error function under function to be called */ + status = lua_pcall(L, 0, LUA_MULTRET, 1); + lua_pushboolean(L, (status == 0)); + lua_replace(L, 1); + return lua_gettop(L); /* return status + all results */ +} + + +static int luaB_tostring (lua_State *L) { + luaL_checkany(L, 1); + if (luaL_callmeta(L, 1, "__tostring")) /* is there a metafield? */ + return 1; /* use its value */ + switch (lua_type(L, 1)) { + case LUA_TNUMBER: + lua_pushstring(L, lua_tostring(L, 1)); + break; + case LUA_TSTRING: + lua_pushvalue(L, 1); + break; + case LUA_TBOOLEAN: + lua_pushstring(L, (lua_toboolean(L, 1) ? "true" : "false")); + break; + case LUA_TNIL: + lua_pushliteral(L, "nil"); + break; + default: + lua_pushfstring(L, "%s: %p", luaL_typename(L, 1), lua_topointer(L, 1)); + break; + } + return 1; +} + + +static int luaB_newproxy (lua_State *L) { + lua_settop(L, 1); + lua_newuserdata(L, 0); /* create proxy */ + if (lua_toboolean(L, 1) == 0) + return 1; /* no metatable */ + else if (lua_isboolean(L, 1)) { + lua_newtable(L); /* create a new metatable `m' ... */ + lua_pushvalue(L, -1); /* ... and mark `m' as a valid metatable */ + lua_pushboolean(L, 1); + lua_rawset(L, lua_upvalueindex(1)); /* weaktable[m] = true */ + } + else { + int validproxy = 0; /* to check if weaktable[metatable(u)] == true */ + if (lua_getmetatable(L, 1)) { + lua_rawget(L, lua_upvalueindex(1)); + validproxy = lua_toboolean(L, -1); + lua_pop(L, 1); /* remove value */ + } + luaL_argcheck(L, validproxy, 1, "boolean or proxy expected"); + lua_getmetatable(L, 1); /* metatable is valid; get it */ + } + lua_setmetatable(L, 2); + return 1; +} + + +static const luaL_Reg base_funcs[] = { + {"assert", luaB_assert}, + {"collectgarbage", luaB_collectgarbage}, + {"dofile", luaB_dofile}, + {"error", luaB_error}, + {"gcinfo", luaB_gcinfo}, + {"getfenv", luaB_getfenv}, + {"getmetatable", luaB_getmetatable}, + {"loadfile", luaB_loadfile}, + {"load", luaB_load}, + {"loadstring", luaB_loadstring}, + {"next", luaB_next}, + {"pcall", luaB_pcall}, + {"print", luaB_print}, + {"rawequal", luaB_rawequal}, + {"rawget", luaB_rawget}, + {"rawset", luaB_rawset}, + {"select", luaB_select}, + {"setfenv", luaB_setfenv}, + {"setmetatable", luaB_setmetatable}, + {"tonumber", luaB_tonumber}, + {"tostring", luaB_tostring}, + {"type", luaB_type}, + {"unpack", luaB_unpack}, + {"xpcall", luaB_xpcall}, + {NULL, NULL} +}; + + +/* +** {====================================================== +** Coroutine library +** ======================================================= +*/ + +#define CO_RUN 0 /* running */ +#define CO_SUS 1 /* suspended */ +#define CO_NOR 2 /* 'normal' (it resumed another coroutine) */ +#define CO_DEAD 3 + +static const char *const statnames[] = + {"running", "suspended", "normal", "dead"}; + +static int costatus (lua_State *L, lua_State *co) { + if (L == co) return CO_RUN; + switch (lua_status(co)) { + case LUA_YIELD: + return CO_SUS; + case 0: { + lua_Debug ar; + if (lua_getstack(co, 0, &ar) > 0) /* does it have frames? */ + return CO_NOR; /* it is running */ + else if (lua_gettop(co) == 0) + return CO_DEAD; + else + return CO_SUS; /* initial state */ + } + default: /* some error occured */ + return CO_DEAD; + } +} + + +static int luaB_costatus (lua_State *L) { + lua_State *co = lua_tothread(L, 1); + luaL_argcheck(L, co, 1, "coroutine expected"); + lua_pushstring(L, statnames[costatus(L, co)]); + return 1; +} + + +static int auxresume (lua_State *L, lua_State *co, int narg) { + int status = costatus(L, co); + if (!lua_checkstack(co, narg)) + luaL_error(L, "too many arguments to resume"); + if (status != CO_SUS) { + lua_pushfstring(L, "cannot resume %s coroutine", statnames[status]); + return -1; /* error flag */ + } + lua_xmove(L, co, narg); + lua_setlevel(L, co); + status = lua_resume(co, narg); + if (status == 0 || status == LUA_YIELD) { + int nres = lua_gettop(co); + if (!lua_checkstack(L, nres + 1)) + luaL_error(L, "too many results to resume"); + lua_xmove(co, L, nres); /* move yielded values */ + return nres; + } + else { + lua_xmove(co, L, 1); /* move error message */ + return -1; /* error flag */ + } +} + + +static int luaB_coresume (lua_State *L) { + lua_State *co = lua_tothread(L, 1); + int r; + luaL_argcheck(L, co, 1, "coroutine expected"); + r = auxresume(L, co, lua_gettop(L) - 1); + if (r < 0) { + lua_pushboolean(L, 0); + lua_insert(L, -2); + return 2; /* return false + error message */ + } + else { + lua_pushboolean(L, 1); + lua_insert(L, -(r + 1)); + return r + 1; /* return true + `resume' returns */ + } +} + + +static int luaB_auxwrap (lua_State *L) { + lua_State *co = lua_tothread(L, lua_upvalueindex(1)); + int r = auxresume(L, co, lua_gettop(L)); + if (r < 0) { + if (lua_isstring(L, -1)) { /* error object is a string? */ + luaL_where(L, 1); /* add extra info */ + lua_insert(L, -2); + lua_concat(L, 2); + } + lua_error(L); /* propagate error */ + } + return r; +} + + +static int luaB_cocreate (lua_State *L) { + lua_State *NL = lua_newthread(L); + luaL_argcheck(L, lua_isfunction(L, 1) && !lua_iscfunction(L, 1), 1, + "Lua function expected"); + lua_pushvalue(L, 1); /* move function to top */ + lua_xmove(L, NL, 1); /* move function from L to NL */ + return 1; +} + + +static int luaB_cowrap (lua_State *L) { + luaB_cocreate(L); + lua_pushcclosure(L, luaB_auxwrap, 1); + return 1; +} + + +static int luaB_yield (lua_State *L) { + return lua_yield(L, lua_gettop(L)); +} + + +static int luaB_corunning (lua_State *L) { + if (lua_pushthread(L)) + lua_pushnil(L); /* main thread is not a coroutine */ + return 1; +} + + +static const luaL_Reg co_funcs[] = { + {"create", luaB_cocreate}, + {"resume", luaB_coresume}, + {"running", luaB_corunning}, + {"status", luaB_costatus}, + {"wrap", luaB_cowrap}, + {"yield", luaB_yield}, + {NULL, NULL} +}; + +/* }====================================================== */ + + +static void auxopen (lua_State *L, const char *name, + lua_CFunction f, lua_CFunction u) { + lua_pushcfunction(L, u); + lua_pushcclosure(L, f, 1); + lua_setfield(L, -2, name); +} + + +static void base_open (lua_State *L) { + /* set global _G */ + lua_pushvalue(L, LUA_GLOBALSINDEX); + lua_setglobal(L, "_G"); + /* open lib into global table */ + luaL_register(L, "_G", base_funcs); + lua_pushliteral(L, LUA_VERSION); + lua_setglobal(L, "_VERSION"); /* set global _VERSION */ + /* `ipairs' and `pairs' need auxiliary functions as upvalues */ + auxopen(L, "ipairs", luaB_ipairs, ipairsaux); + auxopen(L, "pairs", luaB_pairs, luaB_next); + /* `newproxy' needs a weaktable as upvalue */ + lua_createtable(L, 0, 1); /* new table `w' */ + lua_pushvalue(L, -1); /* `w' will be its own metatable */ + lua_setmetatable(L, -2); + lua_pushliteral(L, "kv"); + lua_setfield(L, -2, "__mode"); /* metatable(w).__mode = "kv" */ + lua_pushcclosure(L, luaB_newproxy, 1); + lua_setglobal(L, "newproxy"); /* set global `newproxy' */ +} + + +LUALIB_API int luaopen_base (lua_State *L) { + base_open(L); + return 1; +} + +LUALIB_API int luaopen_coroutine (lua_State *L) { + luaL_register(L, LUA_COLIBNAME, co_funcs); + return 1; +} + diff --git a/src/lua/lcode.c b/src/lua/lcode.c new file mode 100644 index 0000000..679cb9c --- /dev/null +++ b/src/lua/lcode.c @@ -0,0 +1,831 @@ +/* +** $Id: lcode.c,v 2.25.1.5 2011/01/31 14:53:16 roberto Exp $ +** Code generator for Lua +** See Copyright Notice in lua.h +*/ + + +#include + +#define lcode_c +#define LUA_CORE + +#include "lua.h" + +#include "lcode.h" +#include "ldebug.h" +#include "ldo.h" +#include "lgc.h" +#include "llex.h" +#include "lmem.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" +#include "ltable.h" + + +#define hasjumps(e) ((e)->t != (e)->f) + + +static int isnumeral(expdesc *e) { + return (e->k == VKNUM && e->t == NO_JUMP && e->f == NO_JUMP); +} + + +void luaK_nil (FuncState *fs, int from, int n) { + Instruction *previous; + if (fs->pc > fs->lasttarget) { /* no jumps to current position? */ + if (fs->pc == 0) { /* function start? */ + if (from >= fs->nactvar) + return; /* positions are already clean */ + } + else { + previous = &fs->f->code[fs->pc-1]; + if (GET_OPCODE(*previous) == OP_LOADNIL) { + int pfrom = GETARG_A(*previous); + int pto = GETARG_B(*previous); + if (pfrom <= from && from <= pto+1) { /* can connect both? */ + if (from+n-1 > pto) + SETARG_B(*previous, from+n-1); + return; + } + } + } + } + luaK_codeABC(fs, OP_LOADNIL, from, from+n-1, 0); /* else no optimization */ +} + + +int luaK_jump (FuncState *fs) { + int jpc = fs->jpc; /* save list of jumps to here */ + int j; + fs->jpc = NO_JUMP; + j = luaK_codeAsBx(fs, OP_JMP, 0, NO_JUMP); + luaK_concat(fs, &j, jpc); /* keep them on hold */ + return j; +} + + +void luaK_ret (FuncState *fs, int first, int nret) { + luaK_codeABC(fs, OP_RETURN, first, nret+1, 0); +} + + +static int condjump (FuncState *fs, OpCode op, int A, int B, int C) { + luaK_codeABC(fs, op, A, B, C); + return luaK_jump(fs); +} + + +static void fixjump (FuncState *fs, int pc, int dest) { + Instruction *jmp = &fs->f->code[pc]; + int offset = dest-(pc+1); + lua_assert(dest != NO_JUMP); + if (abs(offset) > MAXARG_sBx) + luaX_syntaxerror(fs->ls, "control structure too long"); + SETARG_sBx(*jmp, offset); +} + + +/* +** returns current `pc' and marks it as a jump target (to avoid wrong +** optimizations with consecutive instructions not in the same basic block). +*/ +int luaK_getlabel (FuncState *fs) { + fs->lasttarget = fs->pc; + return fs->pc; +} + + +static int getjump (FuncState *fs, int pc) { + int offset = GETARG_sBx(fs->f->code[pc]); + if (offset == NO_JUMP) /* point to itself represents end of list */ + return NO_JUMP; /* end of list */ + else + return (pc+1)+offset; /* turn offset into absolute position */ +} + + +static Instruction *getjumpcontrol (FuncState *fs, int pc) { + Instruction *pi = &fs->f->code[pc]; + if (pc >= 1 && testTMode(GET_OPCODE(*(pi-1)))) + return pi-1; + else + return pi; +} + + +/* +** check whether list has any jump that do not produce a value +** (or produce an inverted value) +*/ +static int need_value (FuncState *fs, int list) { + for (; list != NO_JUMP; list = getjump(fs, list)) { + Instruction i = *getjumpcontrol(fs, list); + if (GET_OPCODE(i) != OP_TESTSET) return 1; + } + return 0; /* not found */ +} + + +static int patchtestreg (FuncState *fs, int node, int reg) { + Instruction *i = getjumpcontrol(fs, node); + if (GET_OPCODE(*i) != OP_TESTSET) + return 0; /* cannot patch other instructions */ + if (reg != NO_REG && reg != GETARG_B(*i)) + SETARG_A(*i, reg); + else /* no register to put value or register already has the value */ + *i = CREATE_ABC(OP_TEST, GETARG_B(*i), 0, GETARG_C(*i)); + + return 1; +} + + +static void removevalues (FuncState *fs, int list) { + for (; list != NO_JUMP; list = getjump(fs, list)) + patchtestreg(fs, list, NO_REG); +} + + +static void patchlistaux (FuncState *fs, int list, int vtarget, int reg, + int dtarget) { + while (list != NO_JUMP) { + int next = getjump(fs, list); + if (patchtestreg(fs, list, reg)) + fixjump(fs, list, vtarget); + else + fixjump(fs, list, dtarget); /* jump to default target */ + list = next; + } +} + + +static void dischargejpc (FuncState *fs) { + patchlistaux(fs, fs->jpc, fs->pc, NO_REG, fs->pc); + fs->jpc = NO_JUMP; +} + + +void luaK_patchlist (FuncState *fs, int list, int target) { + if (target == fs->pc) + luaK_patchtohere(fs, list); + else { + lua_assert(target < fs->pc); + patchlistaux(fs, list, target, NO_REG, target); + } +} + + +void luaK_patchtohere (FuncState *fs, int list) { + luaK_getlabel(fs); + luaK_concat(fs, &fs->jpc, list); +} + + +void luaK_concat (FuncState *fs, int *l1, int l2) { + if (l2 == NO_JUMP) return; + else if (*l1 == NO_JUMP) + *l1 = l2; + else { + int list = *l1; + int next; + while ((next = getjump(fs, list)) != NO_JUMP) /* find last element */ + list = next; + fixjump(fs, list, l2); + } +} + + +void luaK_checkstack (FuncState *fs, int n) { + int newstack = fs->freereg + n; + if (newstack > fs->f->maxstacksize) { + if (newstack >= MAXSTACK) + luaX_syntaxerror(fs->ls, "function or expression too complex"); + fs->f->maxstacksize = cast_byte(newstack); + } +} + + +void luaK_reserveregs (FuncState *fs, int n) { + luaK_checkstack(fs, n); + fs->freereg += n; +} + + +static void freereg (FuncState *fs, int reg) { + if (!ISK(reg) && reg >= fs->nactvar) { + fs->freereg--; + lua_assert(reg == fs->freereg); + } +} + + +static void freeexp (FuncState *fs, expdesc *e) { + if (e->k == VNONRELOC) + freereg(fs, e->u.s.info); +} + + +static int addk (FuncState *fs, TValue *k, TValue *v) { + lua_State *L = fs->L; + TValue *idx = luaH_set(L, fs->h, k); + Proto *f = fs->f; + int oldsize = f->sizek; + if (ttisnumber(idx)) { + lua_assert(luaO_rawequalObj(&fs->f->k[cast_int(nvalue(idx))], v)); + return cast_int(nvalue(idx)); + } + else { /* constant not found; create a new entry */ + setnvalue(idx, cast_num(fs->nk)); + luaM_growvector(L, f->k, fs->nk, f->sizek, TValue, + MAXARG_Bx, "constant table overflow"); + while (oldsize < f->sizek) setnilvalue(&f->k[oldsize++]); + setobj(L, &f->k[fs->nk], v); + luaC_barrier(L, f, v); + return fs->nk++; + } +} + + +int luaK_stringK (FuncState *fs, TString *s) { + TValue o; + setsvalue(fs->L, &o, s); + return addk(fs, &o, &o); +} + + +int luaK_numberK (FuncState *fs, lua_Number r) { + TValue o; + setnvalue(&o, r); + return addk(fs, &o, &o); +} + + +static int boolK (FuncState *fs, int b) { + TValue o; + setbvalue(&o, b); + return addk(fs, &o, &o); +} + + +static int nilK (FuncState *fs) { + TValue k, v; + setnilvalue(&v); + /* cannot use nil as key; instead use table itself to represent nil */ + sethvalue(fs->L, &k, fs->h); + return addk(fs, &k, &v); +} + + +void luaK_setreturns (FuncState *fs, expdesc *e, int nresults) { + if (e->k == VCALL) { /* expression is an open function call? */ + SETARG_C(getcode(fs, e), nresults+1); + } + else if (e->k == VVARARG) { + SETARG_B(getcode(fs, e), nresults+1); + SETARG_A(getcode(fs, e), fs->freereg); + luaK_reserveregs(fs, 1); + } +} + + +void luaK_setoneret (FuncState *fs, expdesc *e) { + if (e->k == VCALL) { /* expression is an open function call? */ + e->k = VNONRELOC; + e->u.s.info = GETARG_A(getcode(fs, e)); + } + else if (e->k == VVARARG) { + SETARG_B(getcode(fs, e), 2); + e->k = VRELOCABLE; /* can relocate its simple result */ + } +} + + +void luaK_dischargevars (FuncState *fs, expdesc *e) { + switch (e->k) { + case VLOCAL: { + e->k = VNONRELOC; + break; + } + case VUPVAL: { + e->u.s.info = luaK_codeABC(fs, OP_GETUPVAL, 0, e->u.s.info, 0); + e->k = VRELOCABLE; + break; + } + case VGLOBAL: { + e->u.s.info = luaK_codeABx(fs, OP_GETGLOBAL, 0, e->u.s.info); + e->k = VRELOCABLE; + break; + } + case VINDEXED: { + freereg(fs, e->u.s.aux); + freereg(fs, e->u.s.info); + e->u.s.info = luaK_codeABC(fs, OP_GETTABLE, 0, e->u.s.info, e->u.s.aux); + e->k = VRELOCABLE; + break; + } + case VVARARG: + case VCALL: { + luaK_setoneret(fs, e); + break; + } + default: break; /* there is one value available (somewhere) */ + } +} + + +static int code_label (FuncState *fs, int A, int b, int jump) { + luaK_getlabel(fs); /* those instructions may be jump targets */ + return luaK_codeABC(fs, OP_LOADBOOL, A, b, jump); +} + + +static void discharge2reg (FuncState *fs, expdesc *e, int reg) { + luaK_dischargevars(fs, e); + switch (e->k) { + case VNIL: { + luaK_nil(fs, reg, 1); + break; + } + case VFALSE: case VTRUE: { + luaK_codeABC(fs, OP_LOADBOOL, reg, e->k == VTRUE, 0); + break; + } + case VK: { + luaK_codeABx(fs, OP_LOADK, reg, e->u.s.info); + break; + } + case VKNUM: { + luaK_codeABx(fs, OP_LOADK, reg, luaK_numberK(fs, e->u.nval)); + break; + } + case VRELOCABLE: { + Instruction *pc = &getcode(fs, e); + SETARG_A(*pc, reg); + break; + } + case VNONRELOC: { + if (reg != e->u.s.info) + luaK_codeABC(fs, OP_MOVE, reg, e->u.s.info, 0); + break; + } + default: { + lua_assert(e->k == VVOID || e->k == VJMP); + return; /* nothing to do... */ + } + } + e->u.s.info = reg; + e->k = VNONRELOC; +} + + +static void discharge2anyreg (FuncState *fs, expdesc *e) { + if (e->k != VNONRELOC) { + luaK_reserveregs(fs, 1); + discharge2reg(fs, e, fs->freereg-1); + } +} + + +static void exp2reg (FuncState *fs, expdesc *e, int reg) { + discharge2reg(fs, e, reg); + if (e->k == VJMP) + luaK_concat(fs, &e->t, e->u.s.info); /* put this jump in `t' list */ + if (hasjumps(e)) { + int final; /* position after whole expression */ + int p_f = NO_JUMP; /* position of an eventual LOAD false */ + int p_t = NO_JUMP; /* position of an eventual LOAD true */ + if (need_value(fs, e->t) || need_value(fs, e->f)) { + int fj = (e->k == VJMP) ? NO_JUMP : luaK_jump(fs); + p_f = code_label(fs, reg, 0, 1); + p_t = code_label(fs, reg, 1, 0); + luaK_patchtohere(fs, fj); + } + final = luaK_getlabel(fs); + patchlistaux(fs, e->f, final, reg, p_f); + patchlistaux(fs, e->t, final, reg, p_t); + } + e->f = e->t = NO_JUMP; + e->u.s.info = reg; + e->k = VNONRELOC; +} + + +void luaK_exp2nextreg (FuncState *fs, expdesc *e) { + luaK_dischargevars(fs, e); + freeexp(fs, e); + luaK_reserveregs(fs, 1); + exp2reg(fs, e, fs->freereg - 1); +} + + +int luaK_exp2anyreg (FuncState *fs, expdesc *e) { + luaK_dischargevars(fs, e); + if (e->k == VNONRELOC) { + if (!hasjumps(e)) return e->u.s.info; /* exp is already in a register */ + if (e->u.s.info >= fs->nactvar) { /* reg. is not a local? */ + exp2reg(fs, e, e->u.s.info); /* put value on it */ + return e->u.s.info; + } + } + luaK_exp2nextreg(fs, e); /* default */ + return e->u.s.info; +} + + +void luaK_exp2val (FuncState *fs, expdesc *e) { + if (hasjumps(e)) + luaK_exp2anyreg(fs, e); + else + luaK_dischargevars(fs, e); +} + + +int luaK_exp2RK (FuncState *fs, expdesc *e) { + luaK_exp2val(fs, e); + switch (e->k) { + case VKNUM: + case VTRUE: + case VFALSE: + case VNIL: { + if (fs->nk <= MAXINDEXRK) { /* constant fit in RK operand? */ + e->u.s.info = (e->k == VNIL) ? nilK(fs) : + (e->k == VKNUM) ? luaK_numberK(fs, e->u.nval) : + boolK(fs, (e->k == VTRUE)); + e->k = VK; + return RKASK(e->u.s.info); + } + else break; + } + case VK: { + if (e->u.s.info <= MAXINDEXRK) /* constant fit in argC? */ + return RKASK(e->u.s.info); + else break; + } + default: break; + } + /* not a constant in the right range: put it in a register */ + return luaK_exp2anyreg(fs, e); +} + + +void luaK_storevar (FuncState *fs, expdesc *var, expdesc *ex) { + switch (var->k) { + case VLOCAL: { + freeexp(fs, ex); + exp2reg(fs, ex, var->u.s.info); + return; + } + case VUPVAL: { + int e = luaK_exp2anyreg(fs, ex); + luaK_codeABC(fs, OP_SETUPVAL, e, var->u.s.info, 0); + break; + } + case VGLOBAL: { + int e = luaK_exp2anyreg(fs, ex); + luaK_codeABx(fs, OP_SETGLOBAL, e, var->u.s.info); + break; + } + case VINDEXED: { + int e = luaK_exp2RK(fs, ex); + luaK_codeABC(fs, OP_SETTABLE, var->u.s.info, var->u.s.aux, e); + break; + } + default: { + lua_assert(0); /* invalid var kind to store */ + break; + } + } + freeexp(fs, ex); +} + + +void luaK_self (FuncState *fs, expdesc *e, expdesc *key) { + int func; + luaK_exp2anyreg(fs, e); + freeexp(fs, e); + func = fs->freereg; + luaK_reserveregs(fs, 2); + luaK_codeABC(fs, OP_SELF, func, e->u.s.info, luaK_exp2RK(fs, key)); + freeexp(fs, key); + e->u.s.info = func; + e->k = VNONRELOC; +} + + +static void invertjump (FuncState *fs, expdesc *e) { + Instruction *pc = getjumpcontrol(fs, e->u.s.info); + lua_assert(testTMode(GET_OPCODE(*pc)) && GET_OPCODE(*pc) != OP_TESTSET && + GET_OPCODE(*pc) != OP_TEST); + SETARG_A(*pc, !(GETARG_A(*pc))); +} + + +static int jumponcond (FuncState *fs, expdesc *e, int cond) { + if (e->k == VRELOCABLE) { + Instruction ie = getcode(fs, e); + if (GET_OPCODE(ie) == OP_NOT) { + fs->pc--; /* remove previous OP_NOT */ + return condjump(fs, OP_TEST, GETARG_B(ie), 0, !cond); + } + /* else go through */ + } + discharge2anyreg(fs, e); + freeexp(fs, e); + return condjump(fs, OP_TESTSET, NO_REG, e->u.s.info, cond); +} + + +void luaK_goiftrue (FuncState *fs, expdesc *e) { + int pc; /* pc of last jump */ + luaK_dischargevars(fs, e); + switch (e->k) { + case VK: case VKNUM: case VTRUE: { + pc = NO_JUMP; /* always true; do nothing */ + break; + } + case VJMP: { + invertjump(fs, e); + pc = e->u.s.info; + break; + } + default: { + pc = jumponcond(fs, e, 0); + break; + } + } + luaK_concat(fs, &e->f, pc); /* insert last jump in `f' list */ + luaK_patchtohere(fs, e->t); + e->t = NO_JUMP; +} + + +static void luaK_goiffalse (FuncState *fs, expdesc *e) { + int pc; /* pc of last jump */ + luaK_dischargevars(fs, e); + switch (e->k) { + case VNIL: case VFALSE: { + pc = NO_JUMP; /* always false; do nothing */ + break; + } + case VJMP: { + pc = e->u.s.info; + break; + } + default: { + pc = jumponcond(fs, e, 1); + break; + } + } + luaK_concat(fs, &e->t, pc); /* insert last jump in `t' list */ + luaK_patchtohere(fs, e->f); + e->f = NO_JUMP; +} + + +static void codenot (FuncState *fs, expdesc *e) { + luaK_dischargevars(fs, e); + switch (e->k) { + case VNIL: case VFALSE: { + e->k = VTRUE; + break; + } + case VK: case VKNUM: case VTRUE: { + e->k = VFALSE; + break; + } + case VJMP: { + invertjump(fs, e); + break; + } + case VRELOCABLE: + case VNONRELOC: { + discharge2anyreg(fs, e); + freeexp(fs, e); + e->u.s.info = luaK_codeABC(fs, OP_NOT, 0, e->u.s.info, 0); + e->k = VRELOCABLE; + break; + } + default: { + lua_assert(0); /* cannot happen */ + break; + } + } + /* interchange true and false lists */ + { int temp = e->f; e->f = e->t; e->t = temp; } + removevalues(fs, e->f); + removevalues(fs, e->t); +} + + +void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k) { + t->u.s.aux = luaK_exp2RK(fs, k); + t->k = VINDEXED; +} + + +static int constfolding (OpCode op, expdesc *e1, expdesc *e2) { + lua_Number v1, v2, r; + if (!isnumeral(e1) || !isnumeral(e2)) return 0; + v1 = e1->u.nval; + v2 = e2->u.nval; + switch (op) { + case OP_ADD: r = luai_numadd(v1, v2); break; + case OP_SUB: r = luai_numsub(v1, v2); break; + case OP_MUL: r = luai_nummul(v1, v2); break; + case OP_DIV: + if (v2 == 0) return 0; /* do not attempt to divide by 0 */ + r = luai_numdiv(v1, v2); break; + case OP_MOD: + if (v2 == 0) return 0; /* do not attempt to divide by 0 */ + r = luai_nummod(v1, v2); break; + case OP_POW: r = luai_numpow(v1, v2); break; + case OP_UNM: r = luai_numunm(v1); break; + case OP_LEN: return 0; /* no constant folding for 'len' */ + default: lua_assert(0); r = 0; break; + } + if (luai_numisnan(r)) return 0; /* do not attempt to produce NaN */ + e1->u.nval = r; + return 1; +} + + +static void codearith (FuncState *fs, OpCode op, expdesc *e1, expdesc *e2) { + if (constfolding(op, e1, e2)) + return; + else { + int o2 = (op != OP_UNM && op != OP_LEN) ? luaK_exp2RK(fs, e2) : 0; + int o1 = luaK_exp2RK(fs, e1); + if (o1 > o2) { + freeexp(fs, e1); + freeexp(fs, e2); + } + else { + freeexp(fs, e2); + freeexp(fs, e1); + } + e1->u.s.info = luaK_codeABC(fs, op, 0, o1, o2); + e1->k = VRELOCABLE; + } +} + + +static void codecomp (FuncState *fs, OpCode op, int cond, expdesc *e1, + expdesc *e2) { + int o1 = luaK_exp2RK(fs, e1); + int o2 = luaK_exp2RK(fs, e2); + freeexp(fs, e2); + freeexp(fs, e1); + if (cond == 0 && op != OP_EQ) { + int temp; /* exchange args to replace by `<' or `<=' */ + temp = o1; o1 = o2; o2 = temp; /* o1 <==> o2 */ + cond = 1; + } + e1->u.s.info = condjump(fs, op, cond, o1, o2); + e1->k = VJMP; +} + + +void luaK_prefix (FuncState *fs, UnOpr op, expdesc *e) { + expdesc e2; + e2.t = e2.f = NO_JUMP; e2.k = VKNUM; e2.u.nval = 0; + switch (op) { + case OPR_MINUS: { + if (!isnumeral(e)) + luaK_exp2anyreg(fs, e); /* cannot operate on non-numeric constants */ + codearith(fs, OP_UNM, e, &e2); + break; + } + case OPR_NOT: codenot(fs, e); break; + case OPR_LEN: { + luaK_exp2anyreg(fs, e); /* cannot operate on constants */ + codearith(fs, OP_LEN, e, &e2); + break; + } + default: lua_assert(0); + } +} + + +void luaK_infix (FuncState *fs, BinOpr op, expdesc *v) { + switch (op) { + case OPR_AND: { + luaK_goiftrue(fs, v); + break; + } + case OPR_OR: { + luaK_goiffalse(fs, v); + break; + } + case OPR_CONCAT: { + luaK_exp2nextreg(fs, v); /* operand must be on the `stack' */ + break; + } + case OPR_ADD: case OPR_SUB: case OPR_MUL: case OPR_DIV: + case OPR_MOD: case OPR_POW: { + if (!isnumeral(v)) luaK_exp2RK(fs, v); + break; + } + default: { + luaK_exp2RK(fs, v); + break; + } + } +} + + +void luaK_posfix (FuncState *fs, BinOpr op, expdesc *e1, expdesc *e2) { + switch (op) { + case OPR_AND: { + lua_assert(e1->t == NO_JUMP); /* list must be closed */ + luaK_dischargevars(fs, e2); + luaK_concat(fs, &e2->f, e1->f); + *e1 = *e2; + break; + } + case OPR_OR: { + lua_assert(e1->f == NO_JUMP); /* list must be closed */ + luaK_dischargevars(fs, e2); + luaK_concat(fs, &e2->t, e1->t); + *e1 = *e2; + break; + } + case OPR_CONCAT: { + luaK_exp2val(fs, e2); + if (e2->k == VRELOCABLE && GET_OPCODE(getcode(fs, e2)) == OP_CONCAT) { + lua_assert(e1->u.s.info == GETARG_B(getcode(fs, e2))-1); + freeexp(fs, e1); + SETARG_B(getcode(fs, e2), e1->u.s.info); + e1->k = VRELOCABLE; e1->u.s.info = e2->u.s.info; + } + else { + luaK_exp2nextreg(fs, e2); /* operand must be on the 'stack' */ + codearith(fs, OP_CONCAT, e1, e2); + } + break; + } + case OPR_ADD: codearith(fs, OP_ADD, e1, e2); break; + case OPR_SUB: codearith(fs, OP_SUB, e1, e2); break; + case OPR_MUL: codearith(fs, OP_MUL, e1, e2); break; + case OPR_DIV: codearith(fs, OP_DIV, e1, e2); break; + case OPR_MOD: codearith(fs, OP_MOD, e1, e2); break; + case OPR_POW: codearith(fs, OP_POW, e1, e2); break; + case OPR_EQ: codecomp(fs, OP_EQ, 1, e1, e2); break; + case OPR_NE: codecomp(fs, OP_EQ, 0, e1, e2); break; + case OPR_LT: codecomp(fs, OP_LT, 1, e1, e2); break; + case OPR_LE: codecomp(fs, OP_LE, 1, e1, e2); break; + case OPR_GT: codecomp(fs, OP_LT, 0, e1, e2); break; + case OPR_GE: codecomp(fs, OP_LE, 0, e1, e2); break; + default: lua_assert(0); + } +} + + +void luaK_fixline (FuncState *fs, int line) { + fs->f->lineinfo[fs->pc - 1] = line; +} + + +static int luaK_code (FuncState *fs, Instruction i, int line) { + Proto *f = fs->f; + dischargejpc(fs); /* `pc' will change */ + /* put new instruction in code array */ + luaM_growvector(fs->L, f->code, fs->pc, f->sizecode, Instruction, + MAX_INT, "code size overflow"); + f->code[fs->pc] = i; + /* save corresponding line information */ + luaM_growvector(fs->L, f->lineinfo, fs->pc, f->sizelineinfo, int, + MAX_INT, "code size overflow"); + f->lineinfo[fs->pc] = line; + return fs->pc++; +} + + +int luaK_codeABC (FuncState *fs, OpCode o, int a, int b, int c) { + lua_assert(getOpMode(o) == iABC); + lua_assert(getBMode(o) != OpArgN || b == 0); + lua_assert(getCMode(o) != OpArgN || c == 0); + return luaK_code(fs, CREATE_ABC(o, a, b, c), fs->ls->lastline); +} + + +int luaK_codeABx (FuncState *fs, OpCode o, int a, unsigned int bc) { + lua_assert(getOpMode(o) == iABx || getOpMode(o) == iAsBx); + lua_assert(getCMode(o) == OpArgN); + return luaK_code(fs, CREATE_ABx(o, a, bc), fs->ls->lastline); +} + + +void luaK_setlist (FuncState *fs, int base, int nelems, int tostore) { + int c = (nelems - 1)/LFIELDS_PER_FLUSH + 1; + int b = (tostore == LUA_MULTRET) ? 0 : tostore; + lua_assert(tostore != 0); + if (c <= MAXARG_C) + luaK_codeABC(fs, OP_SETLIST, base, b, c); + else { + luaK_codeABC(fs, OP_SETLIST, base, b, 0); + luaK_code(fs, cast(Instruction, c), fs->ls->lastline); + } + fs->freereg = base + 1; /* free registers with list values */ +} + diff --git a/src/lua/lcode.h b/src/lua/lcode.h new file mode 100644 index 0000000..b941c60 --- /dev/null +++ b/src/lua/lcode.h @@ -0,0 +1,76 @@ +/* +** $Id: lcode.h,v 1.48.1.1 2007/12/27 13:02:25 roberto Exp $ +** Code generator for Lua +** See Copyright Notice in lua.h +*/ + +#ifndef lcode_h +#define lcode_h + +#include "llex.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" + + +/* +** Marks the end of a patch list. It is an invalid value both as an absolute +** address, and as a list link (would link an element to itself). +*/ +#define NO_JUMP (-1) + + +/* +** grep "ORDER OPR" if you change these enums +*/ +typedef enum BinOpr { + OPR_ADD, OPR_SUB, OPR_MUL, OPR_DIV, OPR_MOD, OPR_POW, + OPR_CONCAT, + OPR_NE, OPR_EQ, + OPR_LT, OPR_LE, OPR_GT, OPR_GE, + OPR_AND, OPR_OR, + OPR_NOBINOPR +} BinOpr; + + +typedef enum UnOpr { OPR_MINUS, OPR_NOT, OPR_LEN, OPR_NOUNOPR } UnOpr; + + +#define getcode(fs,e) ((fs)->f->code[(e)->u.s.info]) + +#define luaK_codeAsBx(fs,o,A,sBx) luaK_codeABx(fs,o,A,(sBx)+MAXARG_sBx) + +#define luaK_setmultret(fs,e) luaK_setreturns(fs, e, LUA_MULTRET) + +LUAI_FUNC int luaK_codeABx (FuncState *fs, OpCode o, int A, unsigned int Bx); +LUAI_FUNC int luaK_codeABC (FuncState *fs, OpCode o, int A, int B, int C); +LUAI_FUNC void luaK_fixline (FuncState *fs, int line); +LUAI_FUNC void luaK_nil (FuncState *fs, int from, int n); +LUAI_FUNC void luaK_reserveregs (FuncState *fs, int n); +LUAI_FUNC void luaK_checkstack (FuncState *fs, int n); +LUAI_FUNC int luaK_stringK (FuncState *fs, TString *s); +LUAI_FUNC int luaK_numberK (FuncState *fs, lua_Number r); +LUAI_FUNC void luaK_dischargevars (FuncState *fs, expdesc *e); +LUAI_FUNC int luaK_exp2anyreg (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_exp2nextreg (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_exp2val (FuncState *fs, expdesc *e); +LUAI_FUNC int luaK_exp2RK (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_self (FuncState *fs, expdesc *e, expdesc *key); +LUAI_FUNC void luaK_indexed (FuncState *fs, expdesc *t, expdesc *k); +LUAI_FUNC void luaK_goiftrue (FuncState *fs, expdesc *e); +LUAI_FUNC void luaK_storevar (FuncState *fs, expdesc *var, expdesc *e); +LUAI_FUNC void luaK_setreturns (FuncState *fs, expdesc *e, int nresults); +LUAI_FUNC void luaK_setoneret (FuncState *fs, expdesc *e); +LUAI_FUNC int luaK_jump (FuncState *fs); +LUAI_FUNC void luaK_ret (FuncState *fs, int first, int nret); +LUAI_FUNC void luaK_patchlist (FuncState *fs, int list, int target); +LUAI_FUNC void luaK_patchtohere (FuncState *fs, int list); +LUAI_FUNC void luaK_concat (FuncState *fs, int *l1, int l2); +LUAI_FUNC int luaK_getlabel (FuncState *fs); +LUAI_FUNC void luaK_prefix (FuncState *fs, UnOpr op, expdesc *v); +LUAI_FUNC void luaK_infix (FuncState *fs, BinOpr op, expdesc *v); +LUAI_FUNC void luaK_posfix (FuncState *fs, BinOpr op, expdesc *v1, expdesc *v2); +LUAI_FUNC void luaK_setlist (FuncState *fs, int base, int nelems, int tostore); + + +#endif diff --git a/src/lua/ldblib.c b/src/lua/ldblib.c new file mode 100644 index 0000000..2027eda --- /dev/null +++ b/src/lua/ldblib.c @@ -0,0 +1,398 @@ +/* +** $Id: ldblib.c,v 1.104.1.4 2009/08/04 18:50:18 roberto Exp $ +** Interface from Lua to its debug API +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define ldblib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + + +static int db_getregistry (lua_State *L) { + lua_pushvalue(L, LUA_REGISTRYINDEX); + return 1; +} + + +static int db_getmetatable (lua_State *L) { + luaL_checkany(L, 1); + if (!lua_getmetatable(L, 1)) { + lua_pushnil(L); /* no metatable */ + } + return 1; +} + + +static int db_setmetatable (lua_State *L) { + int t = lua_type(L, 2); + luaL_argcheck(L, t == LUA_TNIL || t == LUA_TTABLE, 2, + "nil or table expected"); + lua_settop(L, 2); + lua_pushboolean(L, lua_setmetatable(L, 1)); + return 1; +} + + +static int db_getfenv (lua_State *L) { + luaL_checkany(L, 1); + lua_getfenv(L, 1); + return 1; +} + + +static int db_setfenv (lua_State *L) { + luaL_checktype(L, 2, LUA_TTABLE); + lua_settop(L, 2); + if (lua_setfenv(L, 1) == 0) + luaL_error(L, LUA_QL("setfenv") + " cannot change environment of given object"); + return 1; +} + + +static void settabss (lua_State *L, const char *i, const char *v) { + lua_pushstring(L, v); + lua_setfield(L, -2, i); +} + + +static void settabsi (lua_State *L, const char *i, int v) { + lua_pushinteger(L, v); + lua_setfield(L, -2, i); +} + + +static lua_State *getthread (lua_State *L, int *arg) { + if (lua_isthread(L, 1)) { + *arg = 1; + return lua_tothread(L, 1); + } + else { + *arg = 0; + return L; + } +} + + +static void treatstackoption (lua_State *L, lua_State *L1, const char *fname) { + if (L == L1) { + lua_pushvalue(L, -2); + lua_remove(L, -3); + } + else + lua_xmove(L1, L, 1); + lua_setfield(L, -2, fname); +} + + +static int db_getinfo (lua_State *L) { + lua_Debug ar; + int arg; + lua_State *L1 = getthread(L, &arg); + const char *options = luaL_optstring(L, arg+2, "flnSu"); + if (lua_isnumber(L, arg+1)) { + if (!lua_getstack(L1, (int)lua_tointeger(L, arg+1), &ar)) { + lua_pushnil(L); /* level out of range */ + return 1; + } + } + else if (lua_isfunction(L, arg+1)) { + lua_pushfstring(L, ">%s", options); + options = lua_tostring(L, -1); + lua_pushvalue(L, arg+1); + lua_xmove(L, L1, 1); + } + else + return luaL_argerror(L, arg+1, "function or level expected"); + if (!lua_getinfo(L1, options, &ar)) + return luaL_argerror(L, arg+2, "invalid option"); + lua_createtable(L, 0, 2); + if (strchr(options, 'S')) { + settabss(L, "source", ar.source); + settabss(L, "short_src", ar.short_src); + settabsi(L, "linedefined", ar.linedefined); + settabsi(L, "lastlinedefined", ar.lastlinedefined); + settabss(L, "what", ar.what); + } + if (strchr(options, 'l')) + settabsi(L, "currentline", ar.currentline); + if (strchr(options, 'u')) + settabsi(L, "nups", ar.nups); + if (strchr(options, 'n')) { + settabss(L, "name", ar.name); + settabss(L, "namewhat", ar.namewhat); + } + if (strchr(options, 'L')) + treatstackoption(L, L1, "activelines"); + if (strchr(options, 'f')) + treatstackoption(L, L1, "func"); + return 1; /* return table */ +} + + +static int db_getlocal (lua_State *L) { + int arg; + lua_State *L1 = getthread(L, &arg); + lua_Debug ar; + const char *name; + if (!lua_getstack(L1, luaL_checkint(L, arg+1), &ar)) /* out of range? */ + return luaL_argerror(L, arg+1, "level out of range"); + name = lua_getlocal(L1, &ar, luaL_checkint(L, arg+2)); + if (name) { + lua_xmove(L1, L, 1); + lua_pushstring(L, name); + lua_pushvalue(L, -2); + return 2; + } + else { + lua_pushnil(L); + return 1; + } +} + + +static int db_setlocal (lua_State *L) { + int arg; + lua_State *L1 = getthread(L, &arg); + lua_Debug ar; + if (!lua_getstack(L1, luaL_checkint(L, arg+1), &ar)) /* out of range? */ + return luaL_argerror(L, arg+1, "level out of range"); + luaL_checkany(L, arg+3); + lua_settop(L, arg+3); + lua_xmove(L, L1, 1); + lua_pushstring(L, lua_setlocal(L1, &ar, luaL_checkint(L, arg+2))); + return 1; +} + + +static int auxupvalue (lua_State *L, int get) { + const char *name; + int n = luaL_checkint(L, 2); + luaL_checktype(L, 1, LUA_TFUNCTION); + if (lua_iscfunction(L, 1)) return 0; /* cannot touch C upvalues from Lua */ + name = get ? lua_getupvalue(L, 1, n) : lua_setupvalue(L, 1, n); + if (name == NULL) return 0; + lua_pushstring(L, name); + lua_insert(L, -(get+1)); + return get + 1; +} + + +static int db_getupvalue (lua_State *L) { + return auxupvalue(L, 1); +} + + +static int db_setupvalue (lua_State *L) { + luaL_checkany(L, 3); + return auxupvalue(L, 0); +} + + + +static const char KEY_HOOK = 'h'; + + +static void hookf (lua_State *L, lua_Debug *ar) { + static const char *const hooknames[] = + {"call", "return", "line", "count", "tail return"}; + lua_pushlightuserdata(L, (void *)&KEY_HOOK); + lua_rawget(L, LUA_REGISTRYINDEX); + lua_pushlightuserdata(L, L); + lua_rawget(L, -2); + if (lua_isfunction(L, -1)) { + lua_pushstring(L, hooknames[(int)ar->event]); + if (ar->currentline >= 0) + lua_pushinteger(L, ar->currentline); + else lua_pushnil(L); + lua_assert(lua_getinfo(L, "lS", ar)); + lua_call(L, 2, 0); + } +} + + +static int makemask (const char *smask, int count) { + int mask = 0; + if (strchr(smask, 'c')) mask |= LUA_MASKCALL; + if (strchr(smask, 'r')) mask |= LUA_MASKRET; + if (strchr(smask, 'l')) mask |= LUA_MASKLINE; + if (count > 0) mask |= LUA_MASKCOUNT; + return mask; +} + + +static char *unmakemask (int mask, char *smask) { + int i = 0; + if (mask & LUA_MASKCALL) smask[i++] = 'c'; + if (mask & LUA_MASKRET) smask[i++] = 'r'; + if (mask & LUA_MASKLINE) smask[i++] = 'l'; + smask[i] = '\0'; + return smask; +} + + +static void gethooktable (lua_State *L) { + lua_pushlightuserdata(L, (void *)&KEY_HOOK); + lua_rawget(L, LUA_REGISTRYINDEX); + if (!lua_istable(L, -1)) { + lua_pop(L, 1); + lua_createtable(L, 0, 1); + lua_pushlightuserdata(L, (void *)&KEY_HOOK); + lua_pushvalue(L, -2); + lua_rawset(L, LUA_REGISTRYINDEX); + } +} + + +static int db_sethook (lua_State *L) { + int arg, mask, count; + lua_Hook func; + lua_State *L1 = getthread(L, &arg); + if (lua_isnoneornil(L, arg+1)) { + lua_settop(L, arg+1); + func = NULL; mask = 0; count = 0; /* turn off hooks */ + } + else { + const char *smask = luaL_checkstring(L, arg+2); + luaL_checktype(L, arg+1, LUA_TFUNCTION); + count = luaL_optint(L, arg+3, 0); + func = hookf; mask = makemask(smask, count); + } + gethooktable(L); + lua_pushlightuserdata(L, L1); + lua_pushvalue(L, arg+1); + lua_rawset(L, -3); /* set new hook */ + lua_pop(L, 1); /* remove hook table */ + lua_sethook(L1, func, mask, count); /* set hooks */ + return 0; +} + + +static int db_gethook (lua_State *L) { + int arg; + lua_State *L1 = getthread(L, &arg); + char buff[5]; + int mask = lua_gethookmask(L1); + lua_Hook hook = lua_gethook(L1); + if (hook != NULL && hook != hookf) /* external hook? */ + lua_pushliteral(L, "external hook"); + else { + gethooktable(L); + lua_pushlightuserdata(L, L1); + lua_rawget(L, -2); /* get hook */ + lua_remove(L, -2); /* remove hook table */ + } + lua_pushstring(L, unmakemask(mask, buff)); + lua_pushinteger(L, lua_gethookcount(L1)); + return 3; +} + + +static int db_debug (lua_State *L) { + for (;;) { + char buffer[250]; + fputs("lua_debug> ", stderr); + if (fgets(buffer, sizeof(buffer), stdin) == 0 || + strcmp(buffer, "cont\n") == 0) + return 0; + if (luaL_loadbuffer(L, buffer, strlen(buffer), "=(debug command)") || + lua_pcall(L, 0, 0, 0)) { + fputs(lua_tostring(L, -1), stderr); + fputs("\n", stderr); + } + lua_settop(L, 0); /* remove eventual returns */ + } +} + + +#define LEVELS1 12 /* size of the first part of the stack */ +#define LEVELS2 10 /* size of the second part of the stack */ + +static int db_errorfb (lua_State *L) { + int level; + int firstpart = 1; /* still before eventual `...' */ + int arg; + lua_State *L1 = getthread(L, &arg); + lua_Debug ar; + if (lua_isnumber(L, arg+2)) { + level = (int)lua_tointeger(L, arg+2); + lua_pop(L, 1); + } + else + level = (L == L1) ? 1 : 0; /* level 0 may be this own function */ + if (lua_gettop(L) == arg) + lua_pushliteral(L, ""); + else if (!lua_isstring(L, arg+1)) return 1; /* message is not a string */ + else lua_pushliteral(L, "\n"); + lua_pushliteral(L, "stack traceback:"); + while (lua_getstack(L1, level++, &ar)) { + if (level > LEVELS1 && firstpart) { + /* no more than `LEVELS2' more levels? */ + if (!lua_getstack(L1, level+LEVELS2, &ar)) + level--; /* keep going */ + else { + lua_pushliteral(L, "\n\t..."); /* too many levels */ + while (lua_getstack(L1, level+LEVELS2, &ar)) /* find last levels */ + level++; + } + firstpart = 0; + continue; + } + lua_pushliteral(L, "\n\t"); + lua_getinfo(L1, "Snl", &ar); + lua_pushfstring(L, "%s:", ar.short_src); + if (ar.currentline > 0) + lua_pushfstring(L, "%d:", ar.currentline); + if (*ar.namewhat != '\0') /* is there a name? */ + lua_pushfstring(L, " in function " LUA_QS, ar.name); + else { + if (*ar.what == 'm') /* main? */ + lua_pushfstring(L, " in main chunk"); + else if (*ar.what == 'C' || *ar.what == 't') + lua_pushliteral(L, " ?"); /* C function or tail call */ + else + lua_pushfstring(L, " in function <%s:%d>", + ar.short_src, ar.linedefined); + } + lua_concat(L, lua_gettop(L) - arg); + } + lua_concat(L, lua_gettop(L) - arg); + return 1; +} + + +static const luaL_Reg dblib[] = { + {"debug", db_debug}, + {"getfenv", db_getfenv}, + {"gethook", db_gethook}, + {"getinfo", db_getinfo}, + {"getlocal", db_getlocal}, + {"getregistry", db_getregistry}, + {"getmetatable", db_getmetatable}, + {"getupvalue", db_getupvalue}, + {"setfenv", db_setfenv}, + {"sethook", db_sethook}, + {"setlocal", db_setlocal}, + {"setmetatable", db_setmetatable}, + {"setupvalue", db_setupvalue}, + {"traceback", db_errorfb}, + {NULL, NULL} +}; + + +LUALIB_API int luaopen_debug (lua_State *L) { + luaL_register(L, LUA_DBLIBNAME, dblib); + return 1; +} + diff --git a/src/lua/ldebug.c b/src/lua/ldebug.c new file mode 100644 index 0000000..d9e8fcf --- /dev/null +++ b/src/lua/ldebug.c @@ -0,0 +1,642 @@ +/* +** $Id: ldebug.c,v 2.29.1.6 2008/05/08 16:56:26 roberto Exp $ +** Debug Interface +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + + +#define ldebug_c +#define LUA_CORE + +#include "lua.h" + +#include "lapi.h" +#include "lcode.h" +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lvm.h" + + + +static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name); + + +static int currentpc (lua_State *L, CallInfo *ci) { + if (!isLua(ci)) return -1; /* function is not a Lua function? */ + if (ci == L->ci) + ci->savedpc = L->savedpc; + return pcRel(ci->savedpc, ci_func(ci)->l.p); +} + + +static int currentline (lua_State *L, CallInfo *ci) { + int pc = currentpc(L, ci); + if (pc < 0) + return -1; /* only active lua functions have current-line information */ + else + return getline(ci_func(ci)->l.p, pc); +} + + +/* +** this function can be called asynchronous (e.g. during a signal) +*/ +LUA_API int lua_sethook (lua_State *L, lua_Hook func, int mask, int count) { + if (func == NULL || mask == 0) { /* turn off hooks? */ + mask = 0; + func = NULL; + } + L->hook = func; + L->basehookcount = count; + resethookcount(L); + L->hookmask = cast_byte(mask); + return 1; +} + + +LUA_API lua_Hook lua_gethook (lua_State *L) { + return L->hook; +} + + +LUA_API int lua_gethookmask (lua_State *L) { + return L->hookmask; +} + + +LUA_API int lua_gethookcount (lua_State *L) { + return L->basehookcount; +} + +LUA_API int lua_gethookcountremaining (lua_State *L) { + return L->hookcount; +} + + +LUA_API int lua_getstack (lua_State *L, int level, lua_Debug *ar) { + int status; + CallInfo *ci; + lua_lock(L); + for (ci = L->ci; level > 0 && ci > L->base_ci; ci--) { + level--; + if (f_isLua(ci)) /* Lua function? */ + level -= ci->tailcalls; /* skip lost tail calls */ + } + if (level == 0 && ci > L->base_ci) { /* level found? */ + status = 1; + ar->i_ci = cast_int(ci - L->base_ci); + } + else if (level < 0) { /* level is of a lost tail call? */ + status = 1; + ar->i_ci = 0; + } + else status = 0; /* no such level */ + lua_unlock(L); + return status; +} + + +static Proto *getluaproto (CallInfo *ci) { + return (isLua(ci) ? ci_func(ci)->l.p : NULL); +} + + +static const char *findlocal (lua_State *L, CallInfo *ci, int n) { + const char *name; + Proto *fp = getluaproto(ci); + if (fp && (name = luaF_getlocalname(fp, n, currentpc(L, ci))) != NULL) + return name; /* is a local variable in a Lua function */ + else { + StkId limit = (ci == L->ci) ? L->top : (ci+1)->func; + if (limit - ci->base >= n && n > 0) /* is 'n' inside 'ci' stack? */ + return "(*temporary)"; + else + return NULL; + } +} + + +LUA_API const char *lua_getlocal (lua_State *L, const lua_Debug *ar, int n) { + CallInfo *ci = L->base_ci + ar->i_ci; + const char *name = findlocal(L, ci, n); + lua_lock(L); + if (name) + luaA_pushobject(L, ci->base + (n - 1)); + lua_unlock(L); + return name; +} + + +LUA_API const char *lua_setlocal (lua_State *L, const lua_Debug *ar, int n) { + CallInfo *ci = L->base_ci + ar->i_ci; + const char *name = findlocal(L, ci, n); + lua_lock(L); + if (name) + setobjs2s(L, ci->base + (n - 1), L->top - 1); + L->top--; /* pop value */ + lua_unlock(L); + return name; +} + + +static void funcinfo (lua_Debug *ar, Closure *cl) { + if (cl->c.isC) { + ar->source = "=[C]"; + ar->linedefined = -1; + ar->lastlinedefined = -1; + ar->what = "C"; + } + else { + ar->source = getstr(cl->l.p->source); + ar->linedefined = cl->l.p->linedefined; + ar->lastlinedefined = cl->l.p->lastlinedefined; + ar->what = (ar->linedefined == 0) ? "main" : "Lua"; + } + luaO_chunkid(ar->short_src, ar->source, LUA_IDSIZE); +} + + +static void info_tailcall (lua_Debug *ar) { + ar->name = ar->namewhat = ""; + ar->what = "tail"; + ar->lastlinedefined = ar->linedefined = ar->currentline = -1; + ar->source = "=(tail call)"; + luaO_chunkid(ar->short_src, ar->source, LUA_IDSIZE); + ar->nups = 0; +} + + +static void collectvalidlines (lua_State *L, Closure *f) { + if (f == NULL || f->c.isC) { + setnilvalue(L->top); + } + else { + Table *t = luaH_new(L, 0, 0); + int *lineinfo = f->l.p->lineinfo; + int i; + for (i=0; il.p->sizelineinfo; i++) + setbvalue(luaH_setnum(L, t, lineinfo[i]), 1); + sethvalue(L, L->top, t); + } + incr_top(L); +} + + +static int auxgetinfo (lua_State *L, const char *what, lua_Debug *ar, + Closure *f, CallInfo *ci) { + int status = 1; + if (f == NULL) { + info_tailcall(ar); + return status; + } + for (; *what; what++) { + switch (*what) { + case 'S': { + funcinfo(ar, f); + break; + } + case 'l': { + ar->currentline = (ci) ? currentline(L, ci) : -1; + break; + } + case 'u': { + ar->nups = f->c.nupvalues; + break; + } + case 'n': { + ar->namewhat = (ci) ? getfuncname(L, ci, &ar->name) : NULL; + if (ar->namewhat == NULL) { + ar->namewhat = ""; /* not found */ + ar->name = NULL; + } + break; + } + case 'L': + case 'f': /* handled by lua_getinfo */ + break; + default: status = 0; /* invalid option */ + } + } + return status; +} + + +LUA_API int lua_getinfo (lua_State *L, const char *what, lua_Debug *ar) { + int status; + Closure *f = NULL; + CallInfo *ci = NULL; + lua_lock(L); + if (*what == '>') { + StkId func = L->top - 1; + luai_apicheck(L, ttisfunction(func)); + what++; /* skip the '>' */ + f = clvalue(func); + L->top--; /* pop function */ + } + else if (ar->i_ci != 0) { /* no tail call? */ + ci = L->base_ci + ar->i_ci; + lua_assert(ttisfunction(ci->func)); + f = clvalue(ci->func); + } + status = auxgetinfo(L, what, ar, f, ci); + if (strchr(what, 'f')) { + if (f == NULL) setnilvalue(L->top); + else setclvalue(L, L->top, f); + incr_top(L); + } + if (strchr(what, 'L')) + collectvalidlines(L, f); + lua_unlock(L); + return status; +} + + +/* +** {====================================================== +** Symbolic Execution and code checker +** ======================================================= +*/ + +#define check(x) if (!(x)) return 0; + +#define checkjump(pt,pc) check(0 <= pc && pc < pt->sizecode) + +#define checkreg(pt,reg) check((reg) < (pt)->maxstacksize) + + + +static int precheck (const Proto *pt) { + check(pt->maxstacksize <= MAXSTACK); + check(pt->numparams+(pt->is_vararg & VARARG_HASARG) <= pt->maxstacksize); + check(!(pt->is_vararg & VARARG_NEEDSARG) || + (pt->is_vararg & VARARG_HASARG)); + check(pt->sizeupvalues <= pt->nups); + check(pt->sizelineinfo == pt->sizecode || pt->sizelineinfo == 0); + check(pt->sizecode > 0 && GET_OPCODE(pt->code[pt->sizecode-1]) == OP_RETURN); + return 1; +} + + +#define checkopenop(pt,pc) luaG_checkopenop((pt)->code[(pc)+1]) + +int luaG_checkopenop (Instruction i) { + switch (GET_OPCODE(i)) { + case OP_CALL: + case OP_TAILCALL: + case OP_RETURN: + case OP_SETLIST: { + check(GETARG_B(i) == 0); + return 1; + } + default: return 0; /* invalid instruction after an open call */ + } +} + + +static int checkArgMode (const Proto *pt, int r, enum OpArgMask mode) { + switch (mode) { + case OpArgN: check(r == 0); break; + case OpArgU: break; + case OpArgR: checkreg(pt, r); break; + case OpArgK: + check(ISK(r) ? INDEXK(r) < pt->sizek : r < pt->maxstacksize); + break; + } + return 1; +} + + +static Instruction symbexec (const Proto *pt, int lastpc, int reg) { + int pc; + int last; /* stores position of last instruction that changed `reg' */ + last = pt->sizecode-1; /* points to final return (a `neutral' instruction) */ + check(precheck(pt)); + for (pc = 0; pc < lastpc; pc++) { + Instruction i = pt->code[pc]; + OpCode op = GET_OPCODE(i); + int a = GETARG_A(i); + int b = 0; + int c = 0; + check(op < NUM_OPCODES); + checkreg(pt, a); + switch (getOpMode(op)) { + case iABC: { + b = GETARG_B(i); + c = GETARG_C(i); + check(checkArgMode(pt, b, getBMode(op))); + check(checkArgMode(pt, c, getCMode(op))); + break; + } + case iABx: { + b = GETARG_Bx(i); + if (getBMode(op) == OpArgK) check(b < pt->sizek); + break; + } + case iAsBx: { + b = GETARG_sBx(i); + if (getBMode(op) == OpArgR) { + int dest = pc+1+b; + check(0 <= dest && dest < pt->sizecode); + if (dest > 0) { + int j; + /* check that it does not jump to a setlist count; this + is tricky, because the count from a previous setlist may + have the same value of an invalid setlist; so, we must + go all the way back to the first of them (if any) */ + for (j = 0; j < dest; j++) { + Instruction d = pt->code[dest-1-j]; + if (!(GET_OPCODE(d) == OP_SETLIST && GETARG_C(d) == 0)) break; + } + /* if 'j' is even, previous value is not a setlist (even if + it looks like one) */ + check((j&1) == 0); + } + } + break; + } + } + if (testAMode(op)) { + if (a == reg) last = pc; /* change register `a' */ + } + if (testTMode(op)) { + check(pc+2 < pt->sizecode); /* check skip */ + check(GET_OPCODE(pt->code[pc+1]) == OP_JMP); + } + switch (op) { + case OP_LOADBOOL: { + if (c == 1) { /* does it jump? */ + check(pc+2 < pt->sizecode); /* check its jump */ + check(GET_OPCODE(pt->code[pc+1]) != OP_SETLIST || + GETARG_C(pt->code[pc+1]) != 0); + } + break; + } + case OP_LOADNIL: { + if (a <= reg && reg <= b) + last = pc; /* set registers from `a' to `b' */ + break; + } + case OP_GETUPVAL: + case OP_SETUPVAL: { + check(b < pt->nups); + break; + } + case OP_GETGLOBAL: + case OP_SETGLOBAL: { + check(ttisstring(&pt->k[b])); + break; + } + case OP_SELF: { + checkreg(pt, a+1); + if (reg == a+1) last = pc; + break; + } + case OP_CONCAT: { + check(b < c); /* at least two operands */ + break; + } + case OP_TFORLOOP: { + check(c >= 1); /* at least one result (control variable) */ + checkreg(pt, a+2+c); /* space for results */ + if (reg >= a+2) last = pc; /* affect all regs above its base */ + break; + } + case OP_FORLOOP: + case OP_FORPREP: + checkreg(pt, a+3); + /* FALLTHRU */ + case OP_JMP: { + int dest = pc+1+b; + /* not full check and jump is forward and do not skip `lastpc'? */ + if (reg != NO_REG && pc < dest && dest <= lastpc) + pc += b; /* do the jump */ + break; + } + case OP_CALL: + case OP_TAILCALL: { + if (b != 0) { + checkreg(pt, a+b-1); + } + c--; /* c = num. returns */ + if (c == LUA_MULTRET) { + check(checkopenop(pt, pc)); + } + else if (c != 0) + checkreg(pt, a+c-1); + if (reg >= a) last = pc; /* affect all registers above base */ + break; + } + case OP_RETURN: { + b--; /* b = num. returns */ + if (b > 0) checkreg(pt, a+b-1); + break; + } + case OP_SETLIST: { + if (b > 0) checkreg(pt, a + b); + if (c == 0) { + pc++; + check(pc < pt->sizecode - 1); + } + break; + } + case OP_CLOSURE: { + int nup, j; + check(b < pt->sizep); + nup = pt->p[b]->nups; + check(pc + nup < pt->sizecode); + for (j = 1; j <= nup; j++) { + OpCode op1 = GET_OPCODE(pt->code[pc + j]); + check(op1 == OP_GETUPVAL || op1 == OP_MOVE); + } + if (reg != NO_REG) /* tracing? */ + pc += nup; /* do not 'execute' these pseudo-instructions */ + break; + } + case OP_VARARG: { + check((pt->is_vararg & VARARG_ISVARARG) && + !(pt->is_vararg & VARARG_NEEDSARG)); + b--; + if (b == LUA_MULTRET) check(checkopenop(pt, pc)); + checkreg(pt, a+b-1); + break; + } + default: break; + } + } + return pt->code[last]; +} + +#undef check +#undef checkjump +#undef checkreg + +/* }====================================================== */ + + +int luaG_checkcode (const Proto *pt) { + return (symbexec(pt, pt->sizecode, NO_REG) != 0); +} + + +static const char *kname (Proto *p, int c) { + if (ISK(c) && ttisstring(&p->k[INDEXK(c)])) + return svalue(&p->k[INDEXK(c)]); + else + return "?"; +} + + +static const char *getobjname (lua_State *L, CallInfo *ci, int stackpos, + const char **name) { + if (isLua(ci)) { /* a Lua function? */ + Proto *p = ci_func(ci)->l.p; + int pc = currentpc(L, ci); + Instruction i; + *name = luaF_getlocalname(p, stackpos+1, pc); + if (*name) /* is a local? */ + return "local"; + i = symbexec(p, pc, stackpos); /* try symbolic execution */ + lua_assert(pc != -1); + switch (GET_OPCODE(i)) { + case OP_GETGLOBAL: { + int g = GETARG_Bx(i); /* global index */ + lua_assert(ttisstring(&p->k[g])); + *name = svalue(&p->k[g]); + return "global"; + } + case OP_MOVE: { + int a = GETARG_A(i); + int b = GETARG_B(i); /* move from `b' to `a' */ + if (b < a) + return getobjname(L, ci, b, name); /* get name for `b' */ + break; + } + case OP_GETTABLE: { + int k = GETARG_C(i); /* key index */ + *name = kname(p, k); + return "field"; + } + case OP_GETUPVAL: { + int u = GETARG_B(i); /* upvalue index */ + *name = p->upvalues ? getstr(p->upvalues[u]) : "?"; + return "upvalue"; + } + case OP_SELF: { + int k = GETARG_C(i); /* key index */ + *name = kname(p, k); + return "method"; + } + default: break; + } + } + return NULL; /* no useful name found */ +} + + +static const char *getfuncname (lua_State *L, CallInfo *ci, const char **name) { + Instruction i; + if ((isLua(ci) && ci->tailcalls > 0) || !isLua(ci - 1)) + return NULL; /* calling function is not Lua (or is unknown) */ + ci--; /* calling function */ + i = ci_func(ci)->l.p->code[currentpc(L, ci)]; + if (GET_OPCODE(i) == OP_CALL || GET_OPCODE(i) == OP_TAILCALL || + GET_OPCODE(i) == OP_TFORLOOP) + return getobjname(L, ci, GETARG_A(i), name); + else + return NULL; /* no useful name can be found */ +} + + +/* only ANSI way to check whether a pointer points to an array */ +static int isinstack (CallInfo *ci, const TValue *o) { + StkId p; + for (p = ci->base; p < ci->top; p++) + if (o == p) return 1; + return 0; +} + + +void luaG_typeerror (lua_State *L, const TValue *o, const char *op) { + const char *name = NULL; + const char *t = luaT_typenames[ttype(o)]; + const char *kind = (isinstack(L->ci, o)) ? + getobjname(L, L->ci, cast_int(o - L->base), &name) : + NULL; + if (kind) + luaG_runerror(L, "attempt to %s %s " LUA_QS " (a %s value)", + op, kind, name, t); + else + luaG_runerror(L, "attempt to %s a %s value", op, t); +} + + +void luaG_concaterror (lua_State *L, StkId p1, StkId p2) { + if (ttisstring(p1) || ttisnumber(p1)) p1 = p2; + lua_assert(!ttisstring(p1) && !ttisnumber(p1)); + luaG_typeerror(L, p1, "concatenate"); +} + + +void luaG_aritherror (lua_State *L, const TValue *p1, const TValue *p2) { + TValue temp; + if (luaV_tonumber(p1, &temp) == NULL) + p2 = p1; /* first operand is wrong */ + luaG_typeerror(L, p2, "perform arithmetic on"); +} + + +int luaG_ordererror (lua_State *L, const TValue *p1, const TValue *p2) { + const char *t1 = luaT_typenames[ttype(p1)]; + const char *t2 = luaT_typenames[ttype(p2)]; + if (t1[2] == t2[2]) + luaG_runerror(L, "attempt to compare two %s values", t1); + else + luaG_runerror(L, "attempt to compare %s with %s", t1, t2); + return 0; +} + + +static void addinfo (lua_State *L, const char *msg) { + CallInfo *ci = L->ci; + if (isLua(ci)) { /* is Lua code? */ + char buff[LUA_IDSIZE]; /* add file:line information */ + int line = currentline(L, ci); + luaO_chunkid(buff, getstr(getluaproto(ci)->source), LUA_IDSIZE); + luaO_pushfstring(L, "%s:%d: %s", buff, line, msg); + } +} + + +void luaG_errormsg (lua_State *L) { + if (L->errfunc != 0) { /* is there an error handling function? */ + StkId errfunc = restorestack(L, L->errfunc); + if (!ttisfunction(errfunc)) luaD_throw(L, LUA_ERRERR); + setobjs2s(L, L->top, L->top - 1); /* move argument */ + setobjs2s(L, L->top - 1, errfunc); /* push function */ + incr_top(L); + luaD_call(L, L->top - 2, 1); /* call it */ + } + luaD_throw(L, LUA_ERRRUN); +} + + +void luaG_runerror (lua_State *L, const char *fmt, ...) { + va_list argp; + va_start(argp, fmt); + addinfo(L, luaO_pushvfstring(L, fmt, argp)); + va_end(argp); + luaG_errormsg(L); +} + diff --git a/src/lua/ldebug.h b/src/lua/ldebug.h new file mode 100644 index 0000000..ba28a97 --- /dev/null +++ b/src/lua/ldebug.h @@ -0,0 +1,33 @@ +/* +** $Id: ldebug.h,v 2.3.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions from Debug Interface module +** See Copyright Notice in lua.h +*/ + +#ifndef ldebug_h +#define ldebug_h + + +#include "lstate.h" + + +#define pcRel(pc, p) (cast(int, (pc) - (p)->code) - 1) + +#define getline(f,pc) (((f)->lineinfo) ? (f)->lineinfo[pc] : 0) + +#define resethookcount(L) (L->hookcount = L->basehookcount) + + +LUAI_FUNC void luaG_typeerror (lua_State *L, const TValue *o, + const char *opname); +LUAI_FUNC void luaG_concaterror (lua_State *L, StkId p1, StkId p2); +LUAI_FUNC void luaG_aritherror (lua_State *L, const TValue *p1, + const TValue *p2); +LUAI_FUNC int luaG_ordererror (lua_State *L, const TValue *p1, + const TValue *p2); +LUAI_FUNC void luaG_runerror (lua_State *L, const char *fmt, ...); +LUAI_FUNC void luaG_errormsg (lua_State *L); +LUAI_FUNC int luaG_checkcode (const Proto *pt); +LUAI_FUNC int luaG_checkopenop (Instruction i); + +#endif diff --git a/src/lua/ldo.c b/src/lua/ldo.c new file mode 100644 index 0000000..30333bf --- /dev/null +++ b/src/lua/ldo.c @@ -0,0 +1,519 @@ +/* +** $Id: ldo.c,v 2.38.1.4 2012/01/18 02:27:10 roberto Exp $ +** Stack and Call structure of Lua +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define ldo_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lundump.h" +#include "lvm.h" +#include "lzio.h" + + + + +/* +** {====================================================== +** Error-recovery functions +** ======================================================= +*/ + + +/* chain list of long jump buffers */ +struct lua_longjmp { + struct lua_longjmp *previous; + luai_jmpbuf b; + volatile int status; /* error code */ +}; + + +void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop) { + switch (errcode) { + case LUA_ERRMEM: { + setsvalue2s(L, oldtop, luaS_newliteral(L, MEMERRMSG)); + break; + } + case LUA_ERRERR: { + setsvalue2s(L, oldtop, luaS_newliteral(L, "error in error handling")); + break; + } + case LUA_ERRSYNTAX: + case LUA_ERRRUN: { + setobjs2s(L, oldtop, L->top - 1); /* error message on current top */ + break; + } + } + L->top = oldtop + 1; +} + + +static void restore_stack_limit (lua_State *L) { + lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK - 1); + if (L->size_ci > LUAI_MAXCALLS) { /* there was an overflow? */ + int inuse = cast_int(L->ci - L->base_ci); + if (inuse + 1 < LUAI_MAXCALLS) /* can `undo' overflow? */ + luaD_reallocCI(L, LUAI_MAXCALLS); + } +} + + +static void resetstack (lua_State *L, int status) { + L->ci = L->base_ci; + L->base = L->ci->base; + luaF_close(L, L->base); /* close eventual pending closures */ + luaD_seterrorobj(L, status, L->base); + L->nCcalls = L->baseCcalls; + L->allowhook = 1; + restore_stack_limit(L); + L->errfunc = 0; + L->errorJmp = NULL; +} + + +void luaD_throw (lua_State *L, int errcode) { + if (L->errorJmp) { + L->errorJmp->status = errcode; + LUAI_THROW(L, L->errorJmp); + } + else { + L->status = cast_byte(errcode); + if (G(L)->panic) { + resetstack(L, errcode); + lua_unlock(L); + G(L)->panic(L); + } + exit(EXIT_FAILURE); + } +} + + +int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud) { + struct lua_longjmp lj; + lj.status = 0; + lj.previous = L->errorJmp; /* chain new error handler */ + L->errorJmp = &lj; + LUAI_TRY(L, &lj, + (*f)(L, ud); + ); + L->errorJmp = lj.previous; /* restore old error handler */ + return lj.status; +} + +/* }====================================================== */ + + +static void correctstack (lua_State *L, TValue *oldstack) { + CallInfo *ci; + GCObject *up; + L->top = (L->top - oldstack) + L->stack; + for (up = L->openupval; up != NULL; up = up->gch.next) + gco2uv(up)->v = (gco2uv(up)->v - oldstack) + L->stack; + for (ci = L->base_ci; ci <= L->ci; ci++) { + ci->top = (ci->top - oldstack) + L->stack; + ci->base = (ci->base - oldstack) + L->stack; + ci->func = (ci->func - oldstack) + L->stack; + } + L->base = (L->base - oldstack) + L->stack; +} + + +void luaD_reallocstack (lua_State *L, int newsize) { + TValue *oldstack = L->stack; + int realsize = newsize + 1 + EXTRA_STACK; + lua_assert(L->stack_last - L->stack == L->stacksize - EXTRA_STACK - 1); + luaM_reallocvector(L, L->stack, L->stacksize, realsize, TValue); + L->stacksize = realsize; + L->stack_last = L->stack+newsize; + correctstack(L, oldstack); +} + + +void luaD_reallocCI (lua_State *L, int newsize) { + CallInfo *oldci = L->base_ci; + luaM_reallocvector(L, L->base_ci, L->size_ci, newsize, CallInfo); + L->size_ci = newsize; + L->ci = (L->ci - oldci) + L->base_ci; + L->end_ci = L->base_ci + L->size_ci - 1; +} + + +void luaD_growstack (lua_State *L, int n) { + if (n <= L->stacksize) /* double size is enough? */ + luaD_reallocstack(L, 2*L->stacksize); + else + luaD_reallocstack(L, L->stacksize + n); +} + + +static CallInfo *growCI (lua_State *L) { + if (L->size_ci > LUAI_MAXCALLS) /* overflow while handling overflow? */ + luaD_throw(L, LUA_ERRERR); + else { + luaD_reallocCI(L, 2*L->size_ci); + if (L->size_ci > LUAI_MAXCALLS) + luaG_runerror(L, "stack overflow"); + } + return ++L->ci; +} + + +void luaD_callhook (lua_State *L, int event, int line) { + lua_Hook hook = L->hook; + if (hook && L->allowhook) { + ptrdiff_t top = savestack(L, L->top); + ptrdiff_t ci_top = savestack(L, L->ci->top); + lua_Debug ar; + ar.event = event; + ar.currentline = line; + if (event == LUA_HOOKTAILRET) + ar.i_ci = 0; /* tail call; no debug information about it */ + else + ar.i_ci = cast_int(L->ci - L->base_ci); + luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */ + L->ci->top = L->top + LUA_MINSTACK; + lua_assert(L->ci->top <= L->stack_last); + L->allowhook = 0; /* cannot call hooks inside a hook */ + lua_unlock(L); + (*hook)(L, &ar); + lua_lock(L); + lua_assert(!L->allowhook); + L->allowhook = 1; + L->ci->top = restorestack(L, ci_top); + L->top = restorestack(L, top); + } +} + + +static StkId adjust_varargs (lua_State *L, Proto *p, int actual) { + int i; + int nfixargs = p->numparams; + Table *htab = NULL; + StkId base, fixed; + for (; actual < nfixargs; ++actual) + setnilvalue(L->top++); +#if defined(LUA_COMPAT_VARARG) + if (p->is_vararg & VARARG_NEEDSARG) { /* compat. with old-style vararg? */ + int nvar = actual - nfixargs; /* number of extra arguments */ + lua_assert(p->is_vararg & VARARG_HASARG); + luaC_checkGC(L); + luaD_checkstack(L, p->maxstacksize); + htab = luaH_new(L, nvar, 1); /* create `arg' table */ + for (i=0; itop - nvar + i); + /* store counter in field `n' */ + setnvalue(luaH_setstr(L, htab, luaS_newliteral(L, "n")), cast_num(nvar)); + } +#endif + /* move fixed parameters to final position */ + fixed = L->top - actual; /* first fixed argument */ + base = L->top; /* final position of first argument */ + for (i=0; itop++, fixed+i); + setnilvalue(fixed+i); + } + /* add `arg' parameter */ + if (htab) { + sethvalue(L, L->top++, htab); + lua_assert(iswhite(obj2gco(htab))); + } + return base; +} + + +static StkId tryfuncTM (lua_State *L, StkId func) { + const TValue *tm = luaT_gettmbyobj(L, func, TM_CALL); + StkId p; + ptrdiff_t funcr = savestack(L, func); + if (!ttisfunction(tm)) + luaG_typeerror(L, func, "call"); + /* Open a hole inside the stack at `func' */ + for (p = L->top; p > func; p--) setobjs2s(L, p, p-1); + incr_top(L); + func = restorestack(L, funcr); /* previous call may change stack */ + setobj2s(L, func, tm); /* tag method is the new function to be called */ + return func; +} + + + +#define inc_ci(L) \ + ((L->ci == L->end_ci) ? growCI(L) : \ + (condhardstacktests(luaD_reallocCI(L, L->size_ci)), ++L->ci)) + + +int luaD_precall (lua_State *L, StkId func, int nresults) { + LClosure *cl; + ptrdiff_t funcr; + if (!ttisfunction(func)) /* `func' is not a function? */ + func = tryfuncTM(L, func); /* check the `function' tag method */ + funcr = savestack(L, func); + cl = &clvalue(func)->l; + L->ci->savedpc = L->savedpc; + if (!cl->isC) { /* Lua function? prepare its call */ + CallInfo *ci; + StkId st, base; + Proto *p = cl->p; + luaD_checkstack(L, p->maxstacksize + p->numparams); + func = restorestack(L, funcr); + if (!p->is_vararg) { /* no varargs? */ + base = func + 1; + if (L->top > base + p->numparams) + L->top = base + p->numparams; + } + else { /* vararg function */ + int nargs = cast_int(L->top - func) - 1; + base = adjust_varargs(L, p, nargs); + func = restorestack(L, funcr); /* previous call may change the stack */ + } + ci = inc_ci(L); /* now `enter' new function */ + ci->func = func; + L->base = ci->base = base; + ci->top = L->base + p->maxstacksize; + lua_assert(ci->top <= L->stack_last); + L->savedpc = p->code; /* starting point */ + ci->tailcalls = 0; + ci->nresults = nresults; + for (st = L->top; st < ci->top; st++) + setnilvalue(st); + L->top = ci->top; + if (L->hookmask & LUA_MASKCALL) { + L->savedpc++; /* hooks assume 'pc' is already incremented */ + luaD_callhook(L, LUA_HOOKCALL, -1); + L->savedpc--; /* correct 'pc' */ + } + return PCRLUA; + } + else { /* if is a C function, call it */ + CallInfo *ci; + int n; + luaD_checkstack(L, LUA_MINSTACK); /* ensure minimum stack size */ + ci = inc_ci(L); /* now `enter' new function */ + ci->func = restorestack(L, funcr); + L->base = ci->base = ci->func + 1; + ci->top = L->top + LUA_MINSTACK; + lua_assert(ci->top <= L->stack_last); + ci->nresults = nresults; + if (L->hookmask & LUA_MASKCALL) + luaD_callhook(L, LUA_HOOKCALL, -1); + lua_unlock(L); + n = (*curr_func(L)->c.f)(L); /* do the actual call */ + lua_lock(L); + if (n < 0) /* yielding? */ + return PCRYIELD; + else { + luaD_poscall(L, L->top - n); + return PCRC; + } + } +} + + +static StkId callrethooks (lua_State *L, StkId firstResult) { + ptrdiff_t fr = savestack(L, firstResult); /* next call may change stack */ + luaD_callhook(L, LUA_HOOKRET, -1); + if (f_isLua(L->ci)) { /* Lua function? */ + while ((L->hookmask & LUA_MASKRET) && L->ci->tailcalls--) /* tail calls */ + luaD_callhook(L, LUA_HOOKTAILRET, -1); + } + return restorestack(L, fr); +} + + +int luaD_poscall (lua_State *L, StkId firstResult) { + StkId res; + int wanted, i; + CallInfo *ci; + if (L->hookmask & LUA_MASKRET) + firstResult = callrethooks(L, firstResult); + ci = L->ci--; + res = ci->func; /* res == final position of 1st result */ + wanted = ci->nresults; + L->base = (ci - 1)->base; /* restore base */ + L->savedpc = (ci - 1)->savedpc; /* restore savedpc */ + /* move results to correct place */ + for (i = wanted; i != 0 && firstResult < L->top; i--) + setobjs2s(L, res++, firstResult++); + while (i-- > 0) + setnilvalue(res++); + L->top = res; + return (wanted - LUA_MULTRET); /* 0 iff wanted == LUA_MULTRET */ +} + + +/* +** Call a function (C or Lua). The function to be called is at *func. +** The arguments are on the stack, right after the function. +** When returns, all the results are on the stack, starting at the original +** function position. +*/ +void luaD_call (lua_State *L, StkId func, int nResults) { + if (++L->nCcalls >= LUAI_MAXCCALLS) { + if (L->nCcalls == LUAI_MAXCCALLS) + luaG_runerror(L, "C stack overflow"); + else if (L->nCcalls >= (LUAI_MAXCCALLS + (LUAI_MAXCCALLS>>3))) + luaD_throw(L, LUA_ERRERR); /* error while handing stack error */ + } + if (luaD_precall(L, func, nResults) == PCRLUA) /* is a Lua function? */ + luaV_execute(L, 1); /* call it */ + L->nCcalls--; + luaC_checkGC(L); +} + + +static void resume (lua_State *L, void *ud) { + StkId firstArg = cast(StkId, ud); + CallInfo *ci = L->ci; + if (L->status == 0) { /* start coroutine? */ + lua_assert(ci == L->base_ci && firstArg > L->base); + if (luaD_precall(L, firstArg - 1, LUA_MULTRET) != PCRLUA) + return; + } + else { /* resuming from previous yield */ + lua_assert(L->status == LUA_YIELD); + L->status = 0; + if (!f_isLua(ci)) { /* `common' yield? */ + /* finish interrupted execution of `OP_CALL' */ + lua_assert(GET_OPCODE(*((ci-1)->savedpc - 1)) == OP_CALL || + GET_OPCODE(*((ci-1)->savedpc - 1)) == OP_TAILCALL); + if (luaD_poscall(L, firstArg)) /* complete it... */ + L->top = L->ci->top; /* and correct top if not multiple results */ + } + else /* yielded inside a hook: just continue its execution */ + L->base = L->ci->base; + } + luaV_execute(L, cast_int(L->ci - L->base_ci)); +} + + +static int resume_error (lua_State *L, const char *msg) { + L->top = L->ci->base; + setsvalue2s(L, L->top, luaS_new(L, msg)); + incr_top(L); + lua_unlock(L); + return LUA_ERRRUN; +} + + +LUA_API int lua_resume (lua_State *L, int nargs) { + int status; + lua_lock(L); + if (L->status != LUA_YIELD && (L->status != 0 || L->ci != L->base_ci)) + return resume_error(L, "cannot resume non-suspended coroutine"); + if (L->nCcalls >= LUAI_MAXCCALLS) + return resume_error(L, "C stack overflow"); + luai_userstateresume(L, nargs); + lua_assert(L->errfunc == 0); + L->baseCcalls = ++L->nCcalls; + status = luaD_rawrunprotected(L, resume, L->top - nargs); + if (status != 0) { /* error? */ + L->status = cast_byte(status); /* mark thread as `dead' */ + luaD_seterrorobj(L, status, L->top); + L->ci->top = L->top; + } + else { + lua_assert(L->nCcalls == L->baseCcalls); + status = L->status; + } + --L->nCcalls; + lua_unlock(L); + return status; +} + + +LUA_API int lua_yield (lua_State *L, int nresults) { + luai_userstateyield(L, nresults); + lua_lock(L); + if (L->nCcalls > L->baseCcalls) + luaG_runerror(L, "attempt to yield across metamethod/C-call boundary"); + L->base = L->top - nresults; /* protect stack slots below */ + L->status = LUA_YIELD; + lua_unlock(L); + return -1; +} + + +int luaD_pcall (lua_State *L, Pfunc func, void *u, + ptrdiff_t old_top, ptrdiff_t ef) { + int status; + unsigned short oldnCcalls = L->nCcalls; + ptrdiff_t old_ci = saveci(L, L->ci); + lu_byte old_allowhooks = L->allowhook; + ptrdiff_t old_errfunc = L->errfunc; + L->errfunc = ef; + status = luaD_rawrunprotected(L, func, u); + if (status != 0) { /* an error occurred? */ + StkId oldtop = restorestack(L, old_top); + luaF_close(L, oldtop); /* close eventual pending closures */ + luaD_seterrorobj(L, status, oldtop); + L->nCcalls = oldnCcalls; + L->ci = restoreci(L, old_ci); + L->base = L->ci->base; + L->savedpc = L->ci->savedpc; + L->allowhook = old_allowhooks; + restore_stack_limit(L); + } + L->errfunc = old_errfunc; + return status; +} + + + +/* +** Execute a protected parser. +*/ +struct SParser { /* data to `f_parser' */ + ZIO *z; + Mbuffer buff; /* buffer to be used by the scanner */ + const char *name; +}; + +static void f_parser (lua_State *L, void *ud) { + int i; + Proto *tf; + Closure *cl; + struct SParser *p = cast(struct SParser *, ud); + int c = luaZ_lookahead(p->z); + luaC_checkGC(L); + tf = ((c == LUA_SIGNATURE[0]) ? luaU_undump : luaY_parser)(L, p->z, + &p->buff, p->name); + cl = luaF_newLclosure(L, tf->nups, hvalue(gt(L))); + cl->l.p = tf; + for (i = 0; i < tf->nups; i++) /* initialize eventual upvalues */ + cl->l.upvals[i] = luaF_newupval(L); + setclvalue(L, L->top, cl); + incr_top(L); +} + + +int luaD_protectedparser (lua_State *L, ZIO *z, const char *name) { + struct SParser p; + int status; + p.z = z; p.name = name; + luaZ_initbuffer(L, &p.buff); + status = luaD_pcall(L, f_parser, &p, savestack(L, L->top), L->errfunc); + luaZ_freebuffer(L, &p.buff); + return status; +} + + diff --git a/src/lua/ldo.h b/src/lua/ldo.h new file mode 100644 index 0000000..98fddac --- /dev/null +++ b/src/lua/ldo.h @@ -0,0 +1,57 @@ +/* +** $Id: ldo.h,v 2.7.1.1 2007/12/27 13:02:25 roberto Exp $ +** Stack and Call structure of Lua +** See Copyright Notice in lua.h +*/ + +#ifndef ldo_h +#define ldo_h + + +#include "lobject.h" +#include "lstate.h" +#include "lzio.h" + + +#define luaD_checkstack(L,n) \ + if ((char *)L->stack_last - (char *)L->top <= (n)*(int)sizeof(TValue)) \ + luaD_growstack(L, n); \ + else condhardstacktests(luaD_reallocstack(L, L->stacksize - EXTRA_STACK - 1)); + + +#define incr_top(L) {luaD_checkstack(L,1); L->top++;} + +#define savestack(L,p) ((char *)(p) - (char *)L->stack) +#define restorestack(L,n) ((TValue *)((char *)L->stack + (n))) + +#define saveci(L,p) ((char *)(p) - (char *)L->base_ci) +#define restoreci(L,n) ((CallInfo *)((char *)L->base_ci + (n))) + + +/* results from luaD_precall */ +#define PCRLUA 0 /* initiated a call to a Lua function */ +#define PCRC 1 /* did a call to a C function */ +#define PCRYIELD 2 /* C funtion yielded */ + + +/* type of protected functions, to be ran by `runprotected' */ +typedef void (*Pfunc) (lua_State *L, void *ud); + +LUAI_FUNC int luaD_protectedparser (lua_State *L, ZIO *z, const char *name); +LUAI_FUNC void luaD_callhook (lua_State *L, int event, int line); +LUAI_FUNC int luaD_precall (lua_State *L, StkId func, int nresults); +LUAI_FUNC void luaD_call (lua_State *L, StkId func, int nResults); +LUAI_FUNC int luaD_pcall (lua_State *L, Pfunc func, void *u, + ptrdiff_t oldtop, ptrdiff_t ef); +LUAI_FUNC int luaD_poscall (lua_State *L, StkId firstResult); +LUAI_FUNC void luaD_reallocCI (lua_State *L, int newsize); +LUAI_FUNC void luaD_reallocstack (lua_State *L, int newsize); +LUAI_FUNC void luaD_growstack (lua_State *L, int n); + +LUAI_FUNC void luaD_throw (lua_State *L, int errcode); +LUAI_FUNC int luaD_rawrunprotected (lua_State *L, Pfunc f, void *ud); + +LUAI_FUNC void luaD_seterrorobj (lua_State *L, int errcode, StkId oldtop); + +#endif + diff --git a/src/lua/ldump.c b/src/lua/ldump.c new file mode 100644 index 0000000..c9d3d48 --- /dev/null +++ b/src/lua/ldump.c @@ -0,0 +1,164 @@ +/* +** $Id: ldump.c,v 2.8.1.1 2007/12/27 13:02:25 roberto Exp $ +** save precompiled Lua chunks +** See Copyright Notice in lua.h +*/ + +#include + +#define ldump_c +#define LUA_CORE + +#include "lua.h" + +#include "lobject.h" +#include "lstate.h" +#include "lundump.h" + +typedef struct { + lua_State* L; + lua_Writer writer; + void* data; + int strip; + int status; +} DumpState; + +#define DumpMem(b,n,size,D) DumpBlock(b,(n)*(size),D) +#define DumpVar(x,D) DumpMem(&x,1,sizeof(x),D) + +static void DumpBlock(const void* b, size_t size, DumpState* D) +{ + if (D->status==0) + { + lua_unlock(D->L); + D->status=(*D->writer)(D->L,b,size,D->data); + lua_lock(D->L); + } +} + +static void DumpChar(int y, DumpState* D) +{ + char x=(char)y; + DumpVar(x,D); +} + +static void DumpInt(int x, DumpState* D) +{ + DumpVar(x,D); +} + +static void DumpNumber(lua_Number x, DumpState* D) +{ + DumpVar(x,D); +} + +static void DumpVector(const void* b, int n, size_t size, DumpState* D) +{ + DumpInt(n,D); + DumpMem(b,n,size,D); +} + +static void DumpString(const TString* s, DumpState* D) +{ + if (s==NULL || getstr(s)==NULL) + { + size_t size=0; + DumpVar(size,D); + } + else + { + size_t size=s->tsv.len+1; /* include trailing '\0' */ + DumpVar(size,D); + DumpBlock(getstr(s),size,D); + } +} + +#define DumpCode(f,D) DumpVector(f->code,f->sizecode,sizeof(Instruction),D) + +static void DumpFunction(const Proto* f, const TString* p, DumpState* D); + +static void DumpConstants(const Proto* f, DumpState* D) +{ + int i,n=f->sizek; + DumpInt(n,D); + for (i=0; ik[i]; + DumpChar(ttype(o),D); + switch (ttype(o)) + { + case LUA_TNIL: + break; + case LUA_TBOOLEAN: + DumpChar(bvalue(o),D); + break; + case LUA_TNUMBER: + DumpNumber(nvalue(o),D); + break; + case LUA_TSTRING: + DumpString(rawtsvalue(o),D); + break; + default: + lua_assert(0); /* cannot happen */ + break; + } + } + n=f->sizep; + DumpInt(n,D); + for (i=0; ip[i],f->source,D); +} + +static void DumpDebug(const Proto* f, DumpState* D) +{ + int i,n; + n= (D->strip) ? 0 : f->sizelineinfo; + DumpVector(f->lineinfo,n,sizeof(int),D); + n= (D->strip) ? 0 : f->sizelocvars; + DumpInt(n,D); + for (i=0; ilocvars[i].varname,D); + DumpInt(f->locvars[i].startpc,D); + DumpInt(f->locvars[i].endpc,D); + } + n= (D->strip) ? 0 : f->sizeupvalues; + DumpInt(n,D); + for (i=0; iupvalues[i],D); +} + +static void DumpFunction(const Proto* f, const TString* p, DumpState* D) +{ + DumpString((f->source==p || D->strip) ? NULL : f->source,D); + DumpInt(f->linedefined,D); + DumpInt(f->lastlinedefined,D); + DumpChar(f->nups,D); + DumpChar(f->numparams,D); + DumpChar(f->is_vararg,D); + DumpChar(f->maxstacksize,D); + DumpCode(f,D); + DumpConstants(f,D); + DumpDebug(f,D); +} + +static void DumpHeader(DumpState* D) +{ + char h[LUAC_HEADERSIZE]; + luaU_header(h); + DumpBlock(h,LUAC_HEADERSIZE,D); +} + +/* +** dump Lua function as precompiled chunk +*/ +int luaU_dump (lua_State* L, const Proto* f, lua_Writer w, void* data, int strip) +{ + DumpState D; + D.L=L; + D.writer=w; + D.data=data; + D.strip=strip; + D.status=0; + DumpHeader(&D); + DumpFunction(f,NULL,&D); + return D.status; +} diff --git a/src/lua/lfunc.c b/src/lua/lfunc.c new file mode 100644 index 0000000..813e88f --- /dev/null +++ b/src/lua/lfunc.c @@ -0,0 +1,174 @@ +/* +** $Id: lfunc.c,v 2.12.1.2 2007/12/28 14:58:43 roberto Exp $ +** Auxiliary functions to manipulate prototypes and closures +** See Copyright Notice in lua.h +*/ + + +#include + +#define lfunc_c +#define LUA_CORE + +#include "lua.h" + +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" + + + +Closure *luaF_newCclosure (lua_State *L, int nelems, Table *e) { + Closure *c = cast(Closure *, luaM_malloc(L, sizeCclosure(nelems))); + luaC_link(L, obj2gco(c), LUA_TFUNCTION); + c->c.isC = 1; + c->c.env = e; + c->c.nupvalues = cast_byte(nelems); + return c; +} + + +Closure *luaF_newLclosure (lua_State *L, int nelems, Table *e) { + Closure *c = cast(Closure *, luaM_malloc(L, sizeLclosure(nelems))); + luaC_link(L, obj2gco(c), LUA_TFUNCTION); + c->l.isC = 0; + c->l.env = e; + c->l.nupvalues = cast_byte(nelems); + while (nelems--) c->l.upvals[nelems] = NULL; + return c; +} + + +UpVal *luaF_newupval (lua_State *L) { + UpVal *uv = luaM_new(L, UpVal); + luaC_link(L, obj2gco(uv), LUA_TUPVAL); + uv->v = &uv->u.value; + setnilvalue(uv->v); + return uv; +} + + +UpVal *luaF_findupval (lua_State *L, StkId level) { + global_State *g = G(L); + GCObject **pp = &L->openupval; + UpVal *p; + UpVal *uv; + while (*pp != NULL && (p = ngcotouv(*pp))->v >= level) { + lua_assert(p->v != &p->u.value); + if (p->v == level) { /* found a corresponding upvalue? */ + if (isdead(g, obj2gco(p))) /* is it dead? */ + changewhite(obj2gco(p)); /* ressurect it */ + return p; + } + pp = &p->next; + } + uv = luaM_new(L, UpVal); /* not found: create a new one */ + uv->tt = LUA_TUPVAL; + uv->marked = luaC_white(g); + uv->v = level; /* current value lives in the stack */ + uv->next = *pp; /* chain it in the proper position */ + *pp = obj2gco(uv); + uv->u.l.prev = &g->uvhead; /* double link it in `uvhead' list */ + uv->u.l.next = g->uvhead.u.l.next; + uv->u.l.next->u.l.prev = uv; + g->uvhead.u.l.next = uv; + lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); + return uv; +} + + +static void unlinkupval (UpVal *uv) { + lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); + uv->u.l.next->u.l.prev = uv->u.l.prev; /* remove from `uvhead' list */ + uv->u.l.prev->u.l.next = uv->u.l.next; +} + + +void luaF_freeupval (lua_State *L, UpVal *uv) { + if (uv->v != &uv->u.value) /* is it open? */ + unlinkupval(uv); /* remove from open list */ + luaM_free(L, uv); /* free upvalue */ +} + + +void luaF_close (lua_State *L, StkId level) { + UpVal *uv; + global_State *g = G(L); + while (L->openupval != NULL && (uv = ngcotouv(L->openupval))->v >= level) { + GCObject *o = obj2gco(uv); + lua_assert(!isblack(o) && uv->v != &uv->u.value); + L->openupval = uv->next; /* remove from `open' list */ + if (isdead(g, o)) + luaF_freeupval(L, uv); /* free upvalue */ + else { + unlinkupval(uv); + setobj(L, &uv->u.value, uv->v); + uv->v = &uv->u.value; /* now current value lives here */ + luaC_linkupval(L, uv); /* link upvalue into `gcroot' list */ + } + } +} + + +Proto *luaF_newproto (lua_State *L) { + Proto *f = luaM_new(L, Proto); + luaC_link(L, obj2gco(f), LUA_TPROTO); + f->k = NULL; + f->sizek = 0; + f->p = NULL; + f->sizep = 0; + f->code = NULL; + f->sizecode = 0; + f->sizelineinfo = 0; + f->sizeupvalues = 0; + f->nups = 0; + f->upvalues = NULL; + f->numparams = 0; + f->is_vararg = 0; + f->maxstacksize = 0; + f->lineinfo = NULL; + f->sizelocvars = 0; + f->locvars = NULL; + f->linedefined = 0; + f->lastlinedefined = 0; + f->source = NULL; + return f; +} + + +void luaF_freeproto (lua_State *L, Proto *f) { + luaM_freearray(L, f->code, f->sizecode, Instruction); + luaM_freearray(L, f->p, f->sizep, Proto *); + luaM_freearray(L, f->k, f->sizek, TValue); + luaM_freearray(L, f->lineinfo, f->sizelineinfo, int); + luaM_freearray(L, f->locvars, f->sizelocvars, struct LocVar); + luaM_freearray(L, f->upvalues, f->sizeupvalues, TString *); + luaM_free(L, f); +} + + +void luaF_freeclosure (lua_State *L, Closure *c) { + int size = (c->c.isC) ? sizeCclosure(c->c.nupvalues) : + sizeLclosure(c->l.nupvalues); + luaM_freemem(L, c, size); +} + + +/* +** Look for n-th local variable at line `line' in function `func'. +** Returns NULL if not found. +*/ +const char *luaF_getlocalname (const Proto *f, int local_number, int pc) { + int i; + for (i = 0; isizelocvars && f->locvars[i].startpc <= pc; i++) { + if (pc < f->locvars[i].endpc) { /* is variable active? */ + local_number--; + if (local_number == 0) + return getstr(f->locvars[i].varname); + } + } + return NULL; /* not found */ +} + diff --git a/src/lua/lfunc.h b/src/lua/lfunc.h new file mode 100644 index 0000000..a68cf51 --- /dev/null +++ b/src/lua/lfunc.h @@ -0,0 +1,34 @@ +/* +** $Id: lfunc.h,v 2.4.1.1 2007/12/27 13:02:25 roberto Exp $ +** Auxiliary functions to manipulate prototypes and closures +** See Copyright Notice in lua.h +*/ + +#ifndef lfunc_h +#define lfunc_h + + +#include "lobject.h" + + +#define sizeCclosure(n) (cast(int, sizeof(CClosure)) + \ + cast(int, sizeof(TValue)*((n)-1))) + +#define sizeLclosure(n) (cast(int, sizeof(LClosure)) + \ + cast(int, sizeof(TValue *)*((n)-1))) + + +LUAI_FUNC Proto *luaF_newproto (lua_State *L); +LUAI_FUNC Closure *luaF_newCclosure (lua_State *L, int nelems, Table *e); +LUAI_FUNC Closure *luaF_newLclosure (lua_State *L, int nelems, Table *e); +LUAI_FUNC UpVal *luaF_newupval (lua_State *L); +LUAI_FUNC UpVal *luaF_findupval (lua_State *L, StkId level); +LUAI_FUNC void luaF_close (lua_State *L, StkId level); +LUAI_FUNC void luaF_freeproto (lua_State *L, Proto *f); +LUAI_FUNC void luaF_freeclosure (lua_State *L, Closure *c); +LUAI_FUNC void luaF_freeupval (lua_State *L, UpVal *uv); +LUAI_FUNC const char *luaF_getlocalname (const Proto *func, int local_number, + int pc); + + +#endif diff --git a/src/lua/lgc.c b/src/lua/lgc.c new file mode 100644 index 0000000..95bbdfb --- /dev/null +++ b/src/lua/lgc.c @@ -0,0 +1,710 @@ +/* +** $Id: lgc.c,v 2.38.1.2 2011/03/18 18:05:38 roberto Exp $ +** Garbage Collector +** See Copyright Notice in lua.h +*/ + +#include + +#define lgc_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" + + +#define GCSTEPSIZE 1024u +#define GCSWEEPMAX 40 +#define GCSWEEPCOST 10 +#define GCFINALIZECOST 100 + + +#define maskmarks cast_byte(~(bitmask(BLACKBIT)|WHITEBITS)) + +#define makewhite(g,x) \ + ((x)->gch.marked = cast_byte(((x)->gch.marked & maskmarks) | luaC_white(g))) + +#define white2gray(x) reset2bits((x)->gch.marked, WHITE0BIT, WHITE1BIT) +#define black2gray(x) resetbit((x)->gch.marked, BLACKBIT) + +#define stringmark(s) reset2bits((s)->tsv.marked, WHITE0BIT, WHITE1BIT) + + +#define isfinalized(u) testbit((u)->marked, FINALIZEDBIT) +#define markfinalized(u) l_setbit((u)->marked, FINALIZEDBIT) + + +#define KEYWEAK bitmask(KEYWEAKBIT) +#define VALUEWEAK bitmask(VALUEWEAKBIT) + + + +#define markvalue(g,o) { checkconsistency(o); \ + if (iscollectable(o) && iswhite(gcvalue(o))) reallymarkobject(g,gcvalue(o)); } + +#define markobject(g,t) { if (iswhite(obj2gco(t))) \ + reallymarkobject(g, obj2gco(t)); } + + +#define setthreshold(g) (g->GCthreshold = (g->estimate/100) * g->gcpause) + + +static void removeentry (Node *n) { + lua_assert(ttisnil(gval(n))); + if (iscollectable(gkey(n))) + setttype(gkey(n), LUA_TDEADKEY); /* dead key; remove it */ +} + + +static void reallymarkobject (global_State *g, GCObject *o) { + lua_assert(iswhite(o) && !isdead(g, o)); + white2gray(o); + switch (o->gch.tt) { + case LUA_TSTRING: { + return; + } + case LUA_TUSERDATA: { + Table *mt = gco2u(o)->metatable; + gray2black(o); /* udata are never gray */ + if (mt) markobject(g, mt); + markobject(g, gco2u(o)->env); + return; + } + case LUA_TUPVAL: { + UpVal *uv = gco2uv(o); + markvalue(g, uv->v); + if (uv->v == &uv->u.value) /* closed? */ + gray2black(o); /* open upvalues are never black */ + return; + } + case LUA_TFUNCTION: { + gco2cl(o)->c.gclist = g->gray; + g->gray = o; + break; + } + case LUA_TTABLE: { + gco2h(o)->gclist = g->gray; + g->gray = o; + break; + } + case LUA_TTHREAD: { + gco2th(o)->gclist = g->gray; + g->gray = o; + break; + } + case LUA_TPROTO: { + gco2p(o)->gclist = g->gray; + g->gray = o; + break; + } + default: lua_assert(0); + } +} + + +static void marktmu (global_State *g) { + GCObject *u = g->tmudata; + if (u) { + do { + u = u->gch.next; + makewhite(g, u); /* may be marked, if left from previous GC */ + reallymarkobject(g, u); + } while (u != g->tmudata); + } +} + + +/* move `dead' udata that need finalization to list `tmudata' */ +size_t luaC_separateudata (lua_State *L, int all) { + global_State *g = G(L); + size_t deadmem = 0; + GCObject **p = &g->mainthread->next; + GCObject *curr; + while ((curr = *p) != NULL) { + if (!(iswhite(curr) || all) || isfinalized(gco2u(curr))) + p = &curr->gch.next; /* don't bother with them */ + else if (fasttm(L, gco2u(curr)->metatable, TM_GC) == NULL) { + markfinalized(gco2u(curr)); /* don't need finalization */ + p = &curr->gch.next; + } + else { /* must call its gc method */ + deadmem += sizeudata(gco2u(curr)); + markfinalized(gco2u(curr)); + *p = curr->gch.next; + /* link `curr' at the end of `tmudata' list */ + if (g->tmudata == NULL) /* list is empty? */ + g->tmudata = curr->gch.next = curr; /* creates a circular list */ + else { + curr->gch.next = g->tmudata->gch.next; + g->tmudata->gch.next = curr; + g->tmudata = curr; + } + } + } + return deadmem; +} + + +static int traversetable (global_State *g, Table *h) { + int i; + int weakkey = 0; + int weakvalue = 0; + const TValue *mode; + if (h->metatable) + markobject(g, h->metatable); + mode = gfasttm(g, h->metatable, TM_MODE); + if (mode && ttisstring(mode)) { /* is there a weak mode? */ + weakkey = (strchr(svalue(mode), 'k') != NULL); + weakvalue = (strchr(svalue(mode), 'v') != NULL); + if (weakkey || weakvalue) { /* is really weak? */ + h->marked &= ~(KEYWEAK | VALUEWEAK); /* clear bits */ + h->marked |= cast_byte((weakkey << KEYWEAKBIT) | + (weakvalue << VALUEWEAKBIT)); + h->gclist = g->weak; /* must be cleared after GC, ... */ + g->weak = obj2gco(h); /* ... so put in the appropriate list */ + } + } + if (weakkey && weakvalue) return 1; + if (!weakvalue) { + i = h->sizearray; + while (i--) + markvalue(g, &h->array[i]); + } + i = sizenode(h); + while (i--) { + Node *n = gnode(h, i); + lua_assert(ttype(gkey(n)) != LUA_TDEADKEY || ttisnil(gval(n))); + if (ttisnil(gval(n))) + removeentry(n); /* remove empty entries */ + else { + lua_assert(!ttisnil(gkey(n))); + if (!weakkey) markvalue(g, gkey(n)); + if (!weakvalue) markvalue(g, gval(n)); + } + } + return weakkey || weakvalue; +} + + +/* +** All marks are conditional because a GC may happen while the +** prototype is still being created +*/ +static void traverseproto (global_State *g, Proto *f) { + int i; + if (f->source) stringmark(f->source); + for (i=0; isizek; i++) /* mark literals */ + markvalue(g, &f->k[i]); + for (i=0; isizeupvalues; i++) { /* mark upvalue names */ + if (f->upvalues[i]) + stringmark(f->upvalues[i]); + } + for (i=0; isizep; i++) { /* mark nested protos */ + if (f->p[i]) + markobject(g, f->p[i]); + } + for (i=0; isizelocvars; i++) { /* mark local-variable names */ + if (f->locvars[i].varname) + stringmark(f->locvars[i].varname); + } +} + + + +static void traverseclosure (global_State *g, Closure *cl) { + markobject(g, cl->c.env); + if (cl->c.isC) { + int i; + for (i=0; ic.nupvalues; i++) /* mark its upvalues */ + markvalue(g, &cl->c.upvalue[i]); + } + else { + int i; + lua_assert(cl->l.nupvalues == cl->l.p->nups); + markobject(g, cl->l.p); + for (i=0; il.nupvalues; i++) /* mark its upvalues */ + markobject(g, cl->l.upvals[i]); + } +} + + +static void checkstacksizes (lua_State *L, StkId max) { + int ci_used = cast_int(L->ci - L->base_ci); /* number of `ci' in use */ + int s_used = cast_int(max - L->stack); /* part of stack in use */ + if (L->size_ci > LUAI_MAXCALLS) /* handling overflow? */ + return; /* do not touch the stacks */ + if (4*ci_used < L->size_ci && 2*BASIC_CI_SIZE < L->size_ci) + luaD_reallocCI(L, L->size_ci/2); /* still big enough... */ + condhardstacktests(luaD_reallocCI(L, ci_used + 1)); + if (4*s_used < L->stacksize && + 2*(BASIC_STACK_SIZE+EXTRA_STACK) < L->stacksize) + luaD_reallocstack(L, L->stacksize/2); /* still big enough... */ + condhardstacktests(luaD_reallocstack(L, s_used)); +} + + +static void traversestack (global_State *g, lua_State *l) { + StkId o, lim; + CallInfo *ci; + markvalue(g, gt(l)); + lim = l->top; + for (ci = l->base_ci; ci <= l->ci; ci++) { + lua_assert(ci->top <= l->stack_last); + if (lim < ci->top) lim = ci->top; + } + for (o = l->stack; o < l->top; o++) + markvalue(g, o); + for (; o <= lim; o++) + setnilvalue(o); + checkstacksizes(l, lim); +} + + +/* +** traverse one gray object, turning it to black. +** Returns `quantity' traversed. +*/ +static l_mem propagatemark (global_State *g) { + GCObject *o = g->gray; + lua_assert(isgray(o)); + gray2black(o); + switch (o->gch.tt) { + case LUA_TTABLE: { + Table *h = gco2h(o); + g->gray = h->gclist; + if (traversetable(g, h)) /* table is weak? */ + black2gray(o); /* keep it gray */ + return sizeof(Table) + sizeof(TValue) * h->sizearray + + sizeof(Node) * (size_t)sizenode(h); + } + case LUA_TFUNCTION: { + Closure *cl = gco2cl(o); + g->gray = cl->c.gclist; + traverseclosure(g, cl); + return (cl->c.isC) ? sizeCclosure(cl->c.nupvalues) : + sizeLclosure(cl->l.nupvalues); + } + case LUA_TTHREAD: { + lua_State *th = gco2th(o); + g->gray = th->gclist; + th->gclist = g->grayagain; + g->grayagain = o; + black2gray(o); + traversestack(g, th); + return sizeof(lua_State) + sizeof(TValue) * th->stacksize + + sizeof(CallInfo) * th->size_ci; + } + case LUA_TPROTO: { + Proto *p = gco2p(o); + g->gray = p->gclist; + traverseproto(g, p); + return sizeof(Proto) + sizeof(Instruction) * p->sizecode + + sizeof(Proto *) * p->sizep + + sizeof(TValue) * p->sizek + + sizeof(int) * p->sizelineinfo + + sizeof(LocVar) * p->sizelocvars + + sizeof(TString *) * p->sizeupvalues; + } + default: lua_assert(0); return 0; + } +} + + +static size_t propagateall (global_State *g) { + size_t m = 0; + while (g->gray) m += propagatemark(g); + return m; +} + + +/* +** The next function tells whether a key or value can be cleared from +** a weak table. Non-collectable objects are never removed from weak +** tables. Strings behave as `values', so are never removed too. for +** other objects: if really collected, cannot keep them; for userdata +** being finalized, keep them in keys, but not in values +*/ +static int iscleared (const TValue *o, int iskey) { + if (!iscollectable(o)) return 0; + if (ttisstring(o)) { + stringmark(rawtsvalue(o)); /* strings are `values', so are never weak */ + return 0; + } + return iswhite(gcvalue(o)) || + (ttisuserdata(o) && (!iskey && isfinalized(uvalue(o)))); +} + + +/* +** clear collected entries from weaktables +*/ +static void cleartable (GCObject *l) { + while (l) { + Table *h = gco2h(l); + int i = h->sizearray; + lua_assert(testbit(h->marked, VALUEWEAKBIT) || + testbit(h->marked, KEYWEAKBIT)); + if (testbit(h->marked, VALUEWEAKBIT)) { + while (i--) { + TValue *o = &h->array[i]; + if (iscleared(o, 0)) /* value was collected? */ + setnilvalue(o); /* remove value */ + } + } + i = sizenode(h); + while (i--) { + Node *n = gnode(h, i); + if (!ttisnil(gval(n)) && /* non-empty entry? */ + (iscleared(key2tval(n), 1) || iscleared(gval(n), 0))) { + setnilvalue(gval(n)); /* remove value ... */ + removeentry(n); /* remove entry from table */ + } + } + l = h->gclist; + } +} + + +static void freeobj (lua_State *L, GCObject *o) { + switch (o->gch.tt) { + case LUA_TPROTO: luaF_freeproto(L, gco2p(o)); break; + case LUA_TFUNCTION: luaF_freeclosure(L, gco2cl(o)); break; + case LUA_TUPVAL: luaF_freeupval(L, gco2uv(o)); break; + case LUA_TTABLE: luaH_free(L, gco2h(o)); break; + case LUA_TTHREAD: { + lua_assert(gco2th(o) != L && gco2th(o) != G(L)->mainthread); + luaE_freethread(L, gco2th(o)); + break; + } + case LUA_TSTRING: { + G(L)->strt.nuse--; + luaM_freemem(L, o, sizestring(gco2ts(o))); + break; + } + case LUA_TUSERDATA: { + luaM_freemem(L, o, sizeudata(gco2u(o))); + break; + } + default: lua_assert(0); + } +} + + + +#define sweepwholelist(L,p) sweeplist(L,p,MAX_LUMEM) + + +static GCObject **sweeplist (lua_State *L, GCObject **p, lu_mem count) { + GCObject *curr; + global_State *g = G(L); + int deadmask = otherwhite(g); + while ((curr = *p) != NULL && count-- > 0) { + if (curr->gch.tt == LUA_TTHREAD) /* sweep open upvalues of each thread */ + sweepwholelist(L, &gco2th(curr)->openupval); + if ((curr->gch.marked ^ WHITEBITS) & deadmask) { /* not dead? */ + lua_assert(!isdead(g, curr) || testbit(curr->gch.marked, FIXEDBIT)); + makewhite(g, curr); /* make it white (for next cycle) */ + p = &curr->gch.next; + } + else { /* must erase `curr' */ + lua_assert(isdead(g, curr) || deadmask == bitmask(SFIXEDBIT)); + *p = curr->gch.next; + if (curr == g->rootgc) /* is the first element of the list? */ + g->rootgc = curr->gch.next; /* adjust first */ + freeobj(L, curr); + } + } + return p; +} + + +static void checkSizes (lua_State *L) { + global_State *g = G(L); + /* check size of string hash */ + if (g->strt.nuse < cast(lu_int32, g->strt.size/4) && + g->strt.size > MINSTRTABSIZE*2) + luaS_resize(L, g->strt.size/2); /* table is too big */ + /* check size of buffer */ + if (luaZ_sizebuffer(&g->buff) > LUA_MINBUFFER*2) { /* buffer too big? */ + size_t newsize = luaZ_sizebuffer(&g->buff) / 2; + luaZ_resizebuffer(L, &g->buff, newsize); + } +} + + +static void GCTM (lua_State *L) { + global_State *g = G(L); + GCObject *o = g->tmudata->gch.next; /* get first element */ + Udata *udata = rawgco2u(o); + const TValue *tm; + /* remove udata from `tmudata' */ + if (o == g->tmudata) /* last element? */ + g->tmudata = NULL; + else + g->tmudata->gch.next = udata->uv.next; + udata->uv.next = g->mainthread->next; /* return it to `root' list */ + g->mainthread->next = o; + makewhite(g, o); + tm = fasttm(L, udata->uv.metatable, TM_GC); + if (tm != NULL) { + lu_byte oldah = L->allowhook; + lu_mem oldt = g->GCthreshold; + L->allowhook = 0; /* stop debug hooks during GC tag method */ + g->GCthreshold = 2*g->totalbytes; /* avoid GC steps */ + setobj2s(L, L->top, tm); + setuvalue(L, L->top+1, udata); + L->top += 2; + luaD_call(L, L->top - 2, 0); + L->allowhook = oldah; /* restore hooks */ + g->GCthreshold = oldt; /* restore threshold */ + } +} + + +/* +** Call all GC tag methods +*/ +void luaC_callGCTM (lua_State *L) { + while (G(L)->tmudata) + GCTM(L); +} + + +void luaC_freeall (lua_State *L) { + global_State *g = G(L); + int i; + g->currentwhite = WHITEBITS | bitmask(SFIXEDBIT); /* mask to collect all elements */ + sweepwholelist(L, &g->rootgc); + for (i = 0; i < g->strt.size; i++) /* free all string lists */ + sweepwholelist(L, &g->strt.hash[i]); +} + + +static void markmt (global_State *g) { + int i; + for (i=0; imt[i]) markobject(g, g->mt[i]); +} + + +/* mark root set */ +static void markroot (lua_State *L) { + global_State *g = G(L); + g->gray = NULL; + g->grayagain = NULL; + g->weak = NULL; + markobject(g, g->mainthread); + /* make global table be traversed before main stack */ + markvalue(g, gt(g->mainthread)); + markvalue(g, registry(L)); + markmt(g); + g->gcstate = GCSpropagate; +} + + +static void remarkupvals (global_State *g) { + UpVal *uv; + for (uv = g->uvhead.u.l.next; uv != &g->uvhead; uv = uv->u.l.next) { + lua_assert(uv->u.l.next->u.l.prev == uv && uv->u.l.prev->u.l.next == uv); + if (isgray(obj2gco(uv))) + markvalue(g, uv->v); + } +} + + +static void atomic (lua_State *L) { + global_State *g = G(L); + size_t udsize; /* total size of userdata to be finalized */ + /* remark occasional upvalues of (maybe) dead threads */ + remarkupvals(g); + /* traverse objects cautch by write barrier and by 'remarkupvals' */ + propagateall(g); + /* remark weak tables */ + g->gray = g->weak; + g->weak = NULL; + lua_assert(!iswhite(obj2gco(g->mainthread))); + markobject(g, L); /* mark running thread */ + markmt(g); /* mark basic metatables (again) */ + propagateall(g); + /* remark gray again */ + g->gray = g->grayagain; + g->grayagain = NULL; + propagateall(g); + udsize = luaC_separateudata(L, 0); /* separate userdata to be finalized */ + marktmu(g); /* mark `preserved' userdata */ + udsize += propagateall(g); /* remark, to propagate `preserveness' */ + cleartable(g->weak); /* remove collected objects from weak tables */ + /* flip current white */ + g->currentwhite = cast_byte(otherwhite(g)); + g->sweepstrgc = 0; + g->sweepgc = &g->rootgc; + g->gcstate = GCSsweepstring; + g->estimate = g->totalbytes - udsize; /* first estimate */ +} + + +static l_mem singlestep (lua_State *L) { + global_State *g = G(L); + /*lua_checkmemory(L);*/ + switch (g->gcstate) { + case GCSpause: { + markroot(L); /* start a new collection */ + return 0; + } + case GCSpropagate: { + if (g->gray) + return propagatemark(g); + else { /* no more `gray' objects */ + atomic(L); /* finish mark phase */ + return 0; + } + } + case GCSsweepstring: { + lu_mem old = g->totalbytes; + sweepwholelist(L, &g->strt.hash[g->sweepstrgc++]); + if (g->sweepstrgc >= g->strt.size) /* nothing more to sweep? */ + g->gcstate = GCSsweep; /* end sweep-string phase */ + lua_assert(old >= g->totalbytes); + g->estimate -= old - g->totalbytes; + return GCSWEEPCOST; + } + case GCSsweep: { + lu_mem old = g->totalbytes; + g->sweepgc = sweeplist(L, g->sweepgc, GCSWEEPMAX); + if (*g->sweepgc == NULL) { /* nothing more to sweep? */ + checkSizes(L); + g->gcstate = GCSfinalize; /* end sweep phase */ + } + lua_assert(old >= g->totalbytes); + g->estimate -= old - g->totalbytes; + return GCSWEEPMAX*GCSWEEPCOST; + } + case GCSfinalize: { + if (g->tmudata) { + GCTM(L); + if (g->estimate > GCFINALIZECOST) + g->estimate -= GCFINALIZECOST; + return GCFINALIZECOST; + } + else { + g->gcstate = GCSpause; /* end collection */ + g->gcdept = 0; + return 0; + } + } + default: lua_assert(0); return 0; + } +} + + +void luaC_step (lua_State *L) { + global_State *g = G(L); + l_mem lim = (GCSTEPSIZE/100) * g->gcstepmul; + if (lim == 0) + lim = (MAX_LUMEM-1)/2; /* no limit */ + g->gcdept += g->totalbytes - g->GCthreshold; + do { + lim -= singlestep(L); + if (g->gcstate == GCSpause) + break; + } while (lim > 0); + if (g->gcstate != GCSpause) { + if (g->gcdept < GCSTEPSIZE) + g->GCthreshold = g->totalbytes + GCSTEPSIZE; /* - lim/g->gcstepmul;*/ + else { + g->gcdept -= GCSTEPSIZE; + g->GCthreshold = g->totalbytes; + } + } + else { + setthreshold(g); + } +} + + +void luaC_fullgc (lua_State *L) { + global_State *g = G(L); + if (g->gcstate <= GCSpropagate) { + /* reset sweep marks to sweep all elements (returning them to white) */ + g->sweepstrgc = 0; + g->sweepgc = &g->rootgc; + /* reset other collector lists */ + g->gray = NULL; + g->grayagain = NULL; + g->weak = NULL; + g->gcstate = GCSsweepstring; + } + lua_assert(g->gcstate != GCSpause && g->gcstate != GCSpropagate); + /* finish any pending sweep phase */ + while (g->gcstate != GCSfinalize) { + lua_assert(g->gcstate == GCSsweepstring || g->gcstate == GCSsweep); + singlestep(L); + } + markroot(L); + while (g->gcstate != GCSpause) { + singlestep(L); + } + setthreshold(g); +} + + +void luaC_barrierf (lua_State *L, GCObject *o, GCObject *v) { + global_State *g = G(L); + lua_assert(isblack(o) && iswhite(v) && !isdead(g, v) && !isdead(g, o)); + lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause); + lua_assert(ttype(&o->gch) != LUA_TTABLE); + /* must keep invariant? */ + if (g->gcstate == GCSpropagate) + reallymarkobject(g, v); /* restore invariant */ + else /* don't mind */ + makewhite(g, o); /* mark as white just to avoid other barriers */ +} + + +void luaC_barrierback (lua_State *L, Table *t) { + global_State *g = G(L); + GCObject *o = obj2gco(t); + lua_assert(isblack(o) && !isdead(g, o)); + lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause); + black2gray(o); /* make table gray (again) */ + t->gclist = g->grayagain; + g->grayagain = o; +} + + +void luaC_link (lua_State *L, GCObject *o, lu_byte tt) { + global_State *g = G(L); + o->gch.next = g->rootgc; + g->rootgc = o; + o->gch.marked = luaC_white(g); + o->gch.tt = tt; +} + + +void luaC_linkupval (lua_State *L, UpVal *uv) { + global_State *g = G(L); + GCObject *o = obj2gco(uv); + o->gch.next = g->rootgc; /* link upvalue into `rootgc' list */ + g->rootgc = o; + if (isgray(o)) { + if (g->gcstate == GCSpropagate) { + gray2black(o); /* closed upvalues need barrier */ + luaC_barrier(L, uv, uv->v); + } + else { /* sweep phase: sweep it (turning it into white) */ + makewhite(g, o); + lua_assert(g->gcstate != GCSfinalize && g->gcstate != GCSpause); + } + } +} + diff --git a/src/lua/lgc.h b/src/lua/lgc.h new file mode 100644 index 0000000..5a8dc60 --- /dev/null +++ b/src/lua/lgc.h @@ -0,0 +1,110 @@ +/* +** $Id: lgc.h,v 2.15.1.1 2007/12/27 13:02:25 roberto Exp $ +** Garbage Collector +** See Copyright Notice in lua.h +*/ + +#ifndef lgc_h +#define lgc_h + + +#include "lobject.h" + + +/* +** Possible states of the Garbage Collector +*/ +#define GCSpause 0 +#define GCSpropagate 1 +#define GCSsweepstring 2 +#define GCSsweep 3 +#define GCSfinalize 4 + + +/* +** some userful bit tricks +*/ +#define resetbits(x,m) ((x) &= cast(lu_byte, ~(m))) +#define setbits(x,m) ((x) |= (m)) +#define testbits(x,m) ((x) & (m)) +#define bitmask(b) (1<<(b)) +#define bit2mask(b1,b2) (bitmask(b1) | bitmask(b2)) +#define l_setbit(x,b) setbits(x, bitmask(b)) +#define resetbit(x,b) resetbits(x, bitmask(b)) +#define testbit(x,b) testbits(x, bitmask(b)) +#define set2bits(x,b1,b2) setbits(x, (bit2mask(b1, b2))) +#define reset2bits(x,b1,b2) resetbits(x, (bit2mask(b1, b2))) +#define test2bits(x,b1,b2) testbits(x, (bit2mask(b1, b2))) + + + +/* +** Layout for bit use in `marked' field: +** bit 0 - object is white (type 0) +** bit 1 - object is white (type 1) +** bit 2 - object is black +** bit 3 - for userdata: has been finalized +** bit 3 - for tables: has weak keys +** bit 4 - for tables: has weak values +** bit 5 - object is fixed (should not be collected) +** bit 6 - object is "super" fixed (only the main thread) +*/ + + +#define WHITE0BIT 0 +#define WHITE1BIT 1 +#define BLACKBIT 2 +#define FINALIZEDBIT 3 +#define KEYWEAKBIT 3 +#define VALUEWEAKBIT 4 +#define FIXEDBIT 5 +#define SFIXEDBIT 6 +#define WHITEBITS bit2mask(WHITE0BIT, WHITE1BIT) + + +#define iswhite(x) test2bits((x)->gch.marked, WHITE0BIT, WHITE1BIT) +#define isblack(x) testbit((x)->gch.marked, BLACKBIT) +#define isgray(x) (!isblack(x) && !iswhite(x)) + +#define otherwhite(g) (g->currentwhite ^ WHITEBITS) +#define isdead(g,v) ((v)->gch.marked & otherwhite(g) & WHITEBITS) + +#define changewhite(x) ((x)->gch.marked ^= WHITEBITS) +#define gray2black(x) l_setbit((x)->gch.marked, BLACKBIT) + +#define valiswhite(x) (iscollectable(x) && iswhite(gcvalue(x))) + +#define luaC_white(g) cast(lu_byte, (g)->currentwhite & WHITEBITS) + + +#define luaC_checkGC(L) { \ + condhardstacktests(luaD_reallocstack(L, L->stacksize - EXTRA_STACK - 1)); \ + if (G(L)->totalbytes >= G(L)->GCthreshold) \ + luaC_step(L); } + + +#define luaC_barrier(L,p,v) { if (valiswhite(v) && isblack(obj2gco(p))) \ + luaC_barrierf(L,obj2gco(p),gcvalue(v)); } + +#define luaC_barriert(L,t,v) { if (valiswhite(v) && isblack(obj2gco(t))) \ + luaC_barrierback(L,t); } + +#define luaC_objbarrier(L,p,o) \ + { if (iswhite(obj2gco(o)) && isblack(obj2gco(p))) \ + luaC_barrierf(L,obj2gco(p),obj2gco(o)); } + +#define luaC_objbarriert(L,t,o) \ + { if (iswhite(obj2gco(o)) && isblack(obj2gco(t))) luaC_barrierback(L,t); } + +LUAI_FUNC size_t luaC_separateudata (lua_State *L, int all); +LUAI_FUNC void luaC_callGCTM (lua_State *L); +LUAI_FUNC void luaC_freeall (lua_State *L); +LUAI_FUNC void luaC_step (lua_State *L); +LUAI_FUNC void luaC_fullgc (lua_State *L); +LUAI_FUNC void luaC_link (lua_State *L, GCObject *o, lu_byte tt); +LUAI_FUNC void luaC_linkupval (lua_State *L, UpVal *uv); +LUAI_FUNC void luaC_barrierf (lua_State *L, GCObject *o, GCObject *v); +LUAI_FUNC void luaC_barrierback (lua_State *L, Table *t); + + +#endif diff --git a/src/lua/linit.c b/src/lua/linit.c new file mode 100644 index 0000000..c1f90df --- /dev/null +++ b/src/lua/linit.c @@ -0,0 +1,38 @@ +/* +** $Id: linit.c,v 1.14.1.1 2007/12/27 13:02:25 roberto Exp $ +** Initialization of libraries for lua.c +** See Copyright Notice in lua.h +*/ + + +#define linit_c +#define LUA_LIB + +#include "lua.h" + +#include "lualib.h" +#include "lauxlib.h" + + +static const luaL_Reg lualibs[] = { + {"", luaopen_base}, + {LUA_LOADLIBNAME, luaopen_package}, + {LUA_TABLIBNAME, luaopen_table}, + {LUA_IOLIBNAME, luaopen_io}, + {LUA_OSLIBNAME, luaopen_os}, + {LUA_STRLIBNAME, luaopen_string}, + {LUA_MATHLIBNAME, luaopen_math}, + {LUA_DBLIBNAME, luaopen_debug}, + {NULL, NULL} +}; + + +LUALIB_API void luaL_openlibs (lua_State *L) { + const luaL_Reg *lib = lualibs; + for (; lib->func; lib++) { + lua_pushcfunction(L, lib->func); + lua_pushstring(L, lib->name); + lua_call(L, 1, 0); + } +} + diff --git a/src/lua/liolib.c b/src/lua/liolib.c new file mode 100644 index 0000000..649f9a5 --- /dev/null +++ b/src/lua/liolib.c @@ -0,0 +1,556 @@ +/* +** $Id: liolib.c,v 2.73.1.4 2010/05/14 15:33:51 roberto Exp $ +** Standard I/O (and system) library +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include + +#define liolib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + + +#define IO_INPUT 1 +#define IO_OUTPUT 2 + + +static const char *const fnames[] = {"input", "output"}; + + +static int pushresult (lua_State *L, int i, const char *filename) { + int en = errno; /* calls to Lua API may change this value */ + if (i) { + lua_pushboolean(L, 1); + return 1; + } + else { + lua_pushnil(L); + if (filename) + lua_pushfstring(L, "%s: %s", filename, strerror(en)); + else + lua_pushfstring(L, "%s", strerror(en)); + lua_pushinteger(L, en); + return 3; + } +} + + +static void fileerror (lua_State *L, int arg, const char *filename) { + lua_pushfstring(L, "%s: %s", filename, strerror(errno)); + luaL_argerror(L, arg, lua_tostring(L, -1)); +} + + +#define tofilep(L) ((FILE **)luaL_checkudata(L, 1, LUA_FILEHANDLE)) + + +static int io_type (lua_State *L) { + void *ud; + luaL_checkany(L, 1); + ud = lua_touserdata(L, 1); + lua_getfield(L, LUA_REGISTRYINDEX, LUA_FILEHANDLE); + if (ud == NULL || !lua_getmetatable(L, 1) || !lua_rawequal(L, -2, -1)) + lua_pushnil(L); /* not a file */ + else if (*((FILE **)ud) == NULL) + lua_pushliteral(L, "closed file"); + else + lua_pushliteral(L, "file"); + return 1; +} + + +static FILE *tofile (lua_State *L) { + FILE **f = tofilep(L); + if (*f == NULL) + luaL_error(L, "attempt to use a closed file"); + return *f; +} + + + +/* +** When creating file handles, always creates a `closed' file handle +** before opening the actual file; so, if there is a memory error, the +** file is not left opened. +*/ +static FILE **newfile (lua_State *L) { + FILE **pf = (FILE **)lua_newuserdata(L, sizeof(FILE *)); + *pf = NULL; /* file handle is currently `closed' */ + luaL_getmetatable(L, LUA_FILEHANDLE); + lua_setmetatable(L, -2); + return pf; +} + + +/* +** function to (not) close the standard files stdin, stdout, and stderr +*/ +static int io_noclose (lua_State *L) { + lua_pushnil(L); + lua_pushliteral(L, "cannot close standard file"); + return 2; +} + + +/* +** function to close 'popen' files +*/ +static int io_pclose (lua_State *L) { + FILE **p = tofilep(L); + int ok = lua_pclose(L, *p); + *p = NULL; + return pushresult(L, ok, NULL); +} + + +/* +** function to close regular files +*/ +static int io_fclose (lua_State *L) { + FILE **p = tofilep(L); + int ok = (fclose(*p) == 0); + *p = NULL; + return pushresult(L, ok, NULL); +} + + +static int aux_close (lua_State *L) { + lua_getfenv(L, 1); + lua_getfield(L, -1, "__close"); + return (lua_tocfunction(L, -1))(L); +} + + +static int io_close (lua_State *L) { + if (lua_isnone(L, 1)) + lua_rawgeti(L, LUA_ENVIRONINDEX, IO_OUTPUT); + tofile(L); /* make sure argument is a file */ + return aux_close(L); +} + + +static int io_gc (lua_State *L) { + FILE *f = *tofilep(L); + /* ignore closed files */ + if (f != NULL) + aux_close(L); + return 0; +} + + +static int io_tostring (lua_State *L) { + FILE *f = *tofilep(L); + if (f == NULL) + lua_pushliteral(L, "file (closed)"); + else + lua_pushfstring(L, "file (%p)", f); + return 1; +} + + +static int io_open (lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + const char *mode = luaL_optstring(L, 2, "r"); + FILE **pf = newfile(L); + *pf = fopen(filename, mode); + return (*pf == NULL) ? pushresult(L, 0, filename) : 1; +} + + +/* +** this function has a separated environment, which defines the +** correct __close for 'popen' files +*/ +static int io_popen (lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + const char *mode = luaL_optstring(L, 2, "r"); + FILE **pf = newfile(L); + *pf = lua_popen(L, filename, mode); + return (*pf == NULL) ? pushresult(L, 0, filename) : 1; +} + + +static int io_tmpfile (lua_State *L) { + FILE **pf = newfile(L); + *pf = tmpfile(); + return (*pf == NULL) ? pushresult(L, 0, NULL) : 1; +} + + +static FILE *getiofile (lua_State *L, int findex) { + FILE *f; + lua_rawgeti(L, LUA_ENVIRONINDEX, findex); + f = *(FILE **)lua_touserdata(L, -1); + if (f == NULL) + luaL_error(L, "standard %s file is closed", fnames[findex - 1]); + return f; +} + + +static int g_iofile (lua_State *L, int f, const char *mode) { + if (!lua_isnoneornil(L, 1)) { + const char *filename = lua_tostring(L, 1); + if (filename) { + FILE **pf = newfile(L); + *pf = fopen(filename, mode); + if (*pf == NULL) + fileerror(L, 1, filename); + } + else { + tofile(L); /* check that it's a valid file handle */ + lua_pushvalue(L, 1); + } + lua_rawseti(L, LUA_ENVIRONINDEX, f); + } + /* return current value */ + lua_rawgeti(L, LUA_ENVIRONINDEX, f); + return 1; +} + + +static int io_input (lua_State *L) { + return g_iofile(L, IO_INPUT, "r"); +} + + +static int io_output (lua_State *L) { + return g_iofile(L, IO_OUTPUT, "w"); +} + + +static int io_readline (lua_State *L); + + +static void aux_lines (lua_State *L, int idx, int toclose) { + lua_pushvalue(L, idx); + lua_pushboolean(L, toclose); /* close/not close file when finished */ + lua_pushcclosure(L, io_readline, 2); +} + + +static int f_lines (lua_State *L) { + tofile(L); /* check that it's a valid file handle */ + aux_lines(L, 1, 0); + return 1; +} + + +static int io_lines (lua_State *L) { + if (lua_isnoneornil(L, 1)) { /* no arguments? */ + /* will iterate over default input */ + lua_rawgeti(L, LUA_ENVIRONINDEX, IO_INPUT); + return f_lines(L); + } + else { + const char *filename = luaL_checkstring(L, 1); + FILE **pf = newfile(L); + *pf = fopen(filename, "r"); + if (*pf == NULL) + fileerror(L, 1, filename); + aux_lines(L, lua_gettop(L), 1); + return 1; + } +} + + +/* +** {====================================================== +** READ +** ======================================================= +*/ + + +static int read_number (lua_State *L, FILE *f) { + lua_Number d; + if (fscanf(f, LUA_NUMBER_SCAN, &d) == 1) { + lua_pushnumber(L, d); + return 1; + } + else { + lua_pushnil(L); /* "result" to be removed */ + return 0; /* read fails */ + } +} + + +static int test_eof (lua_State *L, FILE *f) { + int c = getc(f); + ungetc(c, f); + lua_pushlstring(L, NULL, 0); + return (c != EOF); +} + + +static int read_line (lua_State *L, FILE *f) { + luaL_Buffer b; + luaL_buffinit(L, &b); + for (;;) { + size_t l; + char *p = luaL_prepbuffer(&b); + if (fgets(p, LUAL_BUFFERSIZE, f) == NULL) { /* eof? */ + luaL_pushresult(&b); /* close buffer */ + return (lua_objlen(L, -1) > 0); /* check whether read something */ + } + l = strlen(p); + if (l == 0 || p[l-1] != '\n') + luaL_addsize(&b, l); + else { + luaL_addsize(&b, l - 1); /* do not include `eol' */ + luaL_pushresult(&b); /* close buffer */ + return 1; /* read at least an `eol' */ + } + } +} + + +static int read_chars (lua_State *L, FILE *f, size_t n) { + size_t rlen; /* how much to read */ + size_t nr; /* number of chars actually read */ + luaL_Buffer b; + luaL_buffinit(L, &b); + rlen = LUAL_BUFFERSIZE; /* try to read that much each time */ + do { + char *p = luaL_prepbuffer(&b); + if (rlen > n) rlen = n; /* cannot read more than asked */ + nr = fread(p, sizeof(char), rlen, f); + luaL_addsize(&b, nr); + n -= nr; /* still have to read `n' chars */ + } while (n > 0 && nr == rlen); /* until end of count or eof */ + luaL_pushresult(&b); /* close buffer */ + return (n == 0 || lua_objlen(L, -1) > 0); +} + + +static int g_read (lua_State *L, FILE *f, int first) { + int nargs = lua_gettop(L) - 1; + int success; + int n; + clearerr(f); + if (nargs == 0) { /* no arguments? */ + success = read_line(L, f); + n = first+1; /* to return 1 result */ + } + else { /* ensure stack space for all results and for auxlib's buffer */ + luaL_checkstack(L, nargs+LUA_MINSTACK, "too many arguments"); + success = 1; + for (n = first; nargs-- && success; n++) { + if (lua_type(L, n) == LUA_TNUMBER) { + size_t l = (size_t)lua_tointeger(L, n); + success = (l == 0) ? test_eof(L, f) : read_chars(L, f, l); + } + else { + const char *p = lua_tostring(L, n); + luaL_argcheck(L, p && p[0] == '*', n, "invalid option"); + switch (p[1]) { + case 'n': /* number */ + success = read_number(L, f); + break; + case 'l': /* line */ + success = read_line(L, f); + break; + case 'a': /* file */ + read_chars(L, f, ~((size_t)0)); /* read MAX_SIZE_T chars */ + success = 1; /* always success */ + break; + default: + return luaL_argerror(L, n, "invalid format"); + } + } + } + } + if (ferror(f)) + return pushresult(L, 0, NULL); + if (!success) { + lua_pop(L, 1); /* remove last result */ + lua_pushnil(L); /* push nil instead */ + } + return n - first; +} + + +static int io_read (lua_State *L) { + return g_read(L, getiofile(L, IO_INPUT), 1); +} + + +static int f_read (lua_State *L) { + return g_read(L, tofile(L), 2); +} + + +static int io_readline (lua_State *L) { + FILE *f = *(FILE **)lua_touserdata(L, lua_upvalueindex(1)); + int sucess; + if (f == NULL) /* file is already closed? */ + luaL_error(L, "file is already closed"); + sucess = read_line(L, f); + if (ferror(f)) + return luaL_error(L, "%s", strerror(errno)); + if (sucess) return 1; + else { /* EOF */ + if (lua_toboolean(L, lua_upvalueindex(2))) { /* generator created file? */ + lua_settop(L, 0); + lua_pushvalue(L, lua_upvalueindex(1)); + aux_close(L); /* close it */ + } + return 0; + } +} + +/* }====================================================== */ + + +static int g_write (lua_State *L, FILE *f, int arg) { + int nargs = lua_gettop(L) - 1; + int status = 1; + for (; nargs--; arg++) { + if (lua_type(L, arg) == LUA_TNUMBER) { + /* optimization: could be done exactly as for strings */ + status = status && + fprintf(f, LUA_NUMBER_FMT, lua_tonumber(L, arg)) > 0; + } + else { + size_t l; + const char *s = luaL_checklstring(L, arg, &l); + status = status && (fwrite(s, sizeof(char), l, f) == l); + } + } + return pushresult(L, status, NULL); +} + + +static int io_write (lua_State *L) { + return g_write(L, getiofile(L, IO_OUTPUT), 1); +} + + +static int f_write (lua_State *L) { + return g_write(L, tofile(L), 2); +} + + +static int f_seek (lua_State *L) { + static const int mode[] = {SEEK_SET, SEEK_CUR, SEEK_END}; + static const char *const modenames[] = {"set", "cur", "end", NULL}; + FILE *f = tofile(L); + int op = luaL_checkoption(L, 2, "cur", modenames); + long offset = luaL_optlong(L, 3, 0); + op = fseek(f, offset, mode[op]); + if (op) + return pushresult(L, 0, NULL); /* error */ + else { + lua_pushinteger(L, ftell(f)); + return 1; + } +} + + +static int f_setvbuf (lua_State *L) { + static const int mode[] = {_IONBF, _IOFBF, _IOLBF}; + static const char *const modenames[] = {"no", "full", "line", NULL}; + FILE *f = tofile(L); + int op = luaL_checkoption(L, 2, NULL, modenames); + lua_Integer sz = luaL_optinteger(L, 3, LUAL_BUFFERSIZE); + int res = setvbuf(f, NULL, mode[op], sz); + return pushresult(L, res == 0, NULL); +} + + + +static int io_flush (lua_State *L) { + return pushresult(L, fflush(getiofile(L, IO_OUTPUT)) == 0, NULL); +} + + +static int f_flush (lua_State *L) { + return pushresult(L, fflush(tofile(L)) == 0, NULL); +} + + +static const luaL_Reg iolib[] = { + {"close", io_close}, + {"flush", io_flush}, + {"input", io_input}, + {"lines", io_lines}, + {"open", io_open}, + {"output", io_output}, + {"popen", io_popen}, + {"read", io_read}, + {"tmpfile", io_tmpfile}, + {"type", io_type}, + {"write", io_write}, + {NULL, NULL} +}; + + +static const luaL_Reg flib[] = { + {"close", io_close}, + {"flush", f_flush}, + {"lines", f_lines}, + {"read", f_read}, + {"seek", f_seek}, + {"setvbuf", f_setvbuf}, + {"write", f_write}, + {"__gc", io_gc}, + {"__tostring", io_tostring}, + {NULL, NULL} +}; + + +static void createmeta (lua_State *L) { + luaL_newmetatable(L, LUA_FILEHANDLE); /* create metatable for file handles */ + lua_pushvalue(L, -1); /* push metatable */ + lua_setfield(L, -2, "__index"); /* metatable.__index = metatable */ + luaL_register(L, NULL, flib); /* file methods */ +} + + +static void createstdfile (lua_State *L, FILE *f, int k, const char *fname) { + *newfile(L) = f; + if (k > 0) { + lua_pushvalue(L, -1); + lua_rawseti(L, LUA_ENVIRONINDEX, k); + } + lua_pushvalue(L, -2); /* copy environment */ + lua_setfenv(L, -2); /* set it */ + lua_setfield(L, -3, fname); +} + + +static void newfenv (lua_State *L, lua_CFunction cls) { + lua_createtable(L, 0, 1); + lua_pushcfunction(L, cls); + lua_setfield(L, -2, "__close"); +} + + +LUALIB_API int luaopen_io (lua_State *L) { + createmeta(L); + /* create (private) environment (with fields IO_INPUT, IO_OUTPUT, __close) */ + newfenv(L, io_fclose); + lua_replace(L, LUA_ENVIRONINDEX); + /* open library */ + luaL_register(L, LUA_IOLIBNAME, iolib); + /* create (and set) default files */ + newfenv(L, io_noclose); /* close function for default files */ + createstdfile(L, stdin, IO_INPUT, "stdin"); + createstdfile(L, stdout, IO_OUTPUT, "stdout"); + createstdfile(L, stderr, 0, "stderr"); + lua_pop(L, 1); /* pop environment for default files */ + lua_getfield(L, -1, "popen"); + newfenv(L, io_pclose); /* create environment for 'popen' */ + lua_setfenv(L, -2); /* set fenv for 'popen' */ + lua_pop(L, 1); /* pop 'popen' */ + return 1; +} + diff --git a/src/lua/llex.c b/src/lua/llex.c new file mode 100644 index 0000000..2d2b019 --- /dev/null +++ b/src/lua/llex.c @@ -0,0 +1,464 @@ +/* +** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $ +** Lexical Analyzer +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define llex_c +#define LUA_CORE + +#include "lua.h" + +#include "ldo.h" +#include "llex.h" +#include "lobject.h" +#include "lparser.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "lzio.h" + + + +#define next(ls) (ls->current = zgetc(ls->z)) + + + + +#define currIsNewline(ls) (ls->current == '\n' || ls->current == '\r') + + +/* ORDER RESERVED */ +const char *const luaX_tokens [] = { + "and", "break", "do", "else", "elseif", + "end", "false", "for", "function", "if", + "in", "local", "nil", "not", "or", "repeat", + "return", "then", "true", "until", "while", + "..", "...", "==", ">=", "<=", "~=", + "", "", "", "", + NULL +}; + + +#define save_and_next(ls) (save(ls, ls->current), next(ls)) + + +static void save (LexState *ls, int c) { + Mbuffer *b = ls->buff; + if (b->n + 1 > b->buffsize) { + size_t newsize; + if (b->buffsize >= MAX_SIZET/2) + luaX_lexerror(ls, "lexical element too long", 0); + newsize = b->buffsize * 2; + luaZ_resizebuffer(ls->L, b, newsize); + } + b->buffer[b->n++] = cast(char, c); +} + + +void luaX_init (lua_State *L) { + int i; + for (i=0; itsv.reserved = cast_byte(i+1); /* reserved word */ + } +} + + +#define MAXSRC 80 + + +const char *luaX_token2str (LexState *ls, int token) { + if (token < FIRST_RESERVED) { + lua_assert(token == cast(unsigned char, token)); + return (iscntrl(token)) ? luaO_pushfstring(ls->L, "char(%d)", token) : + luaO_pushfstring(ls->L, "%c", token); + } + else + return luaX_tokens[token-FIRST_RESERVED]; +} + + +static const char *txtToken (LexState *ls, int token) { + switch (token) { + case TK_NAME: + case TK_STRING: + case TK_NUMBER: + save(ls, '\0'); + return luaZ_buffer(ls->buff); + default: + return luaX_token2str(ls, token); + } +} + + +void luaX_lexerror (LexState *ls, const char *msg, int token) { + char buff[MAXSRC]; + luaO_chunkid(buff, getstr(ls->source), MAXSRC); + msg = luaO_pushfstring(ls->L, "%s:%d: %s", buff, ls->linenumber, msg); + if (token) + luaO_pushfstring(ls->L, "%s near " LUA_QS, msg, txtToken(ls, token)); + luaD_throw(ls->L, LUA_ERRSYNTAX); +} + + +void luaX_syntaxerror (LexState *ls, const char *msg) { + luaX_lexerror(ls, msg, ls->t.token); +} + + +TString *luaX_newstring (LexState *ls, const char *str, size_t l) { + lua_State *L = ls->L; + TString *ts = luaS_newlstr(L, str, l); + TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */ + if (ttisnil(o)) { + setbvalue(o, 1); /* make sure `str' will not be collected */ + luaC_checkGC(L); + } + return ts; +} + + +static void inclinenumber (LexState *ls) { + int old = ls->current; + lua_assert(currIsNewline(ls)); + next(ls); /* skip `\n' or `\r' */ + if (currIsNewline(ls) && ls->current != old) + next(ls); /* skip `\n\r' or `\r\n' */ + if (++ls->linenumber >= MAX_INT) + luaX_syntaxerror(ls, "chunk has too many lines"); +} + + +void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, TString *source) { + ls->decpoint = '.'; + ls->L = L; + ls->lookahead.token = TK_EOS; /* no look-ahead token */ + ls->z = z; + ls->fs = NULL; + ls->linenumber = 1; + ls->lastline = 1; + ls->source = source; + luaZ_resizebuffer(ls->L, ls->buff, LUA_MINBUFFER); /* initialize buffer */ + next(ls); /* read first char */ +} + + + +/* +** ======================================================= +** LEXICAL ANALYZER +** ======================================================= +*/ + + + +static int check_next (LexState *ls, const char *set) { + if (!strchr(set, ls->current)) + return 0; + save_and_next(ls); + return 1; +} + + +static void buffreplace (LexState *ls, char from, char to) { + size_t n = luaZ_bufflen(ls->buff); + char *p = luaZ_buffer(ls->buff); + while (n--) + if (p[n] == from) p[n] = to; +} + + +static void trydecpoint (LexState *ls, SemInfo *seminfo) { + /* format error: try to update decimal point separator */ + struct lconv *cv = localeconv(); + char old = ls->decpoint; + ls->decpoint = (cv ? cv->decimal_point[0] : '.'); + buffreplace(ls, old, ls->decpoint); /* try updated decimal separator */ + if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) { + /* format error with correct decimal point: no more options */ + buffreplace(ls, ls->decpoint, '.'); /* undo change (for error message) */ + luaX_lexerror(ls, "malformed number", TK_NUMBER); + } +} + + +/* LUA_NUMBER */ +static void read_numeral (LexState *ls, SemInfo *seminfo) { + lua_assert(isdigit(ls->current)); + do { + save_and_next(ls); + } while (isdigit(ls->current) || ls->current == '.'); + if (check_next(ls, "Ee")) /* `E'? */ + check_next(ls, "+-"); /* optional exponent sign */ + while (isalnum(ls->current) || ls->current == '_') + save_and_next(ls); + save(ls, '\0'); + buffreplace(ls, '.', ls->decpoint); /* follow locale for decimal point */ + if (!luaO_str2d(luaZ_buffer(ls->buff), &seminfo->r)) /* format error? */ + trydecpoint(ls, seminfo); /* try to update decimal point separator */ +} + + +static int skip_sep (LexState *ls) { + int count = 0; + int s = ls->current; + lua_assert(s == '[' || s == ']'); + save_and_next(ls); + while (ls->current == '=') { + save_and_next(ls); + count++; + } + return (ls->current == s) ? count : (-count) - 1; +} + + +static void read_long_string (LexState *ls, SemInfo *seminfo, int sep) { + int cont = 0; + (void)(cont); /* avoid warnings when `cont' is not used */ + save_and_next(ls); /* skip 2nd `[' */ + if (currIsNewline(ls)) /* string starts with a newline? */ + inclinenumber(ls); /* skip it */ + for (;;) { + switch (ls->current) { + case EOZ: + luaX_lexerror(ls, (seminfo) ? "unfinished long string" : + "unfinished long comment", TK_EOS); + break; /* to avoid warnings */ +#if defined(LUA_COMPAT_LSTR) + case '[': { + if (skip_sep(ls) == sep) { + save_and_next(ls); /* skip 2nd `[' */ + cont++; +#if LUA_COMPAT_LSTR == 1 + if (sep == 0) + luaX_lexerror(ls, "nesting of [[...]] is deprecated", '['); +#endif + } + break; + } +#endif + case ']': { + if (skip_sep(ls) == sep) { + save_and_next(ls); /* skip 2nd `]' */ +#if defined(LUA_COMPAT_LSTR) && LUA_COMPAT_LSTR == 2 + cont--; + if (sep == 0 && cont >= 0) break; +#endif + goto endloop; + } + break; + } + case '\n': + case '\r': { + save(ls, '\n'); + inclinenumber(ls); + if (!seminfo) luaZ_resetbuffer(ls->buff); /* avoid wasting space */ + break; + } + default: { + if (seminfo) save_and_next(ls); + else next(ls); + } + } + } endloop: + if (seminfo) + seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + (2 + sep), + luaZ_bufflen(ls->buff) - 2*(2 + sep)); +} + + +static void read_string (LexState *ls, int del, SemInfo *seminfo) { + save_and_next(ls); + while (ls->current != del) { + switch (ls->current) { + case EOZ: + luaX_lexerror(ls, "unfinished string", TK_EOS); + continue; /* to avoid warnings */ + case '\n': + case '\r': + luaX_lexerror(ls, "unfinished string", TK_STRING); + continue; /* to avoid warnings */ + case '\\': { + int c; + next(ls); /* do not save the `\' */ + switch (ls->current) { + case 'a': c = '\a'; break; + case 'b': c = '\b'; break; + case 'f': c = '\f'; break; + case 'n': c = '\n'; break; + case 'r': c = '\r'; break; + case 't': c = '\t'; break; + case 'v': c = '\v'; break; + case '\n': /* go through */ + case '\r': save(ls, '\n'); inclinenumber(ls); continue; + case EOZ: continue; /* will raise an error next loop */ + default: { + if (!isdigit(ls->current)) + save_and_next(ls); /* handles \\, \", \', and \? */ + else { /* \xxx */ + int i = 0; + c = 0; + do { + c = 10*c + (ls->current-'0'); + next(ls); + } while (++i<3 && isdigit(ls->current)); + if (c > UCHAR_MAX) + luaX_lexerror(ls, "escape sequence too large", TK_STRING); + save(ls, c); + } + continue; + } + } + save(ls, c); + next(ls); + continue; + } + default: + save_and_next(ls); + } + } + save_and_next(ls); /* skip delimiter */ + seminfo->ts = luaX_newstring(ls, luaZ_buffer(ls->buff) + 1, + luaZ_bufflen(ls->buff) - 2); +} + + +static int llex (LexState *ls, SemInfo *seminfo) { + luaZ_resetbuffer(ls->buff); + for (;;) { + switch (ls->current) { + case '\n': + case '\r': { + inclinenumber(ls); + continue; + } + case '-': { + next(ls); + if (ls->current != '-') return '-'; + /* else is a comment */ + next(ls); + if (ls->current == '[') { + int sep = skip_sep(ls); + luaZ_resetbuffer(ls->buff); /* `skip_sep' may dirty the buffer */ + if (sep >= 0) { + read_long_string(ls, NULL, sep); /* long comment */ + luaZ_resetbuffer(ls->buff); + continue; + } + } + /* else short comment */ + while (!currIsNewline(ls) && ls->current != EOZ) + next(ls); + continue; + } + case '[': { + int sep = skip_sep(ls); + if (sep >= 0) { + read_long_string(ls, seminfo, sep); + return TK_STRING; + } + else if (sep == -1) return '['; + else luaX_lexerror(ls, "invalid long string delimiter", TK_STRING); + return 0; /* never reached but will silence the FALLTHRU compiler warning */ + } + case '=': { + next(ls); + if (ls->current != '=') return '='; + else { next(ls); return TK_EQ; } + } + case '<': { + next(ls); + if (ls->current != '=') return '<'; + else { next(ls); return TK_LE; } + } + case '>': { + next(ls); + if (ls->current != '=') return '>'; + else { next(ls); return TK_GE; } + } + case '~': { + next(ls); + if (ls->current != '=') return '~'; + else { next(ls); return TK_NE; } + } + case '"': + case '\'': { + read_string(ls, ls->current, seminfo); + return TK_STRING; + } + case '.': { + save_and_next(ls); + if (check_next(ls, ".")) { + if (check_next(ls, ".")) + return TK_DOTS; /* ... */ + else return TK_CONCAT; /* .. */ + } + else if (!isdigit(ls->current)) return '.'; + else { + read_numeral(ls, seminfo); + return TK_NUMBER; + } + } + case EOZ: { + return TK_EOS; + } + default: { + if (isspace(ls->current)) { + lua_assert(!currIsNewline(ls)); + next(ls); + continue; + } + else if (isdigit(ls->current)) { + read_numeral(ls, seminfo); + return TK_NUMBER; + } + else if (isalpha(ls->current) || ls->current == '_') { + /* identifier or reserved word */ + TString *ts; + do { + save_and_next(ls); + } while (isalnum(ls->current) || ls->current == '_'); + ts = luaX_newstring(ls, luaZ_buffer(ls->buff), + luaZ_bufflen(ls->buff)); + if (ts->tsv.reserved > 0) /* reserved word? */ + return ts->tsv.reserved - 1 + FIRST_RESERVED; + else { + seminfo->ts = ts; + return TK_NAME; + } + } + else { + int c = ls->current; + next(ls); + return c; /* single-char tokens (+ - / ...) */ + } + } + } + } +} + + +void luaX_next (LexState *ls) { + ls->lastline = ls->linenumber; + if (ls->lookahead.token != TK_EOS) { /* is there a look-ahead token? */ + ls->t = ls->lookahead; /* use this one */ + ls->lookahead.token = TK_EOS; /* and discharge it */ + } + else + ls->t.token = llex(ls, &ls->t.seminfo); /* read next token */ +} + + +void luaX_lookahead (LexState *ls) { + lua_assert(ls->lookahead.token == TK_EOS); + ls->lookahead.token = llex(ls, &ls->lookahead.seminfo); +} + diff --git a/src/lua/llex.h b/src/lua/llex.h new file mode 100644 index 0000000..a9201ce --- /dev/null +++ b/src/lua/llex.h @@ -0,0 +1,81 @@ +/* +** $Id: llex.h,v 1.58.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lexical Analyzer +** See Copyright Notice in lua.h +*/ + +#ifndef llex_h +#define llex_h + +#include "lobject.h" +#include "lzio.h" + + +#define FIRST_RESERVED 257 + +/* maximum length of a reserved word */ +#define TOKEN_LEN (sizeof("function")/sizeof(char)) + + +/* +* WARNING: if you change the order of this enumeration, +* grep "ORDER RESERVED" +*/ +enum RESERVED { + /* terminal symbols denoted by reserved words */ + TK_AND = FIRST_RESERVED, TK_BREAK, + TK_DO, TK_ELSE, TK_ELSEIF, TK_END, TK_FALSE, TK_FOR, TK_FUNCTION, + TK_IF, TK_IN, TK_LOCAL, TK_NIL, TK_NOT, TK_OR, TK_REPEAT, + TK_RETURN, TK_THEN, TK_TRUE, TK_UNTIL, TK_WHILE, + /* other terminal symbols */ + TK_CONCAT, TK_DOTS, TK_EQ, TK_GE, TK_LE, TK_NE, TK_NUMBER, + TK_NAME, TK_STRING, TK_EOS +}; + +/* number of reserved words */ +#define NUM_RESERVED (cast(int, TK_WHILE-FIRST_RESERVED+1)) + + +/* array with token `names' */ +LUAI_DATA const char *const luaX_tokens []; + + +typedef union { + lua_Number r; + TString *ts; +} SemInfo; /* semantics information */ + + +typedef struct Token { + int token; + SemInfo seminfo; +} Token; + + +typedef struct LexState { + int current; /* current character (charint) */ + int linenumber; /* input line counter */ + int lastline; /* line of last token `consumed' */ + Token t; /* current token */ + Token lookahead; /* look ahead token */ + struct FuncState *fs; /* `FuncState' is private to the parser */ + struct lua_State *L; + ZIO *z; /* input stream */ + Mbuffer *buff; /* buffer for tokens */ + TString *source; /* current source name */ + char decpoint; /* locale decimal point */ +} LexState; + + +LUAI_FUNC void luaX_init (lua_State *L); +LUAI_FUNC void luaX_setinput (lua_State *L, LexState *ls, ZIO *z, + TString *source); +LUAI_FUNC TString *luaX_newstring (LexState *ls, const char *str, size_t l); +LUAI_FUNC void luaX_next (LexState *ls); +LUAI_FUNC void luaX_lookahead (LexState *ls); +LUAI_FUNC void luaX_lexerror (LexState *ls, const char *msg, int token); +LUAI_FUNC void luaX_syntaxerror (LexState *ls, const char *s); +LUAI_FUNC const char *luaX_token2str (LexState *ls, int token); + + +#endif diff --git a/src/lua/llimits.h b/src/lua/llimits.h new file mode 100644 index 0000000..ca8dcb7 --- /dev/null +++ b/src/lua/llimits.h @@ -0,0 +1,128 @@ +/* +** $Id: llimits.h,v 1.69.1.1 2007/12/27 13:02:25 roberto Exp $ +** Limits, basic types, and some other `installation-dependent' definitions +** See Copyright Notice in lua.h +*/ + +#ifndef llimits_h +#define llimits_h + + +#include +#include + + +#include "lua.h" + + +typedef LUAI_UINT32 lu_int32; + +typedef LUAI_UMEM lu_mem; + +typedef LUAI_MEM l_mem; + + + +/* chars used as small naturals (so that `char' is reserved for characters) */ +typedef unsigned char lu_byte; + + +#define MAX_SIZET ((size_t)(~(size_t)0)-2) + +#define MAX_LUMEM ((lu_mem)(~(lu_mem)0)-2) + + +#define MAX_INT (INT_MAX-2) /* maximum value of an int (-2 for safety) */ + +/* +** conversion of pointer to integer +** this is for hashing only; there is no problem if the integer +** cannot hold the whole pointer value +*/ +#define IntPoint(p) ((unsigned int)(lu_mem)(p)) + + + +/* type to ensure maximum alignment */ +typedef LUAI_USER_ALIGNMENT_T L_Umaxalign; + + +/* result of a `usual argument conversion' over lua_Number */ +typedef LUAI_UACNUMBER l_uacNumber; + + +/* internal assertions for in-house debugging */ +#ifdef lua_assert + +#define check_exp(c,e) (lua_assert(c), (e)) +#define api_check(l,e) lua_assert(e) + +#else + +#define lua_assert(c) ((void)0) +#define check_exp(c,e) (e) +#define api_check luai_apicheck + +#endif + + +#ifndef UNUSED +#define UNUSED(x) ((void)(x)) /* to avoid warnings */ +#endif + + +#ifndef cast +#define cast(t, exp) ((t)(exp)) +#endif + +#define cast_byte(i) cast(lu_byte, (i)) +#define cast_num(i) cast(lua_Number, (i)) +#define cast_int(i) cast(int, (i)) + + + +/* +** type for virtual-machine instructions +** must be an unsigned with (at least) 4 bytes (see details in lopcodes.h) +*/ +typedef lu_int32 Instruction; + + + +/* maximum stack for a Lua function */ +#define MAXSTACK 250 + + + +/* minimum size for the string table (must be power of 2) */ +#ifndef MINSTRTABSIZE +#define MINSTRTABSIZE 32 +#endif + + +/* minimum size for string buffer */ +#ifndef LUA_MINBUFFER +#define LUA_MINBUFFER 32 +#endif + + +#ifndef lua_lock +#define lua_lock(L) ((void) 0) +#define lua_unlock(L) ((void) 0) +#endif + +#ifndef luai_threadyield +#define luai_threadyield(L) {lua_unlock(L); lua_lock(L);} +#endif + + +/* +** macro to control inclusion of some hard tests on stack reallocation +*/ +#ifndef HARDSTACKTESTS +#define condhardstacktests(x) ((void)0) +#else +#define condhardstacktests(x) x +#endif + +#endif diff --git a/src/lua/lmathlib.c b/src/lua/lmathlib.c new file mode 100644 index 0000000..0ff1526 --- /dev/null +++ b/src/lua/lmathlib.c @@ -0,0 +1,277 @@ +/* +** $Id: lmathlib.c,v 1.67.1.1 2007/12/27 13:02:25 roberto Exp $ +** Standard mathematical library +** See Copyright Notice in lua.h +*/ + + +#include +#include + +#define lmathlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +#undef PI +#define PI (3.14159265358979323846) +#define RADIANS_PER_DEGREE (PI/180.0) + + + +static int math_abs (lua_State *L) { + lua_pushnumber(L, fabs(luaL_checknumber(L, 1))); + return 1; +} + +static int math_sin (lua_State *L) { + lua_pushnumber(L, sin(luaL_checknumber(L, 1))); + return 1; +} + +static int math_sinh (lua_State *L) { + lua_pushnumber(L, sinh(luaL_checknumber(L, 1))); + return 1; +} + +static int math_cos (lua_State *L) { + lua_pushnumber(L, cos(luaL_checknumber(L, 1))); + return 1; +} + +static int math_cosh (lua_State *L) { + lua_pushnumber(L, cosh(luaL_checknumber(L, 1))); + return 1; +} + +static int math_tan (lua_State *L) { + lua_pushnumber(L, tan(luaL_checknumber(L, 1))); + return 1; +} + +static int math_tanh (lua_State *L) { + lua_pushnumber(L, tanh(luaL_checknumber(L, 1))); + return 1; +} + +static int math_asin (lua_State *L) { + lua_pushnumber(L, asin(luaL_checknumber(L, 1))); + return 1; +} + +static int math_acos (lua_State *L) { + lua_pushnumber(L, acos(luaL_checknumber(L, 1))); + return 1; +} + +static int math_atan (lua_State *L) { + lua_pushnumber(L, atan(luaL_checknumber(L, 1))); + return 1; +} + +static int math_atan2 (lua_State *L) { + lua_pushnumber(L, atan2(luaL_checknumber(L, 1), luaL_checknumber(L, 2))); + return 1; +} + +static int math_ceil (lua_State *L) { + lua_pushnumber(L, ceil(luaL_checknumber(L, 1))); + return 1; +} + +static int math_floor (lua_State *L) { + lua_pushnumber(L, floor(luaL_checknumber(L, 1))); + return 1; +} + +static int math_fmod (lua_State *L) { + lua_pushnumber(L, fmod(luaL_checknumber(L, 1), luaL_checknumber(L, 2))); + return 1; +} + +static int math_modf (lua_State *L) { + double ip; + double fp = modf(luaL_checknumber(L, 1), &ip); + lua_pushnumber(L, ip); + lua_pushnumber(L, fp); + return 2; +} + +static int math_sqrt (lua_State *L) { + lua_pushnumber(L, sqrt(luaL_checknumber(L, 1))); + return 1; +} + +static int math_pow (lua_State *L) { + lua_pushnumber(L, pow(luaL_checknumber(L, 1), luaL_checknumber(L, 2))); + return 1; +} + +static int math_log (lua_State *L) { + lua_pushnumber(L, log(luaL_checknumber(L, 1))); + return 1; +} + +static int math_log10 (lua_State *L) { + lua_pushnumber(L, log10(luaL_checknumber(L, 1))); + return 1; +} + +static int math_exp (lua_State *L) { + lua_pushnumber(L, exp(luaL_checknumber(L, 1))); + return 1; +} + +static int math_deg (lua_State *L) { + lua_pushnumber(L, luaL_checknumber(L, 1)/RADIANS_PER_DEGREE); + return 1; +} + +static int math_rad (lua_State *L) { + lua_pushnumber(L, luaL_checknumber(L, 1)*RADIANS_PER_DEGREE); + return 1; +} + +static int math_frexp (lua_State *L) { + int e; + lua_pushnumber(L, frexp(luaL_checknumber(L, 1), &e)); + lua_pushinteger(L, e); + return 2; +} + +static int math_ldexp (lua_State *L) { + lua_pushnumber(L, ldexp(luaL_checknumber(L, 1), luaL_checkint(L, 2))); + return 1; +} + + + +static int math_min (lua_State *L) { + int n = lua_gettop(L); /* number of arguments */ + lua_Number dmin = luaL_checknumber(L, 1); + int i; + for (i=2; i<=n; i++) { + lua_Number d = luaL_checknumber(L, i); + if (d < dmin) + dmin = d; + } + lua_pushnumber(L, dmin); + return 1; +} + + +static int math_max (lua_State *L) { + int n = lua_gettop(L); /* number of arguments */ + lua_Number dmax = luaL_checknumber(L, 1); + int i; + for (i=2; i<=n; i++) { + lua_Number d = luaL_checknumber(L, i); + if (d > dmax) + dmax = d; + } + lua_pushnumber(L, dmax); + return 1; +} + + +static int math_random (lua_State *L) { + /* the `%' avoids the (rare) case of r==1, and is needed also because on + some systems (SunOS!) `rand()' may return a value larger than RAND_MAX */ + lua_Number r = (lua_Number)(rand()%RAND_MAX) / (lua_Number)RAND_MAX; + switch (lua_gettop(L)) { /* check number of arguments */ + case 0: { /* no arguments */ + lua_pushnumber(L, r); /* Number between 0 and 1 */ + break; + } + case 1: { /* only upper limit */ + int u = luaL_checkint(L, 1); + luaL_argcheck(L, 1<=u, 1, "interval is empty"); + lua_pushnumber(L, floor(r*u)+1); /* int between 1 and `u' */ + break; + } + case 2: { /* lower and upper limits */ + int l = luaL_checkint(L, 1); + int u = luaL_checkint(L, 2); + luaL_argcheck(L, l<=u, 2, "interval is empty"); + lua_pushnumber(L, floor(r*(u-l+1))+l); /* int between `l' and `u' */ + break; + } + default: return luaL_error(L, "wrong number of arguments"); + } + return 1; +} + + +static int math_randomseed (lua_State *L) { + srand(luaL_checkint(L, 1)); + return 0; +} + + +static int math_erf (lua_State *L) { + lua_pushnumber(L, erf(luaL_checknumber(L, 1))); + return 1; +} + + +static int math_erfc (lua_State *L) { + lua_pushnumber(L, erfc(luaL_checknumber(L, 1))); + return 1; +} + + +static const luaL_Reg mathlib[] = { + {"abs", math_abs}, + {"acos", math_acos}, + {"asin", math_asin}, + {"atan2", math_atan2}, + {"atan", math_atan}, + {"ceil", math_ceil}, + {"cosh", math_cosh}, + {"cos", math_cos}, + {"deg", math_deg}, + {"erf", math_erf}, + {"erfc", math_erfc}, + {"exp", math_exp}, + {"floor", math_floor}, + {"fmod", math_fmod}, + {"frexp", math_frexp}, + {"ldexp", math_ldexp}, + {"log10", math_log10}, + {"log", math_log}, + {"max", math_max}, + {"min", math_min}, + {"modf", math_modf}, + {"pow", math_pow}, + {"rad", math_rad}, + {"random", math_random}, + {"randomseed", math_randomseed}, + {"sinh", math_sinh}, + {"sin", math_sin}, + {"sqrt", math_sqrt}, + {"tanh", math_tanh}, + {"tan", math_tan}, + {NULL, NULL} +}; + + +/* +** Open math library +*/ +LUALIB_API int luaopen_math (lua_State *L) { + luaL_register(L, LUA_MATHLIBNAME, mathlib); + lua_pushnumber(L, PI); + lua_setfield(L, -2, "pi"); + lua_pushnumber(L, HUGE_VAL); + lua_setfield(L, -2, "huge"); +#if defined(LUA_COMPAT_MOD) + lua_getfield(L, -1, "fmod"); + lua_setfield(L, -2, "mod"); +#endif + return 1; +} + diff --git a/src/lua/lmem.c b/src/lua/lmem.c new file mode 100644 index 0000000..ae7d8c9 --- /dev/null +++ b/src/lua/lmem.c @@ -0,0 +1,86 @@ +/* +** $Id: lmem.c,v 1.70.1.1 2007/12/27 13:02:25 roberto Exp $ +** Interface to Memory Manager +** See Copyright Notice in lua.h +*/ + + +#include + +#define lmem_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" + + + +/* +** About the realloc function: +** void * frealloc (void *ud, void *ptr, size_t osize, size_t nsize); +** (`osize' is the old size, `nsize' is the new size) +** +** Lua ensures that (ptr == NULL) iff (osize == 0). +** +** * frealloc(ud, NULL, 0, x) creates a new block of size `x' +** +** * frealloc(ud, p, x, 0) frees the block `p' +** (in this specific case, frealloc must return NULL). +** particularly, frealloc(ud, NULL, 0, 0) does nothing +** (which is equivalent to free(NULL) in ANSI C) +** +** frealloc returns NULL if it cannot create or reallocate the area +** (any reallocation to an equal or smaller size cannot fail!) +*/ + + + +#define MINSIZEARRAY 4 + + +void *luaM_growaux_ (lua_State *L, void *block, int *size, size_t size_elems, + int limit, const char *errormsg) { + void *newblock; + int newsize; + if (*size >= limit/2) { /* cannot double it? */ + if (*size >= limit) /* cannot grow even a little? */ + luaG_runerror(L, errormsg); + newsize = limit; /* still have at least one free place */ + } + else { + newsize = (*size)*2; + if (newsize < MINSIZEARRAY) + newsize = MINSIZEARRAY; /* minimum size */ + } + newblock = luaM_reallocv(L, block, *size, newsize, size_elems); + *size = newsize; /* update only when everything else is OK */ + return newblock; +} + + +void *luaM_toobig (lua_State *L) { + luaG_runerror(L, "memory allocation error: block too big"); + return NULL; /* to avoid warnings */ +} + + + +/* +** generic allocation routine. +*/ +void *luaM_realloc_ (lua_State *L, void *block, size_t osize, size_t nsize) { + global_State *g = G(L); + lua_assert((osize == 0) == (block == NULL)); + block = (*g->frealloc)(g->ud, block, osize, nsize); + if (block == NULL && nsize > 0) + luaD_throw(L, LUA_ERRMEM); + lua_assert((nsize == 0) == (block == NULL)); + g->totalbytes = (g->totalbytes - osize) + nsize; + return block; +} + diff --git a/src/lua/lmem.h b/src/lua/lmem.h new file mode 100644 index 0000000..7c2dcb3 --- /dev/null +++ b/src/lua/lmem.h @@ -0,0 +1,49 @@ +/* +** $Id: lmem.h,v 1.31.1.1 2007/12/27 13:02:25 roberto Exp $ +** Interface to Memory Manager +** See Copyright Notice in lua.h +*/ + +#ifndef lmem_h +#define lmem_h + + +#include + +#include "llimits.h" +#include "lua.h" + +#define MEMERRMSG "not enough memory" + + +#define luaM_reallocv(L,b,on,n,e) \ + ((cast(size_t, (n)+1) <= MAX_SIZET/(e)) ? /* +1 to avoid warnings */ \ + luaM_realloc_(L, (b), (on)*(e), (n)*(e)) : \ + luaM_toobig(L)) + +#define luaM_freemem(L, b, s) luaM_realloc_(L, (b), (s), 0) +#define luaM_free(L, b) luaM_realloc_(L, (b), sizeof(*(b)), 0) +#define luaM_freearray(L, b, n, t) luaM_reallocv(L, (b), n, 0, sizeof(t)) + +#define luaM_malloc(L,t) luaM_realloc_(L, NULL, 0, (t)) +#define luaM_new(L,t) cast(t *, luaM_malloc(L, sizeof(t))) +#define luaM_newvector(L,n,t) \ + cast(t *, luaM_reallocv(L, NULL, 0, n, sizeof(t))) + +#define luaM_growvector(L,v,nelems,size,t,limit,e) \ + if ((nelems)+1 > (size)) \ + ((v)=cast(t *, luaM_growaux_(L,v,&(size),sizeof(t),limit,e))) + +#define luaM_reallocvector(L, v,oldn,n,t) \ + ((v)=cast(t *, luaM_reallocv(L, v, oldn, n, sizeof(t)))) + + +LUAI_FUNC void *luaM_realloc_ (lua_State *L, void *block, size_t oldsize, + size_t size); +LUAI_FUNC void *luaM_toobig (lua_State *L); +LUAI_FUNC void *luaM_growaux_ (lua_State *L, void *block, int *size, + size_t size_elem, int limit, + const char *errormsg); + +#endif + diff --git a/src/lua/loadlib.c b/src/lua/loadlib.c new file mode 100644 index 0000000..43697e1 --- /dev/null +++ b/src/lua/loadlib.c @@ -0,0 +1,686 @@ +/* +** $Id: loadlib.c,v 1.52.1.4 2009/09/09 13:17:16 roberto Exp $ +** Dynamic library loader for Lua +** See Copyright Notice in lua.h +** +** This module contains an implementation of loadlib for Unix systems +** that have dlfcn, an implementation for Darwin (Mac OS X), an +** implementation for Windows, and a stub for other systems. +*/ + + +#include +#include +#include + + +#define loadlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +/* prefix for open functions in C libraries */ +#define LUA_POF "luaopen_" + +/* separator for open functions in C libraries */ +#define LUA_OFSEP "_" + + +#define LIBPREFIX "LOADLIB: " + +#define POF LUA_POF +#define LIB_FAIL "open" + + +/* error codes for ll_loadfunc */ +#define ERRLIB 1 +#define ERRFUNC 2 + + +#define LSB_CONFIG "lsb_config" + +static void ll_unloadlib (void *lib); +static void *ll_load (lua_State *L, const char *path); +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym); + + + +#if defined(LUA_DL_DLOPEN) +/* +** {======================================================================== +** This is an implementation of loadlib based on the dlfcn interface. +** The dlfcn interface is available in Linux, SunOS, Solaris, IRIX, FreeBSD, +** NetBSD, AIX 4.2, HPUX 11, and probably most other Unix flavors, at least +** as an emulation layer on top of native functions. +** ========================================================================= +*/ + +#include + +static void ll_unloadlib (void *lib) { + dlclose(lib); +} + + +static void *ll_load (lua_State *L, const char *path) { + void *lib = dlopen(path, RTLD_NOW); + if (lib == NULL) lua_pushstring(L, dlerror()); + return lib; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + lua_CFunction f; + *(void **)(&f)= dlsym(lib, sym); + if (f == NULL) lua_pushstring(L, dlerror()); + return f; +} + +/* }====================================================== */ + + + +#elif defined(LUA_DL_DLL) +/* +** {====================================================================== +** This is an implementation of loadlib for Windows using native functions. +** ======================================================================= +*/ + +#include + +static void pusherror (lua_State *L) { + int error = GetLastError(); + char buffer[128]; + if (FormatMessageA(FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_FROM_SYSTEM, + NULL, error, 0, buffer, sizeof(buffer), NULL)) + lua_pushstring(L, buffer); + else + lua_pushfstring(L, "system error %d\n", error); +} + +static void ll_unloadlib (void *lib) { + FreeLibrary((HINSTANCE)lib); +} + + +static void *ll_load (lua_State *L, const char *path) { + HINSTANCE lib = LoadLibraryA(path); + if (lib == NULL) pusherror(L); + return lib; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + lua_CFunction f = (lua_CFunction)GetProcAddress((HINSTANCE)lib, sym); + if (f == NULL) pusherror(L); + return f; +} + +/* }====================================================== */ + + + +#elif defined(LUA_DL_DYLD) +/* +** {====================================================================== +** Native Mac OS X / Darwin Implementation +** ======================================================================= +*/ + +#include + + +/* Mac appends a `_' before C function names */ +#undef POF +#define POF "_" LUA_POF + + +static void pusherror (lua_State *L) { + const char *err_str; + const char *err_file; + NSLinkEditErrors err; + int err_num; + NSLinkEditError(&err, &err_num, &err_file, &err_str); + lua_pushstring(L, err_str); +} + + +static const char *errorfromcode (NSObjectFileImageReturnCode ret) { + switch (ret) { + case NSObjectFileImageInappropriateFile: + return "file is not a bundle"; + case NSObjectFileImageArch: + return "library is for wrong CPU type"; + case NSObjectFileImageFormat: + return "bad format"; + case NSObjectFileImageAccess: + return "cannot access file"; + case NSObjectFileImageFailure: + default: + return "unable to load library"; + } +} + + +static void ll_unloadlib (void *lib) { + NSUnLinkModule((NSModule)lib, NSUNLINKMODULE_OPTION_RESET_LAZY_REFERENCES); +} + + +static void *ll_load (lua_State *L, const char *path) { + NSObjectFileImage img; + NSObjectFileImageReturnCode ret; + /* this would be a rare case, but prevents crashing if it happens */ + if(!_dyld_present()) { + lua_pushliteral(L, "dyld not present"); + return NULL; + } + ret = NSCreateObjectFileImageFromFile(path, &img); + if (ret == NSObjectFileImageSuccess) { + NSModule mod = NSLinkModule(img, path, NSLINKMODULE_OPTION_PRIVATE | + NSLINKMODULE_OPTION_RETURN_ON_ERROR); + NSDestroyObjectFileImage(img); + if (mod == NULL) pusherror(L); + return mod; + } + lua_pushstring(L, errorfromcode(ret)); + return NULL; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + NSSymbol nss = NSLookupSymbolInModule((NSModule)lib, sym); + if (nss == NULL) { + lua_pushfstring(L, "symbol " LUA_QS " not found", sym); + return NULL; + } + return (lua_CFunction)NSAddressOfSymbol(nss); +} + +/* }====================================================== */ + + + +#else +/* +** {====================================================== +** Fallback for other systems +** ======================================================= +*/ + +#undef LIB_FAIL +#define LIB_FAIL "absent" + + +#define DLMSG "dynamic libraries not enabled; check your Lua installation" + + +static void ll_unloadlib (void *lib) { + (void)lib; /* to avoid warnings */ +} + + +static void *ll_load (lua_State *L, const char *path) { + (void)path; /* to avoid warnings */ + lua_pushliteral(L, DLMSG); + return NULL; +} + + +static lua_CFunction ll_sym (lua_State *L, void *lib, const char *sym) { + (void)lib; (void)sym; /* to avoid warnings */ + lua_pushliteral(L, DLMSG); + return NULL; +} + +/* }====================================================== */ +#endif + + + +static void **ll_register (lua_State *L, const char *path) { + void **plib; + lua_pushfstring(L, "%s%s", LIBPREFIX, path); + lua_gettable(L, LUA_REGISTRYINDEX); /* check library in registry? */ + if (!lua_isnil(L, -1)) /* is there an entry? */ + plib = (void **)lua_touserdata(L, -1); + else { /* no entry yet; create one */ + lua_pop(L, 1); + plib = (void **)lua_newuserdata(L, sizeof(const void *)); + *plib = NULL; + luaL_getmetatable(L, "_LOADLIB"); + lua_setmetatable(L, -2); + lua_pushfstring(L, "%s%s", LIBPREFIX, path); + lua_pushvalue(L, -2); + lua_settable(L, LUA_REGISTRYINDEX); + } + return plib; +} + + +/* +** __gc tag method: calls library's `ll_unloadlib' function with the lib +** handle +*/ +static int gctm (lua_State *L) { + void **lib = (void **)luaL_checkudata(L, 1, "_LOADLIB"); + if (*lib) ll_unloadlib(*lib); + *lib = NULL; /* mark library as closed */ + return 0; +} + + +static int ll_loadfunc (lua_State *L, const char *path, const char *sym) { + void **reg = ll_register(L, path); + if (*reg == NULL) *reg = ll_load(L, path); + if (*reg == NULL) + return ERRLIB; /* unable to load library */ + else { + lua_CFunction f = ll_sym(L, *reg, sym); + if (f == NULL) + return ERRFUNC; /* unable to find function */ + lua_pushcfunction(L, f); + return 0; /* return function */ + } +} + + +/* +** {====================================================== +** 'require' function +** ======================================================= +*/ + + +static int readable (const char *filename) { + FILE *f = fopen(filename, "r"); /* try to open file */ + if (f == NULL) return 0; /* open failed */ + fclose(f); + return 1; +} + + +static const char *pushnexttemplate (lua_State *L, const char *path) { + const char *l; + while (*path == *LUA_PATHSEP) path++; /* skip separators */ + if (*path == '\0') return NULL; /* no more templates */ + l = strchr(path, *LUA_PATHSEP); /* find next separator */ + if (l == NULL) l = path + strlen(path); + lua_pushlstring(L, path, l - path); /* template */ + return l; +} + + +static const char *findfile (lua_State *L, const char *name, + const char *pname) { + const char *path; + int i = 0; + while (name[i]) { + if (!isalnum(name[i]) && name[i] != '_' && name[i] != '.') { + luaL_error(L, "invalid module name '%s'", name); + return NULL; // never reached, just silences the compiler + } + ++i; + } + lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); + if (lua_type(L, -1) != LUA_TTABLE) { + return NULL; + } + name = luaL_gsub(L, name, ".", LUA_DIRSEP); + lua_getfield(L, -2, pname); + path = lua_tostring(L, -1); + if (path == NULL) + luaL_error(L, "no '%s' configuration was specified for the sandbox; " + "external modules have been disabled", pname); + lua_pushliteral(L, ""); /* error accumulator */ + while ((path = pushnexttemplate(L, path)) != NULL) { + const char *filename; + filename = luaL_gsub(L, lua_tostring(L, -1), LUA_PATH_MARK, name); + lua_remove(L, -2); /* remove path template */ + if (readable(filename)) /* does file exist and is readable? */ + return filename; /* return that file name */ + lua_pushfstring(L, "\n\tno file " LUA_QS, filename); + lua_remove(L, -2); /* remove file name */ + lua_concat(L, 2); /* add entry to possible error message */ + } + return NULL; /* not found */ +} + + +static void loaderror (lua_State *L, const char *filename) { + luaL_error(L, "error loading module " LUA_QS " from file " LUA_QS ":\n\t%s", + lua_tostring(L, 1), filename, lua_tostring(L, -1)); +} + + +static int loader_Lua (lua_State *L) { + const char *filename; + const char *name = luaL_checkstring(L, 1); + filename = findfile(L, name, "path"); + if (filename == NULL) return 1; /* library not found in this path */ + if (luaL_loadfile(L, filename) != 0) + loaderror(L, filename); + return 1; /* library loaded successfully */ +} + + +static const char *mkfuncname (lua_State *L, const char *modname) { + const char *funcname; + const char *mark = strchr(modname, *LUA_IGMARK); + if (mark) modname = mark + 1; + funcname = luaL_gsub(L, modname, ".", LUA_OFSEP); + funcname = lua_pushfstring(L, POF"%s", funcname); + lua_remove(L, -2); /* remove 'gsub' result */ + return funcname; +} + + +static int loader_C (lua_State *L) { + const char *funcname; + const char *name = luaL_checkstring(L, 1); + const char *filename = findfile(L, name, "cpath"); + if (filename == NULL) return 1; /* library not found in this path */ + funcname = mkfuncname(L, name); + if (ll_loadfunc(L, filename, funcname) != 0) + loaderror(L, filename); + return 1; /* library loaded successfully */ +} + + +static int loader_Croot (lua_State *L) { + const char *funcname; + const char *filename; + const char *name = luaL_checkstring(L, 1); + const char *p = strchr(name, '.'); + int stat; + if (p == NULL) return 0; /* is root */ + lua_pushlstring(L, name, p - name); + filename = findfile(L, lua_tostring(L, -1), "cpath"); + if (filename == NULL) return 1; /* root not found */ + funcname = mkfuncname(L, name); + if ((stat = ll_loadfunc(L, filename, funcname)) != 0) { + if (stat != ERRFUNC) loaderror(L, filename); /* real error */ + lua_pushfstring(L, "\n\tno module " LUA_QS " in file " LUA_QS, + name, filename); + return 1; /* function not found */ + } + return 1; +} + + +static int loader_preload (lua_State *L) { + const char *name = luaL_checkstring(L, 1); + lua_getfield(L, LUA_ENVIRONINDEX, "preload"); + if (!lua_istable(L, -1)) + luaL_error(L, LUA_QL("preload") " must be a table"); + lua_getfield(L, -1, name); + return 1; +} + +// If necessary, add an empty metatable to flag the table as non-data +// during preservation. +static void add_empty_metatable(lua_State* L) +{ + if (lua_getmetatable(L, -1) == 0) { + lua_newtable(L); + lua_setmetatable(L, -2); + } else { + lua_pop(L, 1); + } +} + + +static void remove_entries(lua_State* L, const char* name) +{ + lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 1); + return; + } + lua_getfield(L, -1, "remove_entries"); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 2); + return; + } + lua_getfield(L, -1, name); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 3); + return; + } + int n = 1; + while (1) { + lua_rawgeti(L, -1, n); + if (lua_type(L, -1) == LUA_TNIL) { + lua_pop(L, 1); + break; + } + lua_pushnil(L); + lua_settable(L, -6); + ++n; + } + lua_pop(L, 3); +} + + +static int is_disabled(lua_State* L, const char* name) +{ + int status = 0; + lua_getfield(L, LUA_REGISTRYINDEX, LSB_CONFIG); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 1); + return status; + } + lua_getfield(L, -1, "disable_modules"); + if (lua_type(L, -1) != LUA_TTABLE) { + lua_pop(L, 2); + return status; + } + lua_getfield(L, -1, name); + if (lua_type(L, -1) != LUA_TNIL) { + status = 1; + } + lua_pop(L, 3); + return status; +} + + +static const int sentinel_ = 0; +#define sentinel ((void *)&sentinel_) + + +static int ll_require (lua_State *L) { + const char *name = luaL_checkstring(L, 1); + if (is_disabled(L, name)) { + luaL_error(L, "module " LUA_QS " disabled", name); + } + int i; + lua_settop(L, 1); /* _LOADED table will be at index 2 */ + lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); + lua_getfield(L, 2, name); + if (lua_toboolean(L, -1)) { /* is it there? */ + if (lua_touserdata(L, -1) == sentinel) /* check loops */ + luaL_error(L, "loop or previous error loading module " LUA_QS, name); + return 1; /* package is already loaded */ + } + /* else must load it; iterate over available loaders */ + lua_getfield(L, LUA_ENVIRONINDEX, "loaders"); + if (!lua_istable(L, -1)) + luaL_error(L, LUA_QL("loaders") " must be a table"); + lua_pushliteral(L, ""); /* error message accumulator */ + for (i=1; ; i++) { + lua_rawgeti(L, -2, i); /* get a loader */ + if (lua_isnil(L, -1)) + luaL_error(L, "module " LUA_QS " not found:%s", + name, lua_tostring(L, -2)); + lua_pushstring(L, name); + lua_call(L, 1, 1); /* call it */ + if (lua_isfunction(L, -1)) /* did it find module? */ + break; /* module loaded successfully */ + else if (lua_isstring(L, -1)) /* loader returned error message? */ + lua_concat(L, 2); /* accumulate it */ + else + lua_pop(L, 1); + } + lua_pushlightuserdata(L, sentinel); + lua_setfield(L, 2, name); /* _LOADED[name] = sentinel */ + lua_pushstring(L, name); /* pass name as argument to module */ + lua_call(L, 1, 1); /* run loaded module */ + if (!lua_isnil(L, -1)) /* non-nil return? */ + lua_setfield(L, 2, name); /* _LOADED[name] = returned value */ + lua_getfield(L, 2, name); + if (lua_touserdata(L, -1) == sentinel) { /* module did not set a value? */ + lua_pushboolean(L, 1); /* use true as result */ + lua_pushvalue(L, -1); /* extra copy to be returned */ + lua_setfield(L, 2, name); /* _LOADED[name] = true */ + } + add_empty_metatable(L); + remove_entries(L, name); + return 1; +} + +/* }====================================================== */ + + + +/* +** {====================================================== +** 'module' function +** ======================================================= +*/ + + +static void setfenv (lua_State *L) { + lua_Debug ar; + if (lua_getstack(L, 1, &ar) == 0 || + lua_getinfo(L, "f", &ar) == 0 || /* get calling function */ + lua_iscfunction(L, -1)) + luaL_error(L, LUA_QL("module") " not called from a Lua function"); + lua_pushvalue(L, -2); + lua_setfenv(L, -2); + lua_pop(L, 1); +} + + +static void dooptions (lua_State *L, int n) { + int i; + for (i = 2; i <= n; i++) { + lua_pushvalue(L, i); /* get option (a function) */ + lua_pushvalue(L, -2); /* module */ + lua_call(L, 1, 0); + } +} + + +static void modinit (lua_State *L, const char *modname) { + const char *dot; + lua_pushvalue(L, -1); + lua_setfield(L, -2, "_M"); /* module._M = module */ + lua_pushstring(L, modname); + lua_setfield(L, -2, "_NAME"); + dot = strrchr(modname, '.'); /* look for last dot in module name */ + if (dot == NULL) dot = modname; + else dot++; + /* set _PACKAGE as package name (full module name minus last part) */ + lua_pushlstring(L, modname, dot - modname); + lua_setfield(L, -2, "_PACKAGE"); +} + + +static int ll_module (lua_State *L) { + const char *modname = luaL_checkstring(L, 1); + int loaded = lua_gettop(L) + 1; /* index of _LOADED table */ + lua_getfield(L, LUA_REGISTRYINDEX, "_LOADED"); + lua_getfield(L, loaded, modname); /* get _LOADED[modname] */ + if (!lua_istable(L, -1)) { /* not found? */ + lua_pop(L, 1); /* remove previous result */ + /* try global variable (and create one if it does not exist) */ + if (luaL_findtable(L, LUA_GLOBALSINDEX, modname, 1) != NULL) + return luaL_error(L, "name conflict for module " LUA_QS, modname); + lua_pushvalue(L, -1); + lua_setfield(L, loaded, modname); /* _LOADED[modname] = new table */ + } + /* check whether table already has a _NAME field */ + lua_getfield(L, -1, "_NAME"); + if (!lua_isnil(L, -1)) /* is table an initialized module? */ + lua_pop(L, 1); + else { /* no; initialize it */ + lua_pop(L, 1); + modinit(L, modname); + } + lua_pushvalue(L, -1); + setfenv(L); + dooptions(L, loaded - 1); + return 0; +} + + +static int ll_seeall (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + if (!lua_getmetatable(L, 1)) { + lua_createtable(L, 0, 1); /* create new metatable */ + lua_pushvalue(L, -1); + lua_setmetatable(L, 1); + } + lua_pushvalue(L, LUA_GLOBALSINDEX); + lua_setfield(L, -2, "__index"); /* mt.__index = _G */ + return 0; +} + + +/* }====================================================== */ + + + +static const luaL_Reg pk_funcs[] = { + {"seeall", ll_seeall}, + {NULL, NULL} +}; + + +static const luaL_Reg ll_funcs[] = { + {"module", ll_module}, + {"require", ll_require}, + {NULL, NULL} +}; + + +static const lua_CFunction loaders[] = + {loader_preload, loader_Lua, loader_C, loader_Croot, NULL}; + + +LUALIB_API int luaopen_package (lua_State *L) { + int i; + /* create new type _LOADLIB */ + luaL_newmetatable(L, "_LOADLIB"); + lua_pushcfunction(L, gctm); + lua_setfield(L, -2, "__gc"); + /* create `package' environment table (not exposed to Lua) */ + lua_newtable(L); + lua_pushvalue(L, -1); + lua_replace(L, LUA_ENVIRONINDEX); + /* create `loaders' table */ + lua_createtable(L, sizeof(loaders)/sizeof(loaders[0]) - 1, 0); + /* fill it with pre-defined loaders */ + for (i=0; loaders[i] != NULL; i++) { + lua_pushcfunction(L, loaders[i]); + lua_rawseti(L, -2, i+1); + } + lua_setfield(L, -2, "loaders"); /* put it in field `loaders' */ + + /* set field `preload' */ + luaL_findtable(L, LUA_REGISTRYINDEX, "_PRELOADED", 0); + lua_setfield(L, -2, "preload"); + + /* open lib into global table */ + lua_pushvalue(L, LUA_GLOBALSINDEX); + luaL_register(L, NULL, ll_funcs); + lua_pop(L, 1); + + luaL_register(L, LUA_LOADLIBNAME, pk_funcs); + return 1; /* return 'package' table */ +} diff --git a/src/lua/lobject.c b/src/lua/lobject.c new file mode 100644 index 0000000..4ff5073 --- /dev/null +++ b/src/lua/lobject.c @@ -0,0 +1,214 @@ +/* +** $Id: lobject.c,v 2.22.1.1 2007/12/27 13:02:25 roberto Exp $ +** Some generic functions over Lua objects +** See Copyright Notice in lua.h +*/ + +#include +#include +#include +#include +#include + +#define lobject_c +#define LUA_CORE + +#include "lua.h" + +#include "ldo.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "lvm.h" + + + +const TValue luaO_nilobject_ = {{NULL}, LUA_TNIL}; + + +/* +** converts an integer to a "floating point byte", represented as +** (eeeeexxx), where the real value is (1xxx) * 2^(eeeee - 1) if +** eeeee != 0 and (xxx) otherwise. +*/ +int luaO_int2fb (unsigned int x) { + int e = 0; /* expoent */ + while (x >= 16) { + x = (x+1) >> 1; + e++; + } + if (x < 8) return x; + else return ((e+1) << 3) | (cast_int(x) - 8); +} + + +/* converts back */ +int luaO_fb2int (int x) { + int e = (x >> 3) & 31; + if (e == 0) return x; + else return ((x & 7)+8) << (e - 1); +} + + +int luaO_log2 (unsigned int x) { + static const lu_byte log_2[256] = { + 0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 + }; + int l = -1; + while (x >= 256) { l += 8; x >>= 8; } + return l + log_2[x]; + +} + + +int luaO_rawequalObj (const TValue *t1, const TValue *t2) { + if (ttype(t1) != ttype(t2)) return 0; + else switch (ttype(t1)) { + case LUA_TNIL: + return 1; + case LUA_TNUMBER: + return luai_numeq(nvalue(t1), nvalue(t2)); + case LUA_TBOOLEAN: + return bvalue(t1) == bvalue(t2); /* boolean true must be 1 !! */ + case LUA_TLIGHTUSERDATA: + return pvalue(t1) == pvalue(t2); + default: + lua_assert(iscollectable(t1)); + return gcvalue(t1) == gcvalue(t2); + } +} + + +int luaO_str2d (const char *s, lua_Number *result) { + char *endptr; + *result = lua_str2number(s, &endptr); + if (endptr == s) return 0; /* conversion failed */ + if (*endptr == 'x' || *endptr == 'X') /* maybe an hexadecimal constant? */ + *result = cast_num(strtoul(s, &endptr, 16)); + if (*endptr == '\0') return 1; /* most common case */ + while (isspace(cast(unsigned char, *endptr))) endptr++; + if (*endptr != '\0') return 0; /* invalid trailing characters? */ + return 1; +} + + + +static void pushstr (lua_State *L, const char *str) { + setsvalue2s(L, L->top, luaS_new(L, str)); + incr_top(L); +} + + +/* this function handles only `%d', `%c', %f, %p, and `%s' formats */ +const char *luaO_pushvfstring (lua_State *L, const char *fmt, va_list argp) { + int n = 1; + pushstr(L, ""); + for (;;) { + const char *e = strchr(fmt, '%'); + if (e == NULL) break; + setsvalue2s(L, L->top, luaS_newlstr(L, fmt, e-fmt)); + incr_top(L); + switch (*(e+1)) { + case 's': { + const char *s = va_arg(argp, char *); + if (s == NULL) s = "(null)"; + pushstr(L, s); + break; + } + case 'c': { + char buff[2]; + buff[0] = cast(char, va_arg(argp, int)); + buff[1] = '\0'; + pushstr(L, buff); + break; + } + case 'd': { + setnvalue(L->top, cast_num(va_arg(argp, int))); + incr_top(L); + break; + } + case 'f': { + setnvalue(L->top, cast_num(va_arg(argp, l_uacNumber))); + incr_top(L); + break; + } + case 'p': { + char buff[4*sizeof(void *) + 8]; /* should be enough space for a `%p' */ + sprintf(buff, "%p", va_arg(argp, void *)); + pushstr(L, buff); + break; + } + case '%': { + pushstr(L, "%"); + break; + } + default: { + char buff[3]; + buff[0] = '%'; + buff[1] = *(e+1); + buff[2] = '\0'; + pushstr(L, buff); + break; + } + } + n += 2; + fmt = e+2; + } + pushstr(L, fmt); + luaV_concat(L, n+1, cast_int(L->top - L->base) - 1); + L->top -= n; + return svalue(L->top - 1); +} + + +const char *luaO_pushfstring (lua_State *L, const char *fmt, ...) { + const char *msg; + va_list argp; + va_start(argp, fmt); + msg = luaO_pushvfstring(L, fmt, argp); + va_end(argp); + return msg; +} + + +void luaO_chunkid (char *out, const char *source, size_t bufflen) { + if (*source == '=') { + strncpy(out, source+1, bufflen); /* remove first char */ + out[bufflen-1] = '\0'; /* ensures null termination */ + } + else { /* out = "source", or "...source" */ + if (*source == '@') { + size_t l; + source++; /* skip the `@' */ + bufflen -= sizeof(" '...' "); + l = strlen(source); + strcpy(out, ""); + if (l > bufflen) { + source += (l-bufflen); /* get last part of file name */ + strcat(out, "..."); + } + strcat(out, source); + } + else { /* out = [string "string"] */ + size_t len = strcspn(source, "\n\r"); /* stop at first newline */ + bufflen -= sizeof(" [string \"...\"] "); + if (len > bufflen) len = bufflen; + strcpy(out, "[string \""); + if (source[len] != '\0') { /* must truncate? */ + strncat(out, source, len); + strcat(out, "..."); + } + else + strcat(out, source); + strcat(out, "\"]"); + } + } +} diff --git a/src/lua/lobject.h b/src/lua/lobject.h new file mode 100644 index 0000000..f1e447e --- /dev/null +++ b/src/lua/lobject.h @@ -0,0 +1,381 @@ +/* +** $Id: lobject.h,v 2.20.1.2 2008/08/06 13:29:48 roberto Exp $ +** Type definitions for Lua objects +** See Copyright Notice in lua.h +*/ + + +#ifndef lobject_h +#define lobject_h + + +#include + + +#include "llimits.h" +#include "lua.h" + + +/* tags for values visible from Lua */ +#define LAST_TAG LUA_TTHREAD + +#define NUM_TAGS (LAST_TAG+1) + + +/* +** Extra tags for non-values +*/ +#define LUA_TPROTO (LAST_TAG+1) +#define LUA_TUPVAL (LAST_TAG+2) +#define LUA_TDEADKEY (LAST_TAG+3) + + +/* +** Union of all collectable objects +*/ +typedef union GCObject GCObject; + + +/* +** Common Header for all collectable objects (in macro form, to be +** included in other objects) +*/ +#define CommonHeader GCObject *next; lu_byte tt; lu_byte marked + + +/* +** Common header in struct form +*/ +typedef struct GCheader { + CommonHeader; +} GCheader; + + + + +/* +** Union of all Lua values +*/ +typedef union { + GCObject *gc; + void *p; + lua_Number n; + int b; +} Value; + + +/* +** Tagged Values +*/ + +#define TValuefields Value value; int tt + +typedef struct lua_TValue { + TValuefields; +} TValue; + + +/* Macros to test type */ +#define ttisnil(o) (ttype(o) == LUA_TNIL) +#define ttisnumber(o) (ttype(o) == LUA_TNUMBER) +#define ttisstring(o) (ttype(o) == LUA_TSTRING) +#define ttistable(o) (ttype(o) == LUA_TTABLE) +#define ttisfunction(o) (ttype(o) == LUA_TFUNCTION) +#define ttisboolean(o) (ttype(o) == LUA_TBOOLEAN) +#define ttisuserdata(o) (ttype(o) == LUA_TUSERDATA) +#define ttisthread(o) (ttype(o) == LUA_TTHREAD) +#define ttislightuserdata(o) (ttype(o) == LUA_TLIGHTUSERDATA) + +/* Macros to access values */ +#define ttype(o) ((o)->tt) +#define gcvalue(o) check_exp(iscollectable(o), (o)->value.gc) +#define pvalue(o) check_exp(ttislightuserdata(o), (o)->value.p) +#define nvalue(o) check_exp(ttisnumber(o), (o)->value.n) +#define rawtsvalue(o) check_exp(ttisstring(o), &(o)->value.gc->ts) +#define tsvalue(o) (&rawtsvalue(o)->tsv) +#define rawuvalue(o) check_exp(ttisuserdata(o), &(o)->value.gc->u) +#define uvalue(o) (&rawuvalue(o)->uv) +#define clvalue(o) check_exp(ttisfunction(o), &(o)->value.gc->cl) +#define hvalue(o) check_exp(ttistable(o), &(o)->value.gc->h) +#define bvalue(o) check_exp(ttisboolean(o), (o)->value.b) +#define thvalue(o) check_exp(ttisthread(o), &(o)->value.gc->th) + +#define l_isfalse(o) (ttisnil(o) || (ttisboolean(o) && bvalue(o) == 0)) + +/* +** for internal debug only +*/ +#define checkconsistency(obj) \ + lua_assert(!iscollectable(obj) || (ttype(obj) == (obj)->value.gc->gch.tt)) + +#define checkliveness(g,obj) \ + lua_assert(!iscollectable(obj) || \ + ((ttype(obj) == (obj)->value.gc->gch.tt) && !isdead(g, (obj)->value.gc))) + + +/* Macros to set values */ +#define setnilvalue(obj) ((obj)->tt=LUA_TNIL) + +#define setnvalue(obj,x) \ + { TValue *i_o=(obj); i_o->value.n=(x); i_o->tt=LUA_TNUMBER; } + +#define setpvalue(obj,x) \ + { TValue *i_o=(obj); i_o->value.p=(x); i_o->tt=LUA_TLIGHTUSERDATA; } + +#define setbvalue(obj,x) \ + { TValue *i_o=(obj); i_o->value.b=(x); i_o->tt=LUA_TBOOLEAN; } + +#define setsvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TSTRING; \ + checkliveness(G(L),i_o); } + +#define setuvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TUSERDATA; \ + checkliveness(G(L),i_o); } + +#define setthvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TTHREAD; \ + checkliveness(G(L),i_o); } + +#define setclvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TFUNCTION; \ + checkliveness(G(L),i_o); } + +#define sethvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TTABLE; \ + checkliveness(G(L),i_o); } + +#define setptvalue(L,obj,x) \ + { TValue *i_o=(obj); \ + i_o->value.gc=cast(GCObject *, (x)); i_o->tt=LUA_TPROTO; \ + checkliveness(G(L),i_o); } + + + + +#define setobj(L,obj1,obj2) \ + { const TValue *o2=(obj2); TValue *o1=(obj1); \ + o1->value = o2->value; o1->tt=o2->tt; \ + checkliveness(G(L),o1); } + + +/* +** different types of sets, according to destination +*/ + +/* from stack to (same) stack */ +#define setobjs2s setobj +/* to stack (not from same stack) */ +#define setobj2s setobj +#define setsvalue2s setsvalue +#define sethvalue2s sethvalue +#define setptvalue2s setptvalue +/* from table to same table */ +#define setobjt2t setobj +/* to table */ +#define setobj2t setobj +/* to new object */ +#define setobj2n setobj +#define setsvalue2n setsvalue + +#define setttype(obj, tt) (ttype(obj) = (tt)) + + +#define iscollectable(o) (ttype(o) >= LUA_TSTRING) + + + +typedef TValue *StkId; /* index to stack elements */ + + +/* +** String headers for string table +*/ +typedef union TString { + L_Umaxalign dummy; /* ensures maximum alignment for strings */ + struct { + CommonHeader; + lu_byte reserved; + unsigned int hash; + size_t len; + } tsv; +} TString; + + +#define getstr(ts) cast(const char *, (ts) + 1) +#define svalue(o) getstr(rawtsvalue(o)) + + + +typedef union Udata { + L_Umaxalign dummy; /* ensures maximum alignment for `local' udata */ + struct { + CommonHeader; + struct Table *metatable; + struct Table *env; + size_t len; + } uv; +} Udata; + + + + +/* +** Function Prototypes +*/ +typedef struct Proto { + CommonHeader; + TValue *k; /* constants used by the function */ + Instruction *code; + struct Proto **p; /* functions defined inside the function */ + int *lineinfo; /* map from opcodes to source lines */ + struct LocVar *locvars; /* information about local variables */ + TString **upvalues; /* upvalue names */ + TString *source; + int sizeupvalues; + int sizek; /* size of `k' */ + int sizecode; + int sizelineinfo; + int sizep; /* size of `p' */ + int sizelocvars; + int linedefined; + int lastlinedefined; + GCObject *gclist; + lu_byte nups; /* number of upvalues */ + lu_byte numparams; + lu_byte is_vararg; + lu_byte maxstacksize; +} Proto; + + +/* masks for new-style vararg */ +#define VARARG_HASARG 1 +#define VARARG_ISVARARG 2 +#define VARARG_NEEDSARG 4 + + +typedef struct LocVar { + TString *varname; + int startpc; /* first point where variable is active */ + int endpc; /* first point where variable is dead */ +} LocVar; + + + +/* +** Upvalues +*/ + +typedef struct UpVal { + CommonHeader; + TValue *v; /* points to stack or to its own value */ + union { + TValue value; /* the value (when closed) */ + struct { /* double linked list (when open) */ + struct UpVal *prev; + struct UpVal *next; + } l; + } u; +} UpVal; + + +/* +** Closures +*/ + +#define ClosureHeader \ + CommonHeader; lu_byte isC; lu_byte nupvalues; GCObject *gclist; \ + struct Table *env + +typedef struct CClosure { + ClosureHeader; + lua_CFunction f; + TValue upvalue[1]; +} CClosure; + + +typedef struct LClosure { + ClosureHeader; + struct Proto *p; + UpVal *upvals[1]; +} LClosure; + + +typedef union Closure { + CClosure c; + LClosure l; +} Closure; + + +#define iscfunction(o) (ttype(o) == LUA_TFUNCTION && clvalue(o)->c.isC) +#define isLfunction(o) (ttype(o) == LUA_TFUNCTION && !clvalue(o)->c.isC) + + +/* +** Tables +*/ + +typedef union TKey { + struct { + TValuefields; + struct Node *next; /* for chaining */ + } nk; + TValue tvk; +} TKey; + + +typedef struct Node { + TValue i_val; + TKey i_key; +} Node; + + +typedef struct Table { + CommonHeader; + lu_byte flags; /* 1<

lsizenode)) + + +#define luaO_nilobject (&luaO_nilobject_) + +LUAI_DATA const TValue luaO_nilobject_; + +#define ceillog2(x) (luaO_log2((x)-1) + 1) + +LUAI_FUNC int luaO_log2 (unsigned int x); +LUAI_FUNC int luaO_int2fb (unsigned int x); +LUAI_FUNC int luaO_fb2int (int x); +LUAI_FUNC int luaO_rawequalObj (const TValue *t1, const TValue *t2); +LUAI_FUNC int luaO_str2d (const char *s, lua_Number *result); +LUAI_FUNC const char *luaO_pushvfstring (lua_State *L, const char *fmt, + va_list argp); +LUAI_FUNC const char *luaO_pushfstring (lua_State *L, const char *fmt, ...); +LUAI_FUNC void luaO_chunkid (char *out, const char *source, size_t len); + + +#endif + diff --git a/src/lua/lopcodes.c b/src/lua/lopcodes.c new file mode 100644 index 0000000..4cc7452 --- /dev/null +++ b/src/lua/lopcodes.c @@ -0,0 +1,102 @@ +/* +** $Id: lopcodes.c,v 1.37.1.1 2007/12/27 13:02:25 roberto Exp $ +** See Copyright Notice in lua.h +*/ + + +#define lopcodes_c +#define LUA_CORE + + +#include "lopcodes.h" + + +/* ORDER OP */ + +const char *const luaP_opnames[NUM_OPCODES+1] = { + "MOVE", + "LOADK", + "LOADBOOL", + "LOADNIL", + "GETUPVAL", + "GETGLOBAL", + "GETTABLE", + "SETGLOBAL", + "SETUPVAL", + "SETTABLE", + "NEWTABLE", + "SELF", + "ADD", + "SUB", + "MUL", + "DIV", + "MOD", + "POW", + "UNM", + "NOT", + "LEN", + "CONCAT", + "JMP", + "EQ", + "LT", + "LE", + "TEST", + "TESTSET", + "CALL", + "TAILCALL", + "RETURN", + "FORLOOP", + "FORPREP", + "TFORLOOP", + "SETLIST", + "CLOSE", + "CLOSURE", + "VARARG", + NULL +}; + + +#define opmode(t,a,b,c,m) (((t)<<7) | ((a)<<6) | ((b)<<4) | ((c)<<2) | (m)) + +const lu_byte luaP_opmodes[NUM_OPCODES] = { +/* T A B C mode opcode */ + opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_MOVE */ + ,opmode(0, 1, OpArgK, OpArgN, iABx) /* OP_LOADK */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_LOADBOOL */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_LOADNIL */ + ,opmode(0, 1, OpArgU, OpArgN, iABC) /* OP_GETUPVAL */ + ,opmode(0, 1, OpArgK, OpArgN, iABx) /* OP_GETGLOBAL */ + ,opmode(0, 1, OpArgR, OpArgK, iABC) /* OP_GETTABLE */ + ,opmode(0, 0, OpArgK, OpArgN, iABx) /* OP_SETGLOBAL */ + ,opmode(0, 0, OpArgU, OpArgN, iABC) /* OP_SETUPVAL */ + ,opmode(0, 0, OpArgK, OpArgK, iABC) /* OP_SETTABLE */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_NEWTABLE */ + ,opmode(0, 1, OpArgR, OpArgK, iABC) /* OP_SELF */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_ADD */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_SUB */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_MUL */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_DIV */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_MOD */ + ,opmode(0, 1, OpArgK, OpArgK, iABC) /* OP_POW */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_UNM */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_NOT */ + ,opmode(0, 1, OpArgR, OpArgN, iABC) /* OP_LEN */ + ,opmode(0, 1, OpArgR, OpArgR, iABC) /* OP_CONCAT */ + ,opmode(0, 0, OpArgR, OpArgN, iAsBx) /* OP_JMP */ + ,opmode(1, 0, OpArgK, OpArgK, iABC) /* OP_EQ */ + ,opmode(1, 0, OpArgK, OpArgK, iABC) /* OP_LT */ + ,opmode(1, 0, OpArgK, OpArgK, iABC) /* OP_LE */ + ,opmode(1, 1, OpArgR, OpArgU, iABC) /* OP_TEST */ + ,opmode(1, 1, OpArgR, OpArgU, iABC) /* OP_TESTSET */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_CALL */ + ,opmode(0, 1, OpArgU, OpArgU, iABC) /* OP_TAILCALL */ + ,opmode(0, 0, OpArgU, OpArgN, iABC) /* OP_RETURN */ + ,opmode(0, 1, OpArgR, OpArgN, iAsBx) /* OP_FORLOOP */ + ,opmode(0, 1, OpArgR, OpArgN, iAsBx) /* OP_FORPREP */ + ,opmode(1, 0, OpArgN, OpArgU, iABC) /* OP_TFORLOOP */ + ,opmode(0, 0, OpArgU, OpArgU, iABC) /* OP_SETLIST */ + ,opmode(0, 0, OpArgN, OpArgN, iABC) /* OP_CLOSE */ + ,opmode(0, 1, OpArgU, OpArgN, iABx) /* OP_CLOSURE */ + ,opmode(0, 1, OpArgU, OpArgN, iABC) /* OP_VARARG */ +}; + diff --git a/src/lua/lopcodes.h b/src/lua/lopcodes.h new file mode 100644 index 0000000..41224d6 --- /dev/null +++ b/src/lua/lopcodes.h @@ -0,0 +1,268 @@ +/* +** $Id: lopcodes.h,v 1.125.1.1 2007/12/27 13:02:25 roberto Exp $ +** Opcodes for Lua virtual machine +** See Copyright Notice in lua.h +*/ + +#ifndef lopcodes_h +#define lopcodes_h + +#include "llimits.h" + + +/*=========================================================================== + We assume that instructions are unsigned numbers. + All instructions have an opcode in the first 6 bits. + Instructions can have the following fields: + `A' : 8 bits + `B' : 9 bits + `C' : 9 bits + `Bx' : 18 bits (`B' and `C' together) + `sBx' : signed Bx + + A signed argument is represented in excess K; that is, the number + value is the unsigned value minus K. K is exactly the maximum value + for that argument (so that -max is represented by 0, and +max is + represented by 2*max), which is half the maximum for the corresponding + unsigned argument. +===========================================================================*/ + + +enum OpMode {iABC, iABx, iAsBx}; /* basic instruction format */ + + +/* +** size and position of opcode arguments. +*/ +#define SIZE_C 9 +#define SIZE_B 9 +#define SIZE_Bx (SIZE_C + SIZE_B) +#define SIZE_A 8 + +#define SIZE_OP 6 + +#define POS_OP 0 +#define POS_A (POS_OP + SIZE_OP) +#define POS_C (POS_A + SIZE_A) +#define POS_B (POS_C + SIZE_C) +#define POS_Bx POS_C + + +/* +** limits for opcode arguments. +** we use (signed) int to manipulate most arguments, +** so they must fit in LUAI_BITSINT-1 bits (-1 for sign) +*/ +#if SIZE_Bx < LUAI_BITSINT-1 +#define MAXARG_Bx ((1<>1) /* `sBx' is signed */ +#else +#define MAXARG_Bx MAX_INT +#define MAXARG_sBx MAX_INT +#endif + + +#define MAXARG_A ((1<>POS_OP) & MASK1(SIZE_OP,0))) +#define SET_OPCODE(i,o) ((i) = (((i)&MASK0(SIZE_OP,POS_OP)) | \ + ((cast(Instruction, o)<>POS_A) & MASK1(SIZE_A,0))) +#define SETARG_A(i,u) ((i) = (((i)&MASK0(SIZE_A,POS_A)) | \ + ((cast(Instruction, u)<>POS_B) & MASK1(SIZE_B,0))) +#define SETARG_B(i,b) ((i) = (((i)&MASK0(SIZE_B,POS_B)) | \ + ((cast(Instruction, b)<>POS_C) & MASK1(SIZE_C,0))) +#define SETARG_C(i,b) ((i) = (((i)&MASK0(SIZE_C,POS_C)) | \ + ((cast(Instruction, b)<>POS_Bx) & MASK1(SIZE_Bx,0))) +#define SETARG_Bx(i,b) ((i) = (((i)&MASK0(SIZE_Bx,POS_Bx)) | \ + ((cast(Instruction, b)< C) then pc++ */ +OP_TESTSET,/* A B C if (R(B) <=> C) then R(A) := R(B) else pc++ */ + +OP_CALL,/* A B C R(A), ... ,R(A+C-2) := R(A)(R(A+1), ... ,R(A+B-1)) */ +OP_TAILCALL,/* A B C return R(A)(R(A+1), ... ,R(A+B-1)) */ +OP_RETURN,/* A B return R(A), ... ,R(A+B-2) (see note) */ + +OP_FORLOOP,/* A sBx R(A)+=R(A+2); + if R(A) =) R(A)*/ +OP_CLOSURE,/* A Bx R(A) := closure(KPROTO[Bx], R(A), ... ,R(A+n)) */ + +OP_VARARG/* A B R(A), R(A+1), ..., R(A+B-1) = vararg */ +} OpCode; + + +#define NUM_OPCODES (cast(int, OP_VARARG) + 1) + + + +/*=========================================================================== + Notes: + (*) In OP_CALL, if (B == 0) then B = top. C is the number of returns - 1, + and can be 0: OP_CALL then sets `top' to last_result+1, so + next open instruction (OP_CALL, OP_RETURN, OP_SETLIST) may use `top'. + + (*) In OP_VARARG, if (B == 0) then use actual number of varargs and + set top (like in OP_CALL with C == 0). + + (*) In OP_RETURN, if (B == 0) then return up to `top' + + (*) In OP_SETLIST, if (B == 0) then B = `top'; + if (C == 0) then next `instruction' is real C + + (*) For comparisons, A specifies what condition the test should accept + (true or false). + + (*) All `skips' (pc++) assume that next instruction is a jump +===========================================================================*/ + + +/* +** masks for instruction properties. The format is: +** bits 0-1: op mode +** bits 2-3: C arg mode +** bits 4-5: B arg mode +** bit 6: instruction set register A +** bit 7: operator is a test +*/ + +enum OpArgMask { + OpArgN, /* argument is not used */ + OpArgU, /* argument is used */ + OpArgR, /* argument is a register or a jump offset */ + OpArgK /* argument is a constant or register/constant */ +}; + +LUAI_DATA const lu_byte luaP_opmodes[NUM_OPCODES]; + +#define getOpMode(m) (cast(enum OpMode, luaP_opmodes[m] & 3)) +#define getBMode(m) (cast(enum OpArgMask, (luaP_opmodes[m] >> 4) & 3)) +#define getCMode(m) (cast(enum OpArgMask, (luaP_opmodes[m] >> 2) & 3)) +#define testAMode(m) (luaP_opmodes[m] & (1 << 6)) +#define testTMode(m) (luaP_opmodes[m] & (1 << 7)) + + +LUAI_DATA const char *const luaP_opnames[NUM_OPCODES+1]; /* opcode names */ + + +/* number of list items to accumulate before a SETLIST instruction */ +#define LFIELDS_PER_FLUSH 50 + + +#endif diff --git a/src/lua/loslib.c b/src/lua/loslib.c new file mode 100644 index 0000000..b822e2d --- /dev/null +++ b/src/lua/loslib.c @@ -0,0 +1,322 @@ +/* +** $Id: loslib.c,v 1.19.1.3 2008/01/18 16:38:18 roberto Exp $ +** Standard Operating System library +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include +#include + +#define loslib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +static int os_pushresult (lua_State *L, int i, const char *filename) { + int en = errno; /* calls to Lua API may change this value */ + if (i) { + lua_pushboolean(L, 1); + return 1; + } + else { + lua_pushnil(L); + lua_pushfstring(L, "%s: %s", filename, strerror(en)); + lua_pushinteger(L, en); + return 3; + } +} +/* +** list of valid conversion specifiers for the 'strftime' function +*/ +#if !defined(LUA_STRFTIMEOPTIONS) + +#if !defined(LUA_USE_POSIX) +#define LUA_STRFTIMEOPTIONS { "aAbBcdHIjmMpSUwWxXyYzZ%", "" } +#else +#define LUA_STRFTIMEOPTIONS \ + { "aAbBcCdDeFgGhHIjklmMnprRSstTuUVwWxXyYzZ%", "" \ + "", "E", "cCxXyY", \ + "O", "deHImMSuUVwWy" } +#endif + +#endif + + + +/* +** By default, Lua uses tmpnam except when POSIX is available, where it +** uses mkstemp. +*/ +#if defined(LUA_USE_MKSTEMP) +#include +#define LUA_TMPNAMBUFSIZE 32 +#define lua_tmpnam(b,e) { \ + strcpy(b, "/tmp/lua_XXXXXX"); \ + e = mkstemp(b); \ + if (e != -1) close(e); \ + e = (e == -1); } + +#elif !defined(lua_tmpnam) + +#define LUA_TMPNAMBUFSIZE L_tmpnam +#define lua_tmpnam(b,e) { e = (tmpnam(b) == NULL); } + +#endif + + +/* +** By default, Lua uses gmtime/localtime, except when POSIX is available, +** where it uses gmtime_r/localtime_r +*/ +#if defined(LUA_USE_GMTIME_R) + +#define l_gmtime(t,r) gmtime_r(t,r) +#define l_localtime(t,r) localtime_r(t,r) + +#elif !defined(l_gmtime) + +#define l_gmtime(t,r) ((void)r, gmtime(t)) +#define l_localtime(t,r) ((void)r, localtime(t)) + +#endif + + + +static int os_execute (lua_State *L) { + lua_pushinteger(L, system(luaL_optstring(L, 1, NULL))); + return 1; +} + + +static int os_remove (lua_State *L) { + const char *filename = luaL_checkstring(L, 1); + return os_pushresult(L, remove(filename) == 0, filename); +} + + +static int os_rename (lua_State *L) { + const char *fromname = luaL_checkstring(L, 1); + const char *toname = luaL_checkstring(L, 2); + return os_pushresult(L, rename(fromname, toname) == 0, fromname); +} + + +static int os_tmpname (lua_State *L) { + char buff[LUA_TMPNAMBUFSIZE]; + int err; + lua_tmpnam(buff, err); + if (err) + return luaL_error(L, "unable to generate a unique filename"); + lua_pushstring(L, buff); + return 1; +} + + +static int os_getenv (lua_State *L) { + lua_pushstring(L, getenv(luaL_checkstring(L, 1))); /* if NULL push nil */ + return 1; +} + + +static int os_clock (lua_State *L) { + lua_pushnumber(L, ((lua_Number)clock())/(lua_Number)CLOCKS_PER_SEC); + return 1; +} + + +/* +** {====================================================== +** Time/Date operations +** { year=%Y, month=%m, day=%d, hour=%H, min=%M, sec=%S, +** wday=%w+1, yday=%j, isdst=? } +** ======================================================= +*/ + +static void setfield (lua_State *L, const char *key, int value) { + lua_pushinteger(L, value); + lua_setfield(L, -2, key); +} + +static void setboolfield (lua_State *L, const char *key, int value) { + if (value < 0) /* undefined? */ + return; /* does not set field */ + lua_pushboolean(L, value); + lua_setfield(L, -2, key); +} + +static int getboolfield (lua_State *L, const char *key) { + int res; + lua_getfield(L, -1, key); + res = lua_isnil(L, -1) ? -1 : lua_toboolean(L, -1); + lua_pop(L, 1); + return res; +} + + +static int getfield (lua_State *L, const char *key, int d) { + int res; + lua_getfield(L, -1, key); + if (lua_isnumber(L, -1)) + res = (int)lua_tointeger(L, -1); + else { + if (d < 0) + return luaL_error(L, "field " LUA_QS " missing in date table", key); + res = d; + } + lua_pop(L, 1); + return res; +} + + +static const char *checkoption (lua_State *L, const char *conv, char *buff) { + static const char *const options[] = LUA_STRFTIMEOPTIONS; + unsigned int i; + for (i = 0; i < sizeof(options)/sizeof(options[0]); i += 2) { + if (*conv != '\0' && strchr(options[i], *conv) != NULL) { + buff[1] = *conv; + if (*options[i + 1] == '\0') { /* one-char conversion specifier? */ + buff[2] = '\0'; /* end buffer */ + return conv + 1; + } + else if (*(conv + 1) != '\0' && + strchr(options[i + 1], *(conv + 1)) != NULL) { + buff[2] = *(conv + 1); /* valid two-char conversion specifier */ + buff[3] = '\0'; /* end buffer */ + return conv + 2; + } + } + } + luaL_argerror(L, 1, + lua_pushfstring(L, "invalid conversion specifier '%%%s'", conv)); + return conv; /* to avoid warnings */ +} + + +static int os_date (lua_State *L) { + const char *s = luaL_optstring(L, 1, "%c"); + time_t t = luaL_opt(L, (time_t)luaL_checknumber, 2, time(NULL)); + struct tm tmr, *stm; + if (*s == '!') { /* UTC? */ + stm = l_gmtime(&t, &tmr); + s++; /* skip `!' */ + } + else + stm = l_localtime(&t, &tmr); + if (stm == NULL) /* invalid date? */ + lua_pushnil(L); + else if (strcmp(s, "*t") == 0) { + lua_createtable(L, 0, 9); /* 9 = number of fields */ + setfield(L, "sec", stm->tm_sec); + setfield(L, "min", stm->tm_min); + setfield(L, "hour", stm->tm_hour); + setfield(L, "day", stm->tm_mday); + setfield(L, "month", stm->tm_mon+1); + setfield(L, "year", stm->tm_year+1900); + setfield(L, "wday", stm->tm_wday+1); + setfield(L, "yday", stm->tm_yday+1); + setboolfield(L, "isdst", stm->tm_isdst); + } + else { + char cc[4]; + luaL_Buffer b; + cc[0] = '%'; + luaL_buffinit(L, &b); + while (*s) { + if (*s != '%') /* no conversion specifier? */ + luaL_addchar(&b, *s++); + else { + size_t reslen; + char buff[200]; /* should be big enough for any conversion result */ + s = checkoption(L, s + 1, cc); + reslen = strftime(buff, sizeof(buff), cc, stm); + luaL_addlstring(&b, buff, reslen); + } + } + luaL_pushresult(&b); + } + return 1; +} + + +static int os_time (lua_State *L) { + time_t t; + if (lua_isnoneornil(L, 1)) /* called without args? */ + t = time(NULL); /* get current time */ + else { + struct tm ts; + luaL_checktype(L, 1, LUA_TTABLE); + lua_settop(L, 1); /* make sure table is at the top */ + ts.tm_sec = getfield(L, "sec", 0); + ts.tm_min = getfield(L, "min", 0); + ts.tm_hour = getfield(L, "hour", 12); + ts.tm_mday = getfield(L, "day", -1); + ts.tm_mon = getfield(L, "month", -1) - 1; + ts.tm_year = getfield(L, "year", -1) - 1900; + ts.tm_isdst = getboolfield(L, "isdst"); + t = mktime(&ts); + } + if (t == (time_t)(-1)) + lua_pushnil(L); + else + lua_pushnumber(L, (lua_Number)t); + return 1; +} + + +static int os_difftime (lua_State *L) { + lua_pushnumber(L, difftime((time_t)(luaL_checknumber(L, 1)), + (time_t)(luaL_optnumber(L, 2, 0)))); + return 1; +} + +/* }====================================================== */ + + +static int os_setlocale (lua_State *L) { + static const int cat[] = {LC_ALL, LC_COLLATE, LC_CTYPE, LC_MONETARY, + LC_NUMERIC, LC_TIME}; + static const char *const catnames[] = {"all", "collate", "ctype", "monetary", + "numeric", "time", NULL}; + const char *l = luaL_optstring(L, 1, NULL); + int op = luaL_checkoption(L, 2, "all", catnames); + lua_pushstring(L, setlocale(cat[op], l)); + return 1; +} + + +static int os_exit (lua_State *L) { + exit(luaL_optint(L, 1, EXIT_SUCCESS)); +} + +static const luaL_Reg syslib[] = { + {"clock", os_clock}, + {"date", os_date}, + {"difftime", os_difftime}, + {"execute", os_execute}, + {"exit", os_exit}, + {"getenv", os_getenv}, + {"remove", os_remove}, + {"rename", os_rename}, + {"setlocale", os_setlocale}, + {"time", os_time}, + {"tmpname", os_tmpname}, + {NULL, NULL} +}; + +/* }====================================================== */ + + + +LUALIB_API int luaopen_os (lua_State *L) { + luaL_register(L, LUA_OSLIBNAME, syslib); + return 1; +} + diff --git a/src/lua/lparser.c b/src/lua/lparser.c new file mode 100644 index 0000000..dda7488 --- /dev/null +++ b/src/lua/lparser.c @@ -0,0 +1,1339 @@ +/* +** $Id: lparser.c,v 2.42.1.4 2011/10/21 19:31:42 roberto Exp $ +** Lua Parser +** See Copyright Notice in lua.h +*/ + + +#include + +#define lparser_c +#define LUA_CORE + +#include "lua.h" + +#include "lcode.h" +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "llex.h" +#include "lmem.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lparser.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" + + + +#define hasmultret(k) ((k) == VCALL || (k) == VVARARG) + +#define getlocvar(fs, i) ((fs)->f->locvars[(fs)->actvar[i]]) + +#define luaY_checklimit(fs,v,l,m) if ((v)>(l)) errorlimit(fs,l,m) + + +/* +** nodes for block list (list of active blocks) +*/ +typedef struct BlockCnt { + struct BlockCnt *previous; /* chain */ + int breaklist; /* list of jumps out of this loop */ + lu_byte nactvar; /* # active locals outside the breakable structure */ + lu_byte upval; /* true if some variable in the block is an upvalue */ + lu_byte isbreakable; /* true if `block' is a loop */ +} BlockCnt; + + + +/* +** prototypes for recursive non-terminal functions +*/ +static void chunk (LexState *ls); +static void expr (LexState *ls, expdesc *v); + + +static void anchor_token (LexState *ls) { + if (ls->t.token == TK_NAME || ls->t.token == TK_STRING) { + TString *ts = ls->t.seminfo.ts; + luaX_newstring(ls, getstr(ts), ts->tsv.len); + } +} + + +static void error_expected (LexState *ls, int token) { + luaX_syntaxerror(ls, + luaO_pushfstring(ls->L, LUA_QS " expected", luaX_token2str(ls, token))); +} + + +static void errorlimit (FuncState *fs, int limit, const char *what) { + const char *msg = (fs->f->linedefined == 0) ? + luaO_pushfstring(fs->L, "main function has more than %d %s", limit, what) : + luaO_pushfstring(fs->L, "function at line %d has more than %d %s", + fs->f->linedefined, limit, what); + luaX_lexerror(fs->ls, msg, 0); +} + + +static int testnext (LexState *ls, int c) { + if (ls->t.token == c) { + luaX_next(ls); + return 1; + } + else return 0; +} + + +static void check (LexState *ls, int c) { + if (ls->t.token != c) + error_expected(ls, c); +} + +static void checknext (LexState *ls, int c) { + check(ls, c); + luaX_next(ls); +} + + +#define check_condition(ls,c,msg) { if (!(c)) luaX_syntaxerror(ls, msg); } + + + +static void check_match (LexState *ls, int what, int who, int where) { + if (!testnext(ls, what)) { + if (where == ls->linenumber) + error_expected(ls, what); + else { + luaX_syntaxerror(ls, luaO_pushfstring(ls->L, + LUA_QS " expected (to close " LUA_QS " at line %d)", + luaX_token2str(ls, what), luaX_token2str(ls, who), where)); + } + } +} + + +static TString *str_checkname (LexState *ls) { + TString *ts; + check(ls, TK_NAME); + ts = ls->t.seminfo.ts; + luaX_next(ls); + return ts; +} + + +static void init_exp (expdesc *e, expkind k, int i) { + e->f = e->t = NO_JUMP; + e->k = k; + e->u.s.info = i; +} + + +static void codestring (LexState *ls, expdesc *e, TString *s) { + init_exp(e, VK, luaK_stringK(ls->fs, s)); +} + + +static void checkname(LexState *ls, expdesc *e) { + codestring(ls, e, str_checkname(ls)); +} + + +static int registerlocalvar (LexState *ls, TString *varname) { + FuncState *fs = ls->fs; + Proto *f = fs->f; + int oldsize = f->sizelocvars; + luaM_growvector(ls->L, f->locvars, fs->nlocvars, f->sizelocvars, + LocVar, SHRT_MAX, "too many local variables"); + while (oldsize < f->sizelocvars) f->locvars[oldsize++].varname = NULL; + f->locvars[fs->nlocvars].varname = varname; + luaC_objbarrier(ls->L, f, varname); + return fs->nlocvars++; +} + + +#define new_localvarliteral(ls,v,n) \ + new_localvar(ls, luaX_newstring(ls, "" v, (sizeof(v)/sizeof(char))-1), n) + + +static void new_localvar (LexState *ls, TString *name, int n) { + FuncState *fs = ls->fs; + luaY_checklimit(fs, fs->nactvar+n+1, LUAI_MAXVARS, "local variables"); + fs->actvar[fs->nactvar+n] = cast(unsigned short, registerlocalvar(ls, name)); +} + + +static void adjustlocalvars (LexState *ls, int nvars) { + FuncState *fs = ls->fs; + fs->nactvar = cast_byte(fs->nactvar + nvars); + for (; nvars; nvars--) { + getlocvar(fs, fs->nactvar - nvars).startpc = fs->pc; + } +} + + +static void removevars (LexState *ls, int tolevel) { + FuncState *fs = ls->fs; + while (fs->nactvar > tolevel) + getlocvar(fs, --fs->nactvar).endpc = fs->pc; +} + + +static int indexupvalue (FuncState *fs, TString *name, expdesc *v) { + int i; + Proto *f = fs->f; + int oldsize = f->sizeupvalues; + for (i=0; inups; i++) { + if (fs->upvalues[i].k == v->k && fs->upvalues[i].info == v->u.s.info) { + lua_assert(f->upvalues[i] == name); + return i; + } + } + /* new one */ + luaY_checklimit(fs, f->nups + 1, LUAI_MAXUPVALUES, "upvalues"); + luaM_growvector(fs->L, f->upvalues, f->nups, f->sizeupvalues, + TString *, MAX_INT, ""); + while (oldsize < f->sizeupvalues) f->upvalues[oldsize++] = NULL; + f->upvalues[f->nups] = name; + luaC_objbarrier(fs->L, f, name); + lua_assert(v->k == VLOCAL || v->k == VUPVAL); + fs->upvalues[f->nups].k = cast_byte(v->k); + fs->upvalues[f->nups].info = cast_byte(v->u.s.info); + return f->nups++; +} + + +static int searchvar (FuncState *fs, TString *n) { + int i; + for (i=fs->nactvar-1; i >= 0; i--) { + if (n == getlocvar(fs, i).varname) + return i; + } + return -1; /* not found */ +} + + +static void markupval (FuncState *fs, int level) { + BlockCnt *bl = fs->bl; + while (bl && bl->nactvar > level) bl = bl->previous; + if (bl) bl->upval = 1; +} + + +static int singlevaraux (FuncState *fs, TString *n, expdesc *var, int base) { + if (fs == NULL) { /* no more levels? */ + init_exp(var, VGLOBAL, NO_REG); /* default is global variable */ + return VGLOBAL; + } + else { + int v = searchvar(fs, n); /* look up at current level */ + if (v >= 0) { + init_exp(var, VLOCAL, v); + if (!base) + markupval(fs, v); /* local will be used as an upval */ + return VLOCAL; + } + else { /* not found at current level; try upper one */ + if (singlevaraux(fs->prev, n, var, 0) == VGLOBAL) + return VGLOBAL; + var->u.s.info = indexupvalue(fs, n, var); /* else was LOCAL or UPVAL */ + var->k = VUPVAL; /* upvalue in this level */ + return VUPVAL; + } + } +} + + +static void singlevar (LexState *ls, expdesc *var) { + TString *varname = str_checkname(ls); + FuncState *fs = ls->fs; + if (singlevaraux(fs, varname, var, 1) == VGLOBAL) + var->u.s.info = luaK_stringK(fs, varname); /* info points to global name */ +} + + +static void adjust_assign (LexState *ls, int nvars, int nexps, expdesc *e) { + FuncState *fs = ls->fs; + int extra = nvars - nexps; + if (hasmultret(e->k)) { + extra++; /* includes call itself */ + if (extra < 0) extra = 0; + luaK_setreturns(fs, e, extra); /* last exp. provides the difference */ + if (extra > 1) luaK_reserveregs(fs, extra-1); + } + else { + if (e->k != VVOID) luaK_exp2nextreg(fs, e); /* close last expression */ + if (extra > 0) { + int reg = fs->freereg; + luaK_reserveregs(fs, extra); + luaK_nil(fs, reg, extra); + } + } +} + + +static void enterlevel (LexState *ls) { + if (++ls->L->nCcalls > LUAI_MAXCCALLS) + luaX_lexerror(ls, "chunk has too many syntax levels", 0); +} + + +#define leavelevel(ls) ((ls)->L->nCcalls--) + + +static void enterblock (FuncState *fs, BlockCnt *bl, lu_byte isbreakable) { + bl->breaklist = NO_JUMP; + bl->isbreakable = isbreakable; + bl->nactvar = fs->nactvar; + bl->upval = 0; + bl->previous = fs->bl; + fs->bl = bl; + lua_assert(fs->freereg == fs->nactvar); +} + + +static void leaveblock (FuncState *fs) { + BlockCnt *bl = fs->bl; + fs->bl = bl->previous; + removevars(fs->ls, bl->nactvar); + if (bl->upval) + luaK_codeABC(fs, OP_CLOSE, bl->nactvar, 0, 0); + /* a block either controls scope or breaks (never both) */ + lua_assert(!bl->isbreakable || !bl->upval); + lua_assert(bl->nactvar == fs->nactvar); + fs->freereg = fs->nactvar; /* free registers */ + luaK_patchtohere(fs, bl->breaklist); +} + + +static void pushclosure (LexState *ls, FuncState *func, expdesc *v) { + FuncState *fs = ls->fs; + Proto *f = fs->f; + int oldsize = f->sizep; + int i; + luaM_growvector(ls->L, f->p, fs->np, f->sizep, Proto *, + MAXARG_Bx, "constant table overflow"); + while (oldsize < f->sizep) f->p[oldsize++] = NULL; + f->p[fs->np++] = func->f; + luaC_objbarrier(ls->L, f, func->f); + init_exp(v, VRELOCABLE, luaK_codeABx(fs, OP_CLOSURE, 0, fs->np-1)); + for (i=0; if->nups; i++) { + OpCode o = (func->upvalues[i].k == VLOCAL) ? OP_MOVE : OP_GETUPVAL; + luaK_codeABC(fs, o, 0, func->upvalues[i].info, 0); + } +} + + +static void open_func (LexState *ls, FuncState *fs) { + lua_State *L = ls->L; + Proto *f = luaF_newproto(L); + fs->f = f; + fs->prev = ls->fs; /* linked list of funcstates */ + fs->ls = ls; + fs->L = L; + ls->fs = fs; + fs->pc = 0; + fs->lasttarget = -1; + fs->jpc = NO_JUMP; + fs->freereg = 0; + fs->nk = 0; + fs->np = 0; + fs->nlocvars = 0; + fs->nactvar = 0; + fs->bl = NULL; + f->source = ls->source; + f->maxstacksize = 2; /* registers 0/1 are always valid */ + fs->h = luaH_new(L, 0, 0); + /* anchor table of constants and prototype (to avoid being collected) */ + sethvalue2s(L, L->top, fs->h); + incr_top(L); + setptvalue2s(L, L->top, f); + incr_top(L); +} + + +static void close_func (LexState *ls) { + lua_State *L = ls->L; + FuncState *fs = ls->fs; + Proto *f = fs->f; + removevars(ls, 0); + luaK_ret(fs, 0, 0); /* final return */ + luaM_reallocvector(L, f->code, f->sizecode, fs->pc, Instruction); + f->sizecode = fs->pc; + luaM_reallocvector(L, f->lineinfo, f->sizelineinfo, fs->pc, int); + f->sizelineinfo = fs->pc; + luaM_reallocvector(L, f->k, f->sizek, fs->nk, TValue); + f->sizek = fs->nk; + luaM_reallocvector(L, f->p, f->sizep, fs->np, Proto *); + f->sizep = fs->np; + luaM_reallocvector(L, f->locvars, f->sizelocvars, fs->nlocvars, LocVar); + f->sizelocvars = fs->nlocvars; + luaM_reallocvector(L, f->upvalues, f->sizeupvalues, f->nups, TString *); + f->sizeupvalues = f->nups; + lua_assert(luaG_checkcode(f)); + lua_assert(fs->bl == NULL); + ls->fs = fs->prev; + /* last token read was anchored in defunct function; must reanchor it */ + if (fs) anchor_token(ls); + L->top -= 2; /* remove table and prototype from the stack */ +} + + +Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, const char *name) { + struct LexState lexstate; + struct FuncState funcstate; + lexstate.buff = buff; + luaX_setinput(L, &lexstate, z, luaS_new(L, name)); + open_func(&lexstate, &funcstate); + funcstate.f->is_vararg = VARARG_ISVARARG; /* main func. is always vararg */ + luaX_next(&lexstate); /* read first token */ + chunk(&lexstate); + check(&lexstate, TK_EOS); + close_func(&lexstate); + lua_assert(funcstate.prev == NULL); + lua_assert(funcstate.f->nups == 0); + lua_assert(lexstate.fs == NULL); + return funcstate.f; +} + + + +/*============================================================*/ +/* GRAMMAR RULES */ +/*============================================================*/ + + +static void field (LexState *ls, expdesc *v) { + /* field -> ['.' | ':'] NAME */ + FuncState *fs = ls->fs; + expdesc key; + luaK_exp2anyreg(fs, v); + luaX_next(ls); /* skip the dot or colon */ + checkname(ls, &key); + luaK_indexed(fs, v, &key); +} + + +static void yindex (LexState *ls, expdesc *v) { + /* index -> '[' expr ']' */ + luaX_next(ls); /* skip the '[' */ + expr(ls, v); + luaK_exp2val(ls->fs, v); + checknext(ls, ']'); +} + + +/* +** {====================================================================== +** Rules for Constructors +** ======================================================================= +*/ + + +struct ConsControl { + expdesc v; /* last list item read */ + expdesc *t; /* table descriptor */ + int nh; /* total number of `record' elements */ + int na; /* total number of array elements */ + int tostore; /* number of array elements pending to be stored */ +}; + + +static void recfield (LexState *ls, struct ConsControl *cc) { + /* recfield -> (NAME | `['exp1`]') = exp1 */ + FuncState *fs = ls->fs; + int reg = ls->fs->freereg; + expdesc key, val; + int rkkey; + if (ls->t.token == TK_NAME) { + luaY_checklimit(fs, cc->nh, MAX_INT, "items in a constructor"); + checkname(ls, &key); + } + else /* ls->t.token == '[' */ + yindex(ls, &key); + cc->nh++; + checknext(ls, '='); + rkkey = luaK_exp2RK(fs, &key); + expr(ls, &val); + luaK_codeABC(fs, OP_SETTABLE, cc->t->u.s.info, rkkey, luaK_exp2RK(fs, &val)); + fs->freereg = reg; /* free registers */ +} + + +static void closelistfield (FuncState *fs, struct ConsControl *cc) { + if (cc->v.k == VVOID) return; /* there is no list item */ + luaK_exp2nextreg(fs, &cc->v); + cc->v.k = VVOID; + if (cc->tostore == LFIELDS_PER_FLUSH) { + luaK_setlist(fs, cc->t->u.s.info, cc->na, cc->tostore); /* flush */ + cc->tostore = 0; /* no more items pending */ + } +} + + +static void lastlistfield (FuncState *fs, struct ConsControl *cc) { + if (cc->tostore == 0) return; + if (hasmultret(cc->v.k)) { + luaK_setmultret(fs, &cc->v); + luaK_setlist(fs, cc->t->u.s.info, cc->na, LUA_MULTRET); + cc->na--; /* do not count last expression (unknown number of elements) */ + } + else { + if (cc->v.k != VVOID) + luaK_exp2nextreg(fs, &cc->v); + luaK_setlist(fs, cc->t->u.s.info, cc->na, cc->tostore); + } +} + + +static void listfield (LexState *ls, struct ConsControl *cc) { + expr(ls, &cc->v); + luaY_checklimit(ls->fs, cc->na, MAX_INT, "items in a constructor"); + cc->na++; + cc->tostore++; +} + + +static void constructor (LexState *ls, expdesc *t) { + /* constructor -> ?? */ + FuncState *fs = ls->fs; + int line = ls->linenumber; + int pc = luaK_codeABC(fs, OP_NEWTABLE, 0, 0, 0); + struct ConsControl cc; + cc.na = cc.nh = cc.tostore = 0; + cc.t = t; + init_exp(t, VRELOCABLE, pc); + init_exp(&cc.v, VVOID, 0); /* no value (yet) */ + luaK_exp2nextreg(ls->fs, t); /* fix it at stack top (for gc) */ + checknext(ls, '{'); + do { + lua_assert(cc.v.k == VVOID || cc.tostore > 0); + if (ls->t.token == '}') break; + closelistfield(fs, &cc); + switch(ls->t.token) { + case TK_NAME: { /* may be listfields or recfields */ + luaX_lookahead(ls); + if (ls->lookahead.token != '=') /* expression? */ + listfield(ls, &cc); + else + recfield(ls, &cc); + break; + } + case '[': { /* constructor_item -> recfield */ + recfield(ls, &cc); + break; + } + default: { /* constructor_part -> listfield */ + listfield(ls, &cc); + break; + } + } + } while (testnext(ls, ',') || testnext(ls, ';')); + check_match(ls, '}', '{', line); + lastlistfield(fs, &cc); + SETARG_B(fs->f->code[pc], luaO_int2fb(cc.na)); /* set initial array size */ + SETARG_C(fs->f->code[pc], luaO_int2fb(cc.nh)); /* set initial table size */ +} + +/* }====================================================================== */ + + + +static void parlist (LexState *ls) { + /* parlist -> [ param { `,' param } ] */ + FuncState *fs = ls->fs; + Proto *f = fs->f; + int nparams = 0; + f->is_vararg = 0; + if (ls->t.token != ')') { /* is `parlist' not empty? */ + do { + switch (ls->t.token) { + case TK_NAME: { /* param -> NAME */ + new_localvar(ls, str_checkname(ls), nparams++); + break; + } + case TK_DOTS: { /* param -> `...' */ + luaX_next(ls); +#if defined(LUA_COMPAT_VARARG) + /* use `arg' as default name */ + new_localvarliteral(ls, "arg", nparams++); + f->is_vararg = VARARG_HASARG | VARARG_NEEDSARG; +#endif + f->is_vararg |= VARARG_ISVARARG; + break; + } + default: luaX_syntaxerror(ls, " or " LUA_QL("...") " expected"); + } + } while (!f->is_vararg && testnext(ls, ',')); + } + adjustlocalvars(ls, nparams); + f->numparams = cast_byte(fs->nactvar - (f->is_vararg & VARARG_HASARG)); + luaK_reserveregs(fs, fs->nactvar); /* reserve register for parameters */ +} + + +static void body (LexState *ls, expdesc *e, int needself, int line) { + /* body -> `(' parlist `)' chunk END */ + FuncState new_fs; + open_func(ls, &new_fs); + new_fs.f->linedefined = line; + checknext(ls, '('); + if (needself) { + new_localvarliteral(ls, "self", 0); + adjustlocalvars(ls, 1); + } + parlist(ls); + checknext(ls, ')'); + chunk(ls); + new_fs.f->lastlinedefined = ls->linenumber; + check_match(ls, TK_END, TK_FUNCTION, line); + close_func(ls); + pushclosure(ls, &new_fs, e); +} + + +static int explist1 (LexState *ls, expdesc *v) { + /* explist1 -> expr { `,' expr } */ + int n = 1; /* at least one expression */ + expr(ls, v); + while (testnext(ls, ',')) { + luaK_exp2nextreg(ls->fs, v); + expr(ls, v); + n++; + } + return n; +} + + +static void funcargs (LexState *ls, expdesc *f) { + FuncState *fs = ls->fs; + expdesc args; + int base, nparams; + int line = ls->linenumber; + switch (ls->t.token) { + case '(': { /* funcargs -> `(' [ explist1 ] `)' */ + if (line != ls->lastline) + luaX_syntaxerror(ls,"ambiguous syntax (function call x new statement)"); + luaX_next(ls); + if (ls->t.token == ')') /* arg list is empty? */ + args.k = VVOID; + else { + explist1(ls, &args); + luaK_setmultret(fs, &args); + } + check_match(ls, ')', '(', line); + break; + } + case '{': { /* funcargs -> constructor */ + constructor(ls, &args); + break; + } + case TK_STRING: { /* funcargs -> STRING */ + codestring(ls, &args, ls->t.seminfo.ts); + luaX_next(ls); /* must use `seminfo' before `next' */ + break; + } + default: { + luaX_syntaxerror(ls, "function arguments expected"); + return; + } + } + lua_assert(f->k == VNONRELOC); + base = f->u.s.info; /* base register for call */ + if (hasmultret(args.k)) + nparams = LUA_MULTRET; /* open call */ + else { + if (args.k != VVOID) + luaK_exp2nextreg(fs, &args); /* close last argument */ + nparams = fs->freereg - (base+1); + } + init_exp(f, VCALL, luaK_codeABC(fs, OP_CALL, base, nparams+1, 2)); + luaK_fixline(fs, line); + fs->freereg = base+1; /* call remove function and arguments and leaves + (unless changed) one result */ +} + + + + +/* +** {====================================================================== +** Expression parsing +** ======================================================================= +*/ + + +static void prefixexp (LexState *ls, expdesc *v) { + /* prefixexp -> NAME | '(' expr ')' */ + switch (ls->t.token) { + case '(': { + int line = ls->linenumber; + luaX_next(ls); + expr(ls, v); + check_match(ls, ')', '(', line); + luaK_dischargevars(ls->fs, v); + return; + } + case TK_NAME: { + singlevar(ls, v); + return; + } + default: { + luaX_syntaxerror(ls, "unexpected symbol"); + return; + } + } +} + + +static void primaryexp (LexState *ls, expdesc *v) { + /* primaryexp -> + prefixexp { `.' NAME | `[' exp `]' | `:' NAME funcargs | funcargs } */ + FuncState *fs = ls->fs; + prefixexp(ls, v); + for (;;) { + switch (ls->t.token) { + case '.': { /* field */ + field(ls, v); + break; + } + case '[': { /* `[' exp1 `]' */ + expdesc key; + luaK_exp2anyreg(fs, v); + yindex(ls, &key); + luaK_indexed(fs, v, &key); + break; + } + case ':': { /* `:' NAME funcargs */ + expdesc key; + luaX_next(ls); + checkname(ls, &key); + luaK_self(fs, v, &key); + funcargs(ls, v); + break; + } + case '(': case TK_STRING: case '{': { /* funcargs */ + luaK_exp2nextreg(fs, v); + funcargs(ls, v); + break; + } + default: return; + } + } +} + + +static void simpleexp (LexState *ls, expdesc *v) { + /* simpleexp -> NUMBER | STRING | NIL | true | false | ... | + constructor | FUNCTION body | primaryexp */ + switch (ls->t.token) { + case TK_NUMBER: { + init_exp(v, VKNUM, 0); + v->u.nval = ls->t.seminfo.r; + break; + } + case TK_STRING: { + codestring(ls, v, ls->t.seminfo.ts); + break; + } + case TK_NIL: { + init_exp(v, VNIL, 0); + break; + } + case TK_TRUE: { + init_exp(v, VTRUE, 0); + break; + } + case TK_FALSE: { + init_exp(v, VFALSE, 0); + break; + } + case TK_DOTS: { /* vararg */ + FuncState *fs = ls->fs; + check_condition(ls, fs->f->is_vararg, + "cannot use " LUA_QL("...") " outside a vararg function"); + fs->f->is_vararg &= ~VARARG_NEEDSARG; /* don't need 'arg' */ + init_exp(v, VVARARG, luaK_codeABC(fs, OP_VARARG, 0, 1, 0)); + break; + } + case '{': { /* constructor */ + constructor(ls, v); + return; + } + case TK_FUNCTION: { + luaX_next(ls); + body(ls, v, 0, ls->linenumber); + return; + } + default: { + primaryexp(ls, v); + return; + } + } + luaX_next(ls); +} + + +static UnOpr getunopr (int op) { + switch (op) { + case TK_NOT: return OPR_NOT; + case '-': return OPR_MINUS; + case '#': return OPR_LEN; + default: return OPR_NOUNOPR; + } +} + + +static BinOpr getbinopr (int op) { + switch (op) { + case '+': return OPR_ADD; + case '-': return OPR_SUB; + case '*': return OPR_MUL; + case '/': return OPR_DIV; + case '%': return OPR_MOD; + case '^': return OPR_POW; + case TK_CONCAT: return OPR_CONCAT; + case TK_NE: return OPR_NE; + case TK_EQ: return OPR_EQ; + case '<': return OPR_LT; + case TK_LE: return OPR_LE; + case '>': return OPR_GT; + case TK_GE: return OPR_GE; + case TK_AND: return OPR_AND; + case TK_OR: return OPR_OR; + default: return OPR_NOBINOPR; + } +} + + +static const struct { + lu_byte left; /* left priority for each binary operator */ + lu_byte right; /* right priority */ +} priority[] = { /* ORDER OPR */ + {6, 6}, {6, 6}, {7, 7}, {7, 7}, {7, 7}, /* `+' `-' `/' `%' */ + {10, 9}, {5, 4}, /* power and concat (right associative) */ + {3, 3}, {3, 3}, /* equality and inequality */ + {3, 3}, {3, 3}, {3, 3}, {3, 3}, /* order */ + {2, 2}, {1, 1} /* logical (and/or) */ +}; + +#define UNARY_PRIORITY 8 /* priority for unary operators */ + + +/* +** subexpr -> (simpleexp | unop subexpr) { binop subexpr } +** where `binop' is any binary operator with a priority higher than `limit' +*/ +static BinOpr subexpr (LexState *ls, expdesc *v, unsigned int limit) { + BinOpr op; + UnOpr uop; + enterlevel(ls); + uop = getunopr(ls->t.token); + if (uop != OPR_NOUNOPR) { + luaX_next(ls); + subexpr(ls, v, UNARY_PRIORITY); + luaK_prefix(ls->fs, uop, v); + } + else simpleexp(ls, v); + /* expand while operators have priorities higher than `limit' */ + op = getbinopr(ls->t.token); + while (op != OPR_NOBINOPR && priority[op].left > limit) { + expdesc v2; + BinOpr nextop; + luaX_next(ls); + luaK_infix(ls->fs, op, v); + /* read sub-expression with higher priority */ + nextop = subexpr(ls, &v2, priority[op].right); + luaK_posfix(ls->fs, op, v, &v2); + op = nextop; + } + leavelevel(ls); + return op; /* return first untreated operator */ +} + + +static void expr (LexState *ls, expdesc *v) { + subexpr(ls, v, 0); +} + +/* }==================================================================== */ + + + +/* +** {====================================================================== +** Rules for Statements +** ======================================================================= +*/ + + +static int block_follow (int token) { + switch (token) { + case TK_ELSE: case TK_ELSEIF: case TK_END: + case TK_UNTIL: case TK_EOS: + return 1; + default: return 0; + } +} + + +static void block (LexState *ls) { + /* block -> chunk */ + FuncState *fs = ls->fs; + BlockCnt bl; + enterblock(fs, &bl, 0); + chunk(ls); + lua_assert(bl.breaklist == NO_JUMP); + leaveblock(fs); +} + + +/* +** structure to chain all variables in the left-hand side of an +** assignment +*/ +struct LHS_assign { + struct LHS_assign *prev; + expdesc v; /* variable (global, local, upvalue, or indexed) */ +}; + + +/* +** check whether, in an assignment to a local variable, the local variable +** is needed in a previous assignment (to a table). If so, save original +** local value in a safe place and use this safe copy in the previous +** assignment. +*/ +static void check_conflict (LexState *ls, struct LHS_assign *lh, expdesc *v) { + FuncState *fs = ls->fs; + int extra = fs->freereg; /* eventual position to save local variable */ + int conflict = 0; + for (; lh; lh = lh->prev) { + if (lh->v.k == VINDEXED) { + if (lh->v.u.s.info == v->u.s.info) { /* conflict? */ + conflict = 1; + lh->v.u.s.info = extra; /* previous assignment will use safe copy */ + } + if (lh->v.u.s.aux == v->u.s.info) { /* conflict? */ + conflict = 1; + lh->v.u.s.aux = extra; /* previous assignment will use safe copy */ + } + } + } + if (conflict) { + luaK_codeABC(fs, OP_MOVE, fs->freereg, v->u.s.info, 0); /* make copy */ + luaK_reserveregs(fs, 1); + } +} + + +static void assignment (LexState *ls, struct LHS_assign *lh, int nvars) { + expdesc e; + check_condition(ls, VLOCAL <= lh->v.k && lh->v.k <= VINDEXED, + "syntax error"); + if (testnext(ls, ',')) { /* assignment -> `,' primaryexp assignment */ + struct LHS_assign nv; + nv.prev = lh; + primaryexp(ls, &nv.v); + if (nv.v.k == VLOCAL) + check_conflict(ls, lh, &nv.v); + luaY_checklimit(ls->fs, nvars, LUAI_MAXCCALLS - ls->L->nCcalls, + "variables in assignment"); + assignment(ls, &nv, nvars+1); + } + else { /* assignment -> `=' explist1 */ + int nexps; + checknext(ls, '='); + nexps = explist1(ls, &e); + if (nexps != nvars) { + adjust_assign(ls, nvars, nexps, &e); + if (nexps > nvars) + ls->fs->freereg -= nexps - nvars; /* remove extra values */ + } + else { + luaK_setoneret(ls->fs, &e); /* close last expression */ + luaK_storevar(ls->fs, &lh->v, &e); + return; /* avoid default */ + } + } + init_exp(&e, VNONRELOC, ls->fs->freereg-1); /* default assignment */ + luaK_storevar(ls->fs, &lh->v, &e); +} + + +static int cond (LexState *ls) { + /* cond -> exp */ + expdesc v; + expr(ls, &v); /* read condition */ + if (v.k == VNIL) v.k = VFALSE; /* `falses' are all equal here */ + luaK_goiftrue(ls->fs, &v); + return v.f; +} + + +static void breakstat (LexState *ls) { + FuncState *fs = ls->fs; + BlockCnt *bl = fs->bl; + int upval = 0; + while (bl && !bl->isbreakable) { + upval |= bl->upval; + bl = bl->previous; + } + if (!bl) + luaX_syntaxerror(ls, "no loop to break"); + if (upval) + luaK_codeABC(fs, OP_CLOSE, bl->nactvar, 0, 0); + luaK_concat(fs, &bl->breaklist, luaK_jump(fs)); +} + + +static void whilestat (LexState *ls, int line) { + /* whilestat -> WHILE cond DO block END */ + FuncState *fs = ls->fs; + int whileinit; + int condexit; + BlockCnt bl; + luaX_next(ls); /* skip WHILE */ + whileinit = luaK_getlabel(fs); + condexit = cond(ls); + enterblock(fs, &bl, 1); + checknext(ls, TK_DO); + block(ls); + luaK_patchlist(fs, luaK_jump(fs), whileinit); + check_match(ls, TK_END, TK_WHILE, line); + leaveblock(fs); + luaK_patchtohere(fs, condexit); /* false conditions finish the loop */ +} + + +static void repeatstat (LexState *ls, int line) { + /* repeatstat -> REPEAT block UNTIL cond */ + int condexit; + FuncState *fs = ls->fs; + int repeat_init = luaK_getlabel(fs); + BlockCnt bl1, bl2; + enterblock(fs, &bl1, 1); /* loop block */ + enterblock(fs, &bl2, 0); /* scope block */ + luaX_next(ls); /* skip REPEAT */ + chunk(ls); + check_match(ls, TK_UNTIL, TK_REPEAT, line); + condexit = cond(ls); /* read condition (inside scope block) */ + if (!bl2.upval) { /* no upvalues? */ + leaveblock(fs); /* finish scope */ + luaK_patchlist(ls->fs, condexit, repeat_init); /* close the loop */ + } + else { /* complete semantics when there are upvalues */ + breakstat(ls); /* if condition then break */ + luaK_patchtohere(ls->fs, condexit); /* else... */ + leaveblock(fs); /* finish scope... */ + luaK_patchlist(ls->fs, luaK_jump(fs), repeat_init); /* and repeat */ + } + leaveblock(fs); /* finish loop */ +} + + +static int exp1 (LexState *ls) { + expdesc e; + int k; + expr(ls, &e); + k = e.k; + luaK_exp2nextreg(ls->fs, &e); + return k; +} + + +static void forbody (LexState *ls, int base, int line, int nvars, int isnum) { + /* forbody -> DO block */ + BlockCnt bl; + FuncState *fs = ls->fs; + int prep, endfor; + adjustlocalvars(ls, 3); /* control variables */ + checknext(ls, TK_DO); + prep = isnum ? luaK_codeAsBx(fs, OP_FORPREP, base, NO_JUMP) : luaK_jump(fs); + enterblock(fs, &bl, 0); /* scope for declared variables */ + adjustlocalvars(ls, nvars); + luaK_reserveregs(fs, nvars); + block(ls); + leaveblock(fs); /* end of scope for declared variables */ + luaK_patchtohere(fs, prep); + endfor = (isnum) ? luaK_codeAsBx(fs, OP_FORLOOP, base, NO_JUMP) : + luaK_codeABC(fs, OP_TFORLOOP, base, 0, nvars); + luaK_fixline(fs, line); /* pretend that `OP_FOR' starts the loop */ + luaK_patchlist(fs, (isnum ? endfor : luaK_jump(fs)), prep + 1); +} + + +static void fornum (LexState *ls, TString *varname, int line) { + /* fornum -> NAME = exp1,exp1[,exp1] forbody */ + FuncState *fs = ls->fs; + int base = fs->freereg; + new_localvarliteral(ls, "(for index)", 0); + new_localvarliteral(ls, "(for limit)", 1); + new_localvarliteral(ls, "(for step)", 2); + new_localvar(ls, varname, 3); + checknext(ls, '='); + exp1(ls); /* initial value */ + checknext(ls, ','); + exp1(ls); /* limit */ + if (testnext(ls, ',')) + exp1(ls); /* optional step */ + else { /* default step = 1 */ + luaK_codeABx(fs, OP_LOADK, fs->freereg, luaK_numberK(fs, 1)); + luaK_reserveregs(fs, 1); + } + forbody(ls, base, line, 1, 1); +} + + +static void forlist (LexState *ls, TString *indexname) { + /* forlist -> NAME {,NAME} IN explist1 forbody */ + FuncState *fs = ls->fs; + expdesc e; + int nvars = 0; + int line; + int base = fs->freereg; + /* create control variables */ + new_localvarliteral(ls, "(for generator)", nvars++); + new_localvarliteral(ls, "(for state)", nvars++); + new_localvarliteral(ls, "(for control)", nvars++); + /* create declared variables */ + new_localvar(ls, indexname, nvars++); + while (testnext(ls, ',')) + new_localvar(ls, str_checkname(ls), nvars++); + checknext(ls, TK_IN); + line = ls->linenumber; + adjust_assign(ls, 3, explist1(ls, &e), &e); + luaK_checkstack(fs, 3); /* extra space to call generator */ + forbody(ls, base, line, nvars - 3, 0); +} + + +static void forstat (LexState *ls, int line) { + /* forstat -> FOR (fornum | forlist) END */ + FuncState *fs = ls->fs; + TString *varname; + BlockCnt bl; + enterblock(fs, &bl, 1); /* scope for loop and control variables */ + luaX_next(ls); /* skip `for' */ + varname = str_checkname(ls); /* first variable name */ + switch (ls->t.token) { + case '=': fornum(ls, varname, line); break; + case ',': case TK_IN: forlist(ls, varname); break; + default: luaX_syntaxerror(ls, LUA_QL("=") " or " LUA_QL("in") " expected"); + } + check_match(ls, TK_END, TK_FOR, line); + leaveblock(fs); /* loop scope (`break' jumps to this point) */ +} + + +static int test_then_block (LexState *ls) { + /* test_then_block -> [IF | ELSEIF] cond THEN block */ + int condexit; + luaX_next(ls); /* skip IF or ELSEIF */ + condexit = cond(ls); + checknext(ls, TK_THEN); + block(ls); /* `then' part */ + return condexit; +} + + +static void ifstat (LexState *ls, int line) { + /* ifstat -> IF cond THEN block {ELSEIF cond THEN block} [ELSE block] END */ + FuncState *fs = ls->fs; + int flist; + int escapelist = NO_JUMP; + flist = test_then_block(ls); /* IF cond THEN block */ + while (ls->t.token == TK_ELSEIF) { + luaK_concat(fs, &escapelist, luaK_jump(fs)); + luaK_patchtohere(fs, flist); + flist = test_then_block(ls); /* ELSEIF cond THEN block */ + } + if (ls->t.token == TK_ELSE) { + luaK_concat(fs, &escapelist, luaK_jump(fs)); + luaK_patchtohere(fs, flist); + luaX_next(ls); /* skip ELSE (after patch, for correct line info) */ + block(ls); /* `else' part */ + } + else + luaK_concat(fs, &escapelist, flist); + luaK_patchtohere(fs, escapelist); + check_match(ls, TK_END, TK_IF, line); +} + + +static void localfunc (LexState *ls) { + expdesc v, b; + FuncState *fs = ls->fs; + new_localvar(ls, str_checkname(ls), 0); + init_exp(&v, VLOCAL, fs->freereg); + luaK_reserveregs(fs, 1); + adjustlocalvars(ls, 1); + body(ls, &b, 0, ls->linenumber); + luaK_storevar(fs, &v, &b); + /* debug information will only see the variable after this point! */ + getlocvar(fs, fs->nactvar - 1).startpc = fs->pc; +} + + +static void localstat (LexState *ls) { + /* stat -> LOCAL NAME {`,' NAME} [`=' explist1] */ + int nvars = 0; + int nexps; + expdesc e; + do { + new_localvar(ls, str_checkname(ls), nvars++); + } while (testnext(ls, ',')); + if (testnext(ls, '=')) + nexps = explist1(ls, &e); + else { + e.k = VVOID; + nexps = 0; + } + adjust_assign(ls, nvars, nexps, &e); + adjustlocalvars(ls, nvars); +} + + +static int funcname (LexState *ls, expdesc *v) { + /* funcname -> NAME {field} [`:' NAME] */ + int needself = 0; + singlevar(ls, v); + while (ls->t.token == '.') + field(ls, v); + if (ls->t.token == ':') { + needself = 1; + field(ls, v); + } + return needself; +} + + +static void funcstat (LexState *ls, int line) { + /* funcstat -> FUNCTION funcname body */ + int needself; + expdesc v, b; + luaX_next(ls); /* skip FUNCTION */ + needself = funcname(ls, &v); + body(ls, &b, needself, line); + luaK_storevar(ls->fs, &v, &b); + luaK_fixline(ls->fs, line); /* definition `happens' in the first line */ +} + + +static void exprstat (LexState *ls) { + /* stat -> func | assignment */ + FuncState *fs = ls->fs; + struct LHS_assign v; + primaryexp(ls, &v.v); + if (v.v.k == VCALL) /* stat -> func */ + SETARG_C(getcode(fs, &v.v), 1); /* call statement uses no results */ + else { /* stat -> assignment */ + v.prev = NULL; + assignment(ls, &v, 1); + } +} + + +static void retstat (LexState *ls) { + /* stat -> RETURN explist */ + FuncState *fs = ls->fs; + expdesc e; + int first, nret; /* registers with returned values */ + luaX_next(ls); /* skip RETURN */ + if (block_follow(ls->t.token) || ls->t.token == ';') + first = nret = 0; /* return no values */ + else { + nret = explist1(ls, &e); /* optional return values */ + if (hasmultret(e.k)) { + luaK_setmultret(fs, &e); + if (e.k == VCALL && nret == 1) { /* tail call? */ + SET_OPCODE(getcode(fs,&e), OP_TAILCALL); + lua_assert(GETARG_A(getcode(fs,&e)) == fs->nactvar); + } + first = fs->nactvar; + nret = LUA_MULTRET; /* return all values */ + } + else { + if (nret == 1) /* only one single value? */ + first = luaK_exp2anyreg(fs, &e); + else { + luaK_exp2nextreg(fs, &e); /* values must go to the `stack' */ + first = fs->nactvar; /* return all `active' values */ + lua_assert(nret == fs->freereg - first); + } + } + } + luaK_ret(fs, first, nret); +} + + +static int statement (LexState *ls) { + int line = ls->linenumber; /* may be needed for error messages */ + switch (ls->t.token) { + case TK_IF: { /* stat -> ifstat */ + ifstat(ls, line); + return 0; + } + case TK_WHILE: { /* stat -> whilestat */ + whilestat(ls, line); + return 0; + } + case TK_DO: { /* stat -> DO block END */ + luaX_next(ls); /* skip DO */ + block(ls); + check_match(ls, TK_END, TK_DO, line); + return 0; + } + case TK_FOR: { /* stat -> forstat */ + forstat(ls, line); + return 0; + } + case TK_REPEAT: { /* stat -> repeatstat */ + repeatstat(ls, line); + return 0; + } + case TK_FUNCTION: { + funcstat(ls, line); /* stat -> funcstat */ + return 0; + } + case TK_LOCAL: { /* stat -> localstat */ + luaX_next(ls); /* skip LOCAL */ + if (testnext(ls, TK_FUNCTION)) /* local function? */ + localfunc(ls); + else + localstat(ls); + return 0; + } + case TK_RETURN: { /* stat -> retstat */ + retstat(ls); + return 1; /* must be last statement */ + } + case TK_BREAK: { /* stat -> breakstat */ + luaX_next(ls); /* skip BREAK */ + breakstat(ls); + return 1; /* must be last statement */ + } + default: { + exprstat(ls); + return 0; /* to avoid warnings */ + } + } +} + + +static void chunk (LexState *ls) { + /* chunk -> { stat [`;'] } */ + int islast = 0; + enterlevel(ls); + while (!islast && !block_follow(ls->t.token)) { + islast = statement(ls); + testnext(ls, ';'); + lua_assert(ls->fs->f->maxstacksize >= ls->fs->freereg && + ls->fs->freereg >= ls->fs->nactvar); + ls->fs->freereg = ls->fs->nactvar; /* free registers */ + } + leavelevel(ls); +} + +/* }====================================================================== */ diff --git a/src/lua/lparser.h b/src/lua/lparser.h new file mode 100644 index 0000000..18836af --- /dev/null +++ b/src/lua/lparser.h @@ -0,0 +1,82 @@ +/* +** $Id: lparser.h,v 1.57.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua Parser +** See Copyright Notice in lua.h +*/ + +#ifndef lparser_h +#define lparser_h + +#include "llimits.h" +#include "lobject.h" +#include "lzio.h" + + +/* +** Expression descriptor +*/ + +typedef enum { + VVOID, /* no value */ + VNIL, + VTRUE, + VFALSE, + VK, /* info = index of constant in `k' */ + VKNUM, /* nval = numerical value */ + VLOCAL, /* info = local register */ + VUPVAL, /* info = index of upvalue in `upvalues' */ + VGLOBAL, /* info = index of table; aux = index of global name in `k' */ + VINDEXED, /* info = table register; aux = index register (or `k') */ + VJMP, /* info = instruction pc */ + VRELOCABLE, /* info = instruction pc */ + VNONRELOC, /* info = result register */ + VCALL, /* info = instruction pc */ + VVARARG /* info = instruction pc */ +} expkind; + +typedef struct expdesc { + expkind k; + union { + struct { int info, aux; } s; + lua_Number nval; + } u; + int t; /* patch list of `exit when true' */ + int f; /* patch list of `exit when false' */ +} expdesc; + + +typedef struct upvaldesc { + lu_byte k; + lu_byte info; +} upvaldesc; + + +struct BlockCnt; /* defined in lparser.c */ + + +/* state needed to generate code for a given function */ +typedef struct FuncState { + Proto *f; /* current function header */ + Table *h; /* table to find (and reuse) elements in `k' */ + struct FuncState *prev; /* enclosing function */ + struct LexState *ls; /* lexical state */ + struct lua_State *L; /* copy of the Lua state */ + struct BlockCnt *bl; /* chain of current blocks */ + int pc; /* next position to code (equivalent to `ncode') */ + int lasttarget; /* `pc' of last `jump target' */ + int jpc; /* list of pending jumps to `pc' */ + int freereg; /* first free register */ + int nk; /* number of elements in `k' */ + int np; /* number of elements in `p' */ + short nlocvars; /* number of elements in `locvars' */ + lu_byte nactvar; /* number of active local variables */ + upvaldesc upvalues[LUAI_MAXUPVALUES]; /* upvalues */ + unsigned short actvar[LUAI_MAXVARS]; /* declared-variable stack */ +} FuncState; + + +LUAI_FUNC Proto *luaY_parser (lua_State *L, ZIO *z, Mbuffer *buff, + const char *name); + + +#endif diff --git a/src/lua/lstate.c b/src/lua/lstate.c new file mode 100644 index 0000000..4313b83 --- /dev/null +++ b/src/lua/lstate.c @@ -0,0 +1,214 @@ +/* +** $Id: lstate.c,v 2.36.1.2 2008/01/03 15:20:39 roberto Exp $ +** Global State +** See Copyright Notice in lua.h +*/ + + +#include + +#define lstate_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "llex.h" +#include "lmem.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" + + +#define state_size(x) (sizeof(x) + LUAI_EXTRASPACE) +#define fromstate(l) (cast(lu_byte *, (l)) - LUAI_EXTRASPACE) +#define tostate(l) (cast(lua_State *, cast(lu_byte *, l) + LUAI_EXTRASPACE)) + + +/* +** Main thread combines a thread state and the global state +*/ +typedef struct LG { + lua_State l; + global_State g; +} LG; + + + +static void stack_init (lua_State *L1, lua_State *L) { + /* initialize CallInfo array */ + L1->base_ci = luaM_newvector(L, BASIC_CI_SIZE, CallInfo); + L1->ci = L1->base_ci; + L1->size_ci = BASIC_CI_SIZE; + L1->end_ci = L1->base_ci + L1->size_ci - 1; + /* initialize stack array */ + L1->stack = luaM_newvector(L, BASIC_STACK_SIZE + EXTRA_STACK, TValue); + L1->stacksize = BASIC_STACK_SIZE + EXTRA_STACK; + L1->top = L1->stack; + L1->stack_last = L1->stack+(L1->stacksize - EXTRA_STACK)-1; + /* initialize first ci */ + L1->ci->func = L1->top; + setnilvalue(L1->top++); /* `function' entry for this `ci' */ + L1->base = L1->ci->base = L1->top; + L1->ci->top = L1->top + LUA_MINSTACK; +} + + +static void freestack (lua_State *L, lua_State *L1) { + luaM_freearray(L, L1->base_ci, L1->size_ci, CallInfo); + luaM_freearray(L, L1->stack, L1->stacksize, TValue); +} + + +/* +** open parts that may cause memory-allocation errors +*/ +static void f_luaopen (lua_State *L, void *ud) { + global_State *g = G(L); + UNUSED(ud); + stack_init(L, L); /* init stack */ + sethvalue(L, gt(L), luaH_new(L, 0, 2)); /* table of globals */ + sethvalue(L, registry(L), luaH_new(L, 0, 2)); /* registry */ + luaS_resize(L, MINSTRTABSIZE); /* initial size of string table */ + luaT_init(L); + luaX_init(L); + luaS_fix(luaS_newliteral(L, MEMERRMSG)); + g->GCthreshold = 4*g->totalbytes; +} + + +static void preinit_state (lua_State *L, global_State *g) { + G(L) = g; + L->stack = NULL; + L->stacksize = 0; + L->errorJmp = NULL; + L->hook = NULL; + L->hookmask = 0; + L->basehookcount = 0; + L->allowhook = 1; + resethookcount(L); + L->openupval = NULL; + L->size_ci = 0; + L->nCcalls = L->baseCcalls = 0; + L->status = 0; + L->base_ci = L->ci = NULL; + L->savedpc = NULL; + L->errfunc = 0; + setnilvalue(gt(L)); +} + + +static void close_state (lua_State *L) { + global_State *g = G(L); + luaF_close(L, L->stack); /* close all upvalues for this thread */ + luaC_freeall(L); /* collect all objects */ + lua_assert(g->rootgc == obj2gco(L)); + lua_assert(g->strt.nuse == 0); + luaM_freearray(L, G(L)->strt.hash, G(L)->strt.size, TString *); + luaZ_freebuffer(L, &g->buff); + freestack(L, L); + lua_assert(g->totalbytes == sizeof(LG)); + (*g->frealloc)(g->ud, fromstate(L), state_size(LG), 0); +} + + +lua_State *luaE_newthread (lua_State *L) { + lua_State *L1 = tostate(luaM_malloc(L, state_size(lua_State))); + luaC_link(L, obj2gco(L1), LUA_TTHREAD); + preinit_state(L1, G(L)); + stack_init(L1, L); /* init stack */ + setobj2n(L, gt(L1), gt(L)); /* share table of globals */ + L1->hookmask = L->hookmask; + L1->basehookcount = L->basehookcount; + L1->hook = L->hook; + resethookcount(L1); + lua_assert(iswhite(obj2gco(L1))); + return L1; +} + + +void luaE_freethread (lua_State *L, lua_State *L1) { + luaF_close(L1, L1->stack); /* close all upvalues for this thread */ + lua_assert(L1->openupval == NULL); + luai_userstatefree(L1); + freestack(L, L1); + luaM_freemem(L, fromstate(L1), state_size(lua_State)); +} + + +LUA_API lua_State *lua_newstate (lua_Alloc f, void *ud) { + int i; + lua_State *L; + global_State *g; + void *l = (*f)(ud, NULL, 0, state_size(LG)); + if (l == NULL) return NULL; + L = tostate(l); + g = &((LG *)L)->g; + L->next = NULL; + L->tt = LUA_TTHREAD; + g->currentwhite = bit2mask(WHITE0BIT, FIXEDBIT); + L->marked = luaC_white(g); + set2bits(L->marked, FIXEDBIT, SFIXEDBIT); + preinit_state(L, g); + g->frealloc = f; + g->ud = ud; + g->mainthread = L; + g->uvhead.u.l.prev = &g->uvhead; + g->uvhead.u.l.next = &g->uvhead; + g->GCthreshold = 0; /* mark it as unfinished state */ + g->strt.size = 0; + g->strt.nuse = 0; + g->strt.hash = NULL; + setnilvalue(registry(L)); + luaZ_initbuffer(L, &g->buff); + g->panic = NULL; + g->gcstate = GCSpause; + g->rootgc = obj2gco(L); + g->sweepstrgc = 0; + g->sweepgc = &g->rootgc; + g->gray = NULL; + g->grayagain = NULL; + g->weak = NULL; + g->tmudata = NULL; + g->totalbytes = sizeof(LG); + g->gcpause = LUAI_GCPAUSE; + g->gcstepmul = LUAI_GCMUL; + g->gcdept = 0; + for (i=0; imt[i] = NULL; + if (luaD_rawrunprotected(L, f_luaopen, NULL) != 0) { + /* memory allocation error: free partial state */ + close_state(L); + L = NULL; + } + else + luai_userstateopen(L); + return L; +} + + +static void callallgcTM (lua_State *L, void *ud) { + UNUSED(ud); + luaC_callGCTM(L); /* call GC metamethods for all udata */ +} + + +LUA_API void lua_close (lua_State *L) { + L = G(L)->mainthread; /* only the main thread can be closed */ + lua_lock(L); + luaF_close(L, L->stack); /* close all upvalues for this thread */ + luaC_separateudata(L, 1); /* separate udata that have GC metamethods */ + L->errfunc = 0; /* no error function during GC metamethods */ + do { /* repeat until no more errors */ + L->ci = L->base_ci; + L->base = L->top = L->ci->base; + L->nCcalls = L->baseCcalls = 0; + } while (luaD_rawrunprotected(L, callallgcTM, NULL) != 0); + lua_assert(G(L)->tmudata == NULL); + luai_userstateclose(L); + close_state(L); +} + diff --git a/src/lua/lstate.h b/src/lua/lstate.h new file mode 100644 index 0000000..3bc575b --- /dev/null +++ b/src/lua/lstate.h @@ -0,0 +1,169 @@ +/* +** $Id: lstate.h,v 2.24.1.2 2008/01/03 15:20:39 roberto Exp $ +** Global State +** See Copyright Notice in lua.h +*/ + +#ifndef lstate_h +#define lstate_h + +#include "lua.h" + +#include "lobject.h" +#include "ltm.h" +#include "lzio.h" + + + +struct lua_longjmp; /* defined in ldo.c */ + + +/* table of globals */ +#define gt(L) (&L->l_gt) + +/* registry */ +#define registry(L) (&G(L)->l_registry) + + +/* extra stack space to handle TM calls and some other extras */ +#define EXTRA_STACK 5 + + +#define BASIC_CI_SIZE 8 + +#define BASIC_STACK_SIZE (2*LUA_MINSTACK) + + + +typedef struct stringtable { + GCObject **hash; + lu_int32 nuse; /* number of elements */ + int size; +} stringtable; + + +/* +** informations about a call +*/ +typedef struct CallInfo { + StkId base; /* base for this function */ + StkId func; /* function index in the stack */ + StkId top; /* top for this function */ + const Instruction *savedpc; + int nresults; /* expected number of results from this function */ + int tailcalls; /* number of tail calls lost under this entry */ +} CallInfo; + + + +#define curr_func(L) (clvalue(L->ci->func)) +#define ci_func(ci) (clvalue((ci)->func)) +#define f_isLua(ci) (!ci_func(ci)->c.isC) +#define isLua(ci) (ttisfunction((ci)->func) && f_isLua(ci)) + + +/* +** `global state', shared by all threads of this state +*/ +typedef struct global_State { + stringtable strt; /* hash table for strings */ + lua_Alloc frealloc; /* function to reallocate memory */ + void *ud; /* auxiliary data to `frealloc' */ + lu_byte currentwhite; + lu_byte gcstate; /* state of garbage collector */ + int sweepstrgc; /* position of sweep in `strt' */ + GCObject *rootgc; /* list of all collectable objects */ + GCObject **sweepgc; /* position of sweep in `rootgc' */ + GCObject *gray; /* list of gray objects */ + GCObject *grayagain; /* list of objects to be traversed atomically */ + GCObject *weak; /* list of weak tables (to be cleared) */ + GCObject *tmudata; /* last element of list of userdata to be GC */ + Mbuffer buff; /* temporary buffer for string concatentation */ + lu_mem GCthreshold; + lu_mem totalbytes; /* number of bytes currently allocated */ + lu_mem estimate; /* an estimate of number of bytes actually in use */ + lu_mem gcdept; /* how much GC is `behind schedule' */ + int gcpause; /* size of pause between successive GCs */ + int gcstepmul; /* GC `granularity' */ + lua_CFunction panic; /* to be called in unprotected errors */ + TValue l_registry; + struct lua_State *mainthread; + UpVal uvhead; /* head of double-linked list of all open upvalues */ + struct Table *mt[NUM_TAGS]; /* metatables for basic types */ + TString *tmname[TM_N]; /* array with tag-method names */ +} global_State; + + +/* +** `per thread' state +*/ +struct lua_State { + CommonHeader; + lu_byte status; + StkId top; /* first free slot in the stack */ + StkId base; /* base of current function */ + global_State *l_G; + CallInfo *ci; /* call info for current function */ + const Instruction *savedpc; /* `savedpc' of current function */ + StkId stack_last; /* last free slot in the stack */ + StkId stack; /* stack base */ + CallInfo *end_ci; /* points after end of ci array*/ + CallInfo *base_ci; /* array of CallInfo's */ + int stacksize; + int size_ci; /* size of array `base_ci' */ + unsigned short nCcalls; /* number of nested C calls */ + unsigned short baseCcalls; /* nested C calls when resuming coroutine */ + lu_byte hookmask; + lu_byte allowhook; + int basehookcount; + int hookcount; + lua_Hook hook; + TValue l_gt; /* table of globals */ + TValue env; /* temporary place for environments */ + GCObject *openupval; /* list of open upvalues in this stack */ + GCObject *gclist; + struct lua_longjmp *errorJmp; /* current error recover point */ + ptrdiff_t errfunc; /* current error handling function (stack index) */ +}; + + +#define G(L) (L->l_G) + + +/* +** Union of all collectable objects +*/ +union GCObject { + GCheader gch; + union TString ts; + union Udata u; + union Closure cl; + struct Table h; + struct Proto p; + struct UpVal uv; + struct lua_State th; /* thread */ +}; + + +/* macros to convert a GCObject into a specific value */ +#define rawgco2ts(o) check_exp((o)->gch.tt == LUA_TSTRING, &((o)->ts)) +#define gco2ts(o) (&rawgco2ts(o)->tsv) +#define rawgco2u(o) check_exp((o)->gch.tt == LUA_TUSERDATA, &((o)->u)) +#define gco2u(o) (&rawgco2u(o)->uv) +#define gco2cl(o) check_exp((o)->gch.tt == LUA_TFUNCTION, &((o)->cl)) +#define gco2h(o) check_exp((o)->gch.tt == LUA_TTABLE, &((o)->h)) +#define gco2p(o) check_exp((o)->gch.tt == LUA_TPROTO, &((o)->p)) +#define gco2uv(o) check_exp((o)->gch.tt == LUA_TUPVAL, &((o)->uv)) +#define ngcotouv(o) \ + check_exp((o) == NULL || (o)->gch.tt == LUA_TUPVAL, &((o)->uv)) +#define gco2th(o) check_exp((o)->gch.tt == LUA_TTHREAD, &((o)->th)) + +/* macro to convert any Lua object into a GCObject */ +#define obj2gco(v) (cast(GCObject *, (v))) + + +LUAI_FUNC lua_State *luaE_newthread (lua_State *L); +LUAI_FUNC void luaE_freethread (lua_State *L, lua_State *L1); + +#endif + diff --git a/src/lua/lstring.c b/src/lua/lstring.c new file mode 100644 index 0000000..4911315 --- /dev/null +++ b/src/lua/lstring.c @@ -0,0 +1,111 @@ +/* +** $Id: lstring.c,v 2.8.1.1 2007/12/27 13:02:25 roberto Exp $ +** String table (keeps all strings handled by Lua) +** See Copyright Notice in lua.h +*/ + + +#include + +#define lstring_c +#define LUA_CORE + +#include "lua.h" + +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" + + + +void luaS_resize (lua_State *L, int newsize) { + GCObject **newhash; + stringtable *tb; + int i; + if (G(L)->gcstate == GCSsweepstring) + return; /* cannot resize during GC traverse */ + newhash = luaM_newvector(L, newsize, GCObject *); + tb = &G(L)->strt; + for (i=0; isize; i++) { + GCObject *p = tb->hash[i]; + while (p) { /* for each node in the list */ + GCObject *next = p->gch.next; /* save next */ + unsigned int h = gco2ts(p)->hash; + int h1 = lmod(h, newsize); /* new position */ + lua_assert(cast_int(h%newsize) == lmod(h, newsize)); + p->gch.next = newhash[h1]; /* chain it */ + newhash[h1] = p; + p = next; + } + } + luaM_freearray(L, tb->hash, tb->size, TString *); + tb->size = newsize; + tb->hash = newhash; +} + + +static TString *newlstr (lua_State *L, const char *str, size_t l, + unsigned int h) { + TString *ts; + stringtable *tb; + if (l+1 > (MAX_SIZET - sizeof(TString))/sizeof(char)) + luaM_toobig(L); + ts = cast(TString *, luaM_malloc(L, (l+1)*sizeof(char)+sizeof(TString))); + ts->tsv.len = l; + ts->tsv.hash = h; + ts->tsv.marked = luaC_white(G(L)); + ts->tsv.tt = LUA_TSTRING; + ts->tsv.reserved = 0; + memcpy(ts+1, str, l*sizeof(char)); + ((char *)(ts+1))[l] = '\0'; /* ending 0 */ + tb = &G(L)->strt; + h = lmod(h, tb->size); + ts->tsv.next = tb->hash[h]; /* chain new entry */ + tb->hash[h] = obj2gco(ts); + tb->nuse++; + if (tb->nuse > cast(lu_int32, tb->size) && tb->size <= MAX_INT/2) + luaS_resize(L, tb->size*2); /* too crowded */ + return ts; +} + + +TString *luaS_newlstr (lua_State *L, const char *str, size_t l) { + GCObject *o; + unsigned int h = cast(unsigned int, l); /* seed */ + size_t step = (l>>5)+1; /* if string is too long, don't hash all its chars */ + size_t l1; + for (l1=l; l1>=step; l1-=step) /* compute hash */ + h = h ^ ((h<<5)+(h>>2)+cast(unsigned char, str[l1-1])); + for (o = G(L)->strt.hash[lmod(h, G(L)->strt.size)]; + o != NULL; + o = o->gch.next) { + TString *ts = rawgco2ts(o); + if (ts->tsv.len == l && (memcmp(str, getstr(ts), l) == 0)) { + /* string may be dead */ + if (isdead(G(L), o)) changewhite(o); + return ts; + } + } + return newlstr(L, str, l, h); /* not found */ +} + + +Udata *luaS_newudata (lua_State *L, size_t s, Table *e) { + Udata *u; + if (s > MAX_SIZET - sizeof(Udata)) + luaM_toobig(L); + u = cast(Udata *, luaM_malloc(L, s + sizeof(Udata))); + u->uv.marked = luaC_white(G(L)); /* is not finalized */ + u->uv.tt = LUA_TUSERDATA; + u->uv.len = s; + u->uv.metatable = NULL; + u->uv.env = e; + /* chain it on udata list (after main thread) */ + u->uv.next = G(L)->mainthread->next; + G(L)->mainthread->next = obj2gco(u); + return u; +} + diff --git a/src/lua/lstring.h b/src/lua/lstring.h new file mode 100644 index 0000000..73a2ff8 --- /dev/null +++ b/src/lua/lstring.h @@ -0,0 +1,31 @@ +/* +** $Id: lstring.h,v 1.43.1.1 2007/12/27 13:02:25 roberto Exp $ +** String table (keep all strings handled by Lua) +** See Copyright Notice in lua.h +*/ + +#ifndef lstring_h +#define lstring_h + + +#include "lgc.h" +#include "lobject.h" +#include "lstate.h" + + +#define sizestring(s) (sizeof(union TString)+((s)->len+1)*sizeof(char)) + +#define sizeudata(u) (sizeof(union Udata)+(u)->len) + +#define luaS_new(L, s) (luaS_newlstr(L, s, strlen(s))) +#define luaS_newliteral(L, s) (luaS_newlstr(L, "" s, \ + (sizeof(s)/sizeof(char))-1)) + +#define luaS_fix(s) l_setbit((s)->tsv.marked, FIXEDBIT) + +LUAI_FUNC void luaS_resize (lua_State *L, int newsize); +LUAI_FUNC Udata *luaS_newudata (lua_State *L, size_t s, Table *e); +LUAI_FUNC TString *luaS_newlstr (lua_State *L, const char *str, size_t l); + + +#endif diff --git a/src/lua/lstrlib.c b/src/lua/lstrlib.c new file mode 100644 index 0000000..7a03489 --- /dev/null +++ b/src/lua/lstrlib.c @@ -0,0 +1,871 @@ +/* +** $Id: lstrlib.c,v 1.132.1.5 2010/05/14 15:34:19 roberto Exp $ +** Standard library for string operations and pattern-matching +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include +#include +#include + +#define lstrlib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +/* macro to `unsign' a character */ +#define uchar(c) ((unsigned char)(c)) + + + +static int str_len (lua_State *L) { + size_t l; + luaL_checklstring(L, 1, &l); + lua_pushinteger(L, l); + return 1; +} + + +static ptrdiff_t posrelat (ptrdiff_t pos, size_t len) { + /* relative string position: negative means back from end */ + if (pos < 0) pos += (ptrdiff_t)len + 1; + return (pos >= 0) ? pos : 0; +} + + +static int str_sub (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + ptrdiff_t start = posrelat(luaL_checkinteger(L, 2), l); + ptrdiff_t end = posrelat(luaL_optinteger(L, 3, -1), l); + if (start < 1) start = 1; + if (end > (ptrdiff_t)l) end = (ptrdiff_t)l; + if (start <= end) + lua_pushlstring(L, s+start-1, end-start+1); + else lua_pushliteral(L, ""); + return 1; +} + + +static int str_reverse (lua_State *L) { + size_t l; + luaL_Buffer b; + const char *s = luaL_checklstring(L, 1, &l); + luaL_buffinit(L, &b); + while (l--) luaL_addchar(&b, s[l]); + luaL_pushresult(&b); + return 1; +} + + +static int str_lower (lua_State *L) { + size_t l; + size_t i; + luaL_Buffer b; + const char *s = luaL_checklstring(L, 1, &l); + luaL_buffinit(L, &b); + for (i=0; i 0) + luaL_addlstring(&b, s, l); + luaL_pushresult(&b); + return 1; +} + + +static int str_byte (lua_State *L) { + size_t l; + const char *s = luaL_checklstring(L, 1, &l); + ptrdiff_t posi = posrelat(luaL_optinteger(L, 2, 1), l); + ptrdiff_t pose = posrelat(luaL_optinteger(L, 3, posi), l); + int n, i; + if (posi <= 0) posi = 1; + if ((size_t)pose > l) pose = l; + if (posi > pose) return 0; /* empty interval; return no values */ + n = (int)(pose - posi + 1); + if (posi + n <= pose) /* overflow? */ + luaL_error(L, "string slice too long"); + luaL_checkstack(L, n, "string slice too long"); + for (i=0; i= ms->level || ms->capture[l].len == CAP_UNFINISHED) + return luaL_error(ms->L, "invalid capture index"); + return l; +} + + +static int capture_to_close (MatchState *ms) { + int level = ms->level; + for (level--; level>=0; level--) + if (ms->capture[level].len == CAP_UNFINISHED) return level; + return luaL_error(ms->L, "invalid pattern capture"); +} + + +static const char *classend (MatchState *ms, const char *p) { + switch (*p++) { + case L_ESC: { + if (*p == '\0') + luaL_error(ms->L, "malformed pattern (ends with " LUA_QL("%%") ")"); + return p+1; + } + case '[': { + if (*p == '^') p++; + do { /* look for a `]' */ + if (*p == '\0') + luaL_error(ms->L, "malformed pattern (missing " LUA_QL("]") ")"); + if (*(p++) == L_ESC && *p != '\0') + p++; /* skip escapes (e.g. `%]') */ + } while (*p != ']'); + return p+1; + } + default: { + return p; + } + } +} + + +static int match_class (int c, int cl) { + int res; + switch (tolower(cl)) { + case 'a' : res = isalpha(c); break; + case 'c' : res = iscntrl(c); break; + case 'd' : res = isdigit(c); break; + case 'l' : res = islower(c); break; + case 'p' : res = ispunct(c); break; + case 's' : res = isspace(c); break; + case 'u' : res = isupper(c); break; + case 'w' : res = isalnum(c); break; + case 'x' : res = isxdigit(c); break; + case 'z' : res = (c == 0); break; + default: return (cl == c); + } + return (islower(cl) ? res : !res); +} + + +static int matchbracketclass (int c, const char *p, const char *ec) { + int sig = 1; + if (*(p+1) == '^') { + sig = 0; + p++; /* skip the `^' */ + } + while (++p < ec) { + if (*p == L_ESC) { + p++; + if (match_class(c, uchar(*p))) + return sig; + } + else if ((*(p+1) == '-') && (p+2 < ec)) { + p+=2; + if (uchar(*(p-2)) <= c && c <= uchar(*p)) + return sig; + } + else if (uchar(*p) == c) return sig; + } + return !sig; +} + + +static int singlematch (int c, const char *p, const char *ep) { + switch (*p) { + case '.': return 1; /* matches any char */ + case L_ESC: return match_class(c, uchar(*(p+1))); + case '[': return matchbracketclass(c, p, ep-1); + default: return (uchar(*p) == c); + } +} + + +static const char *match (MatchState *ms, const char *s, const char *p); + + +static const char *matchbalance (MatchState *ms, const char *s, + const char *p) { + if (*p == 0 || *(p+1) == 0) + luaL_error(ms->L, "unbalanced pattern"); + if (*s != *p) return NULL; + else { + int b = *p; + int e = *(p+1); + int cont = 1; + while (++s < ms->src_end) { + if (*s == e) { + if (--cont == 0) return s+1; + } + else if (*s == b) cont++; + } + } + return NULL; /* string ends out of balance */ +} + + +static const char *max_expand (MatchState *ms, const char *s, + const char *p, const char *ep) { + ptrdiff_t i = 0; /* counts maximum expand for item */ + while ((s+i)src_end && singlematch(uchar(*(s+i)), p, ep)) + i++; + /* keeps trying to match with the maximum repetitions */ + while (i>=0) { + const char *res = match(ms, (s+i), ep+1); + if (res) return res; + i--; /* else didn't match; reduce 1 repetition to try again */ + } + return NULL; +} + + +static const char *min_expand (MatchState *ms, const char *s, + const char *p, const char *ep) { + for (;;) { + const char *res = match(ms, s, ep+1); + if (res != NULL) + return res; + else if (ssrc_end && singlematch(uchar(*s), p, ep)) + s++; /* try with one more repetition */ + else return NULL; + } +} + + +static const char *start_capture (MatchState *ms, const char *s, + const char *p, int what) { + const char *res; + int level = ms->level; + if (level >= LUA_MAXCAPTURES) luaL_error(ms->L, "too many captures"); + ms->capture[level].init = s; + ms->capture[level].len = what; + ms->level = level+1; + if ((res=match(ms, s, p)) == NULL) /* match failed? */ + ms->level--; /* undo capture */ + return res; +} + + +static const char *end_capture (MatchState *ms, const char *s, + const char *p) { + int l = capture_to_close(ms); + const char *res; + ms->capture[l].len = s - ms->capture[l].init; /* close capture */ + if ((res = match(ms, s, p)) == NULL) /* match failed? */ + ms->capture[l].len = CAP_UNFINISHED; /* undo capture */ + return res; +} + + +static const char *match_capture (MatchState *ms, const char *s, int l) { + size_t len; + l = check_capture(ms, l); + len = ms->capture[l].len; + if ((size_t)(ms->src_end-s) >= len && + memcmp(ms->capture[l].init, s, len) == 0) + return s+len; + else return NULL; +} + + +static const char *match (MatchState *ms, const char *s, const char *p) { + init: /* using goto's to optimize tail recursion */ + switch (*p) { + case '(': { /* start capture */ + if (*(p+1) == ')') /* position capture? */ + return start_capture(ms, s, p+2, CAP_POSITION); + else + return start_capture(ms, s, p+1, CAP_UNFINISHED); + } + case ')': { /* end capture */ + return end_capture(ms, s, p+1); + } + case L_ESC: { + switch (*(p+1)) { + case 'b': { /* balanced string? */ + s = matchbalance(ms, s, p+2); + if (s == NULL) return NULL; + p+=4; goto init; /* else return match(ms, s, p+4); */ + } + case 'f': { /* frontier? */ + const char *ep; char previous; + p += 2; + if (*p != '[') + luaL_error(ms->L, "missing " LUA_QL("[") " after " + LUA_QL("%%f") " in pattern"); + ep = classend(ms, p); /* points to what is next */ + previous = (s == ms->src_init) ? '\0' : *(s-1); + if (matchbracketclass(uchar(previous), p, ep-1) || + !matchbracketclass(uchar(*s), p, ep-1)) return NULL; + p=ep; goto init; /* else return match(ms, s, ep); */ + } + default: { + if (isdigit(uchar(*(p+1)))) { /* capture results (%0-%9)? */ + s = match_capture(ms, s, uchar(*(p+1))); + if (s == NULL) return NULL; + p+=2; goto init; /* else return match(ms, s, p+2) */ + } + goto dflt; /* case default */ + } + } + } + case '\0': { /* end of pattern */ + return s; /* match succeeded */ + } + case '$': { + if (*(p+1) == '\0') /* is the `$' the last char in pattern? */ + return (s == ms->src_end) ? s : NULL; /* check end of string */ + else goto dflt; + } + default: dflt: { /* it is a pattern item */ + const char *ep = classend(ms, p); /* points to what is next */ + int m = ssrc_end && singlematch(uchar(*s), p, ep); + switch (*ep) { + case '?': { /* optional */ + const char *res; + if (m && ((res=match(ms, s+1, ep+1)) != NULL)) + return res; + p=ep+1; goto init; /* else return match(ms, s, ep+1); */ + } + case '*': { /* 0 or more repetitions */ + return max_expand(ms, s, p, ep); + } + case '+': { /* 1 or more repetitions */ + return (m ? max_expand(ms, s+1, p, ep) : NULL); + } + case '-': { /* 0 or more repetitions (minimum) */ + return min_expand(ms, s, p, ep); + } + default: { + if (!m) return NULL; + s++; p=ep; goto init; /* else return match(ms, s+1, ep); */ + } + } + } + } +} + + + +static const char *lmemfind (const char *s1, size_t l1, + const char *s2, size_t l2) { + if (l2 == 0) return s1; /* empty strings are everywhere */ + else if (l2 > l1) return NULL; /* avoids a negative `l1' */ + else { + const char *init; /* to search for a `*s2' inside `s1' */ + l2--; /* 1st char will be checked by `memchr' */ + l1 = l1-l2; /* `s2' cannot be found after that */ + while (l1 > 0 && (init = (const char *)memchr(s1, *s2, l1)) != NULL) { + init++; /* 1st char is already checked */ + if (memcmp(init, s2+1, l2) == 0) + return init-1; + else { /* correct `l1' and `s1' to try again */ + l1 -= init-s1; + s1 = init; + } + } + return NULL; /* not found */ + } +} + + +static void push_onecapture (MatchState *ms, int i, const char *s, + const char *e) { + if (i >= ms->level) { + if (i == 0) /* ms->level == 0, too */ + lua_pushlstring(ms->L, s, e - s); /* add whole match */ + else + luaL_error(ms->L, "invalid capture index"); + } + else { + ptrdiff_t l = ms->capture[i].len; + if (l == CAP_UNFINISHED) luaL_error(ms->L, "unfinished capture"); + if (l == CAP_POSITION) + lua_pushinteger(ms->L, ms->capture[i].init - ms->src_init + 1); + else + lua_pushlstring(ms->L, ms->capture[i].init, l); + } +} + + +static int push_captures (MatchState *ms, const char *s, const char *e) { + int i; + int nlevels = (ms->level == 0 && s) ? 1 : ms->level; + luaL_checkstack(ms->L, nlevels, "too many captures"); + for (i = 0; i < nlevels; i++) + push_onecapture(ms, i, s, e); + return nlevels; /* number of strings pushed */ +} + + +static int str_find_aux (lua_State *L, int find) { + size_t l1, l2; + const char *s = luaL_checklstring(L, 1, &l1); + const char *p = luaL_checklstring(L, 2, &l2); + ptrdiff_t init = posrelat(luaL_optinteger(L, 3, 1), l1) - 1; + if (init < 0) init = 0; + else if ((size_t)(init) > l1) init = (ptrdiff_t)l1; + if (find && (lua_toboolean(L, 4) || /* explicit request? */ + strpbrk(p, SPECIALS) == NULL)) { /* or no special characters? */ + /* do a plain search */ + const char *s2 = lmemfind(s+init, l1-init, p, l2); + if (s2) { + lua_pushinteger(L, s2-s+1); + lua_pushinteger(L, s2-s+l2); + return 2; + } + } + else { + MatchState ms; + int anchor = (*p == '^') ? (p++, 1) : 0; + const char *s1=s+init; + ms.L = L; + ms.src_init = s; + ms.src_end = s+l1; + do { + const char *res; + ms.level = 0; + if ((res=match(&ms, s1, p)) != NULL) { + if (find) { + lua_pushinteger(L, s1-s+1); /* start */ + lua_pushinteger(L, res-s); /* end */ + return push_captures(&ms, NULL, 0) + 2; + } + else + return push_captures(&ms, s1, res); + } + } while (s1++ < ms.src_end && !anchor); + } + lua_pushnil(L); /* not found */ + return 1; +} + + +static int str_find (lua_State *L) { + return str_find_aux(L, 1); +} + + +static int str_match (lua_State *L) { + return str_find_aux(L, 0); +} + + +static int gmatch_aux (lua_State *L) { + MatchState ms; + size_t ls; + const char *s = lua_tolstring(L, lua_upvalueindex(1), &ls); + const char *p = lua_tostring(L, lua_upvalueindex(2)); + const char *src; + ms.L = L; + ms.src_init = s; + ms.src_end = s+ls; + for (src = s + (size_t)lua_tointeger(L, lua_upvalueindex(3)); + src <= ms.src_end; + src++) { + const char *e; + ms.level = 0; + if ((e = match(&ms, src, p)) != NULL) { + lua_Integer newstart = e-s; + if (e == src) newstart++; /* empty match? go at least one position */ + lua_pushinteger(L, newstart); + lua_replace(L, lua_upvalueindex(3)); + return push_captures(&ms, src, e); + } + } + return 0; /* not found */ +} + + +static int gmatch (lua_State *L) { + luaL_checkstring(L, 1); + luaL_checkstring(L, 2); + lua_settop(L, 2); + lua_pushinteger(L, 0); + lua_pushcclosure(L, gmatch_aux, 3); + return 1; +} + + +static int gfind_nodef (lua_State *L) { + return luaL_error(L, LUA_QL("string.gfind") " was renamed to " + LUA_QL("string.gmatch")); +} + + +static void add_s (MatchState *ms, luaL_Buffer *b, const char *s, + const char *e) { + size_t l, i; + const char *news = lua_tolstring(ms->L, 3, &l); + for (i = 0; i < l; i++) { + if (news[i] != L_ESC) + luaL_addchar(b, news[i]); + else { + i++; /* skip ESC */ + if (!isdigit(uchar(news[i]))) + luaL_addchar(b, news[i]); + else if (news[i] == '0') + luaL_addlstring(b, s, e - s); + else { + push_onecapture(ms, news[i] - '1', s, e); + luaL_addvalue(b); /* add capture to accumulated result */ + } + } + } +} + + +static void add_value (MatchState *ms, luaL_Buffer *b, const char *s, + const char *e) { + lua_State *L = ms->L; + switch (lua_type(L, 3)) { + case LUA_TNUMBER: + case LUA_TSTRING: { + add_s(ms, b, s, e); + return; + } + case LUA_TFUNCTION: { + int n; + lua_pushvalue(L, 3); + n = push_captures(ms, s, e); + lua_call(L, n, 1); + break; + } + case LUA_TTABLE: { + push_onecapture(ms, 0, s, e); + lua_gettable(L, 3); + break; + } + } + if (!lua_toboolean(L, -1)) { /* nil or false? */ + lua_pop(L, 1); + lua_pushlstring(L, s, e - s); /* keep original text */ + } + else if (!lua_isstring(L, -1)) + luaL_error(L, "invalid replacement value (a %s)", luaL_typename(L, -1)); + luaL_addvalue(b); /* add result to accumulator */ +} + + +static int str_gsub (lua_State *L) { + size_t srcl; + const char *src = luaL_checklstring(L, 1, &srcl); + const char *p = luaL_checkstring(L, 2); + int tr = lua_type(L, 3); + int max_s = luaL_optint(L, 4, srcl+1); + int anchor = (*p == '^') ? (p++, 1) : 0; + int n = 0; + MatchState ms; + luaL_Buffer b; + luaL_argcheck(L, tr == LUA_TNUMBER || tr == LUA_TSTRING || + tr == LUA_TFUNCTION || tr == LUA_TTABLE, 3, + "string/function/table expected"); + luaL_buffinit(L, &b); + ms.L = L; + ms.src_init = src; + ms.src_end = src+srcl; + while (n < max_s) { + const char *e; + ms.level = 0; + e = match(&ms, src, p); + if (e) { + n++; + add_value(&ms, &b, src, e); + } + if (e && e>src) /* non empty match? */ + src = e; /* skip it */ + else if (src < ms.src_end) + luaL_addchar(&b, *src++); + else break; + if (anchor) break; + } + luaL_addlstring(&b, src, ms.src_end-src); + luaL_pushresult(&b); + lua_pushinteger(L, n); /* number of substitutions */ + return 2; +} + +/* }====================================================== */ + + +/* maximum size of each formatted item (> len(format('%99.99f', -1e308))) */ +#define MAX_ITEM 512 +/* valid flags in a format specification */ +#define FLAGS "-+ #0" +/* +** maximum size of each format specification (such as '%-099.99d') +** (+10 accounts for %99.99x plus margin of error) +*/ +#define MAX_FORMAT (sizeof(FLAGS) + sizeof(LUA_INTFRMLEN) + 10) + + +static void addquoted (lua_State *L, luaL_Buffer *b, int arg) { + size_t l; + const char *s = luaL_checklstring(L, arg, &l); + luaL_addchar(b, '"'); + while (l--) { + switch (*s) { + case '"': case '\\': case '\n': { + luaL_addchar(b, '\\'); + luaL_addchar(b, *s); + break; + } + case '\r': { + luaL_addlstring(b, "\\r", 2); + break; + } + case '\0': { + luaL_addlstring(b, "\\000", 4); + break; + } + default: { + luaL_addchar(b, *s); + break; + } + } + s++; + } + luaL_addchar(b, '"'); +} + +static const char *scanformat (lua_State *L, const char *strfrmt, char *form) { + const char *p = strfrmt; + while (*p != '\0' && strchr(FLAGS, *p) != NULL) p++; /* skip flags */ + if ((size_t)(p - strfrmt) >= sizeof(FLAGS)) + luaL_error(L, "invalid format (repeated flags)"); + if (isdigit(uchar(*p))) p++; /* skip width */ + if (isdigit(uchar(*p))) p++; /* (2 digits at most) */ + if (*p == '.') { + p++; + if (isdigit(uchar(*p))) p++; /* skip precision */ + if (isdigit(uchar(*p))) p++; /* (2 digits at most) */ + } + if (isdigit(uchar(*p))) + luaL_error(L, "invalid format (width or precision too long)"); + *(form++) = '%'; + strncpy(form, strfrmt, p - strfrmt + 1); + form += p - strfrmt + 1; + *form = '\0'; + return p; +} + + +static void addintlen (char *form) { + size_t l = strlen(form); + char spec = form[l - 1]; + strcpy(form + l - 1, LUA_INTFRMLEN); + form[l + sizeof(LUA_INTFRMLEN) - 2] = spec; + form[l + sizeof(LUA_INTFRMLEN) - 1] = '\0'; +} + + +static int str_format (lua_State *L) { + int top = lua_gettop(L); + int arg = 1; + size_t sfl; + const char *strfrmt = luaL_checklstring(L, arg, &sfl); + const char *strfrmt_end = strfrmt+sfl; + luaL_Buffer b; + luaL_buffinit(L, &b); + while (strfrmt < strfrmt_end) { + if (*strfrmt != L_ESC) + luaL_addchar(&b, *strfrmt++); + else if (*++strfrmt == L_ESC) + luaL_addchar(&b, *strfrmt++); /* %% */ + else { /* format item */ + char form[MAX_FORMAT]; /* to store the format (`%...') */ + char buff[MAX_ITEM]; /* to store the formatted item */ + if (++arg > top) + luaL_argerror(L, arg, "no value"); + strfrmt = scanformat(L, strfrmt, form); + switch (*strfrmt++) { + case 'c': { + sprintf(buff, form, (int)luaL_checknumber(L, arg)); + break; + } + case 'd': case 'i': { + addintlen(form); + sprintf(buff, form, (LUA_INTFRM_T)luaL_checknumber(L, arg)); + break; + } + case 'o': case 'u': case 'x': case 'X': { + addintlen(form); + sprintf(buff, form, (unsigned LUA_INTFRM_T)luaL_checknumber(L, arg)); + break; + } + case 'e': case 'E': case 'f': + case 'g': case 'G': { + sprintf(buff, form, (double)luaL_checknumber(L, arg)); + break; + } + case 'q': { + addquoted(L, &b, arg); + continue; /* skip the 'addsize' at the end */ + } + case 's': { + size_t l; + const char *s = luaL_checklstring(L, arg, &l); + if (!strchr(form, '.') && l >= 100) { + /* no precision and string is too long to be formatted; + keep original string */ + lua_pushvalue(L, arg); + luaL_addvalue(&b); + continue; /* skip the `addsize' at the end */ + } + else { + sprintf(buff, form, s); + break; + } + } + default: { /* also treat cases `pnLlh' */ + return luaL_error(L, "invalid option " LUA_QL("%%%c") " to " + LUA_QL("format"), *(strfrmt - 1)); + } + } + luaL_addlstring(&b, buff, strlen(buff)); + } + } + luaL_pushresult(&b); + return 1; +} + + +static const luaL_Reg strlib[] = { + {"byte", str_byte}, + {"char", str_char}, + {"dump", str_dump}, + {"find", str_find}, + {"format", str_format}, + {"gfind", gfind_nodef}, + {"gmatch", gmatch}, + {"gsub", str_gsub}, + {"len", str_len}, + {"lower", str_lower}, + {"match", str_match}, + {"rep", str_rep}, + {"reverse", str_reverse}, + {"sub", str_sub}, + {"upper", str_upper}, + {NULL, NULL} +}; + + +static void createmetatable (lua_State *L) { + lua_createtable(L, 0, 1); /* create metatable for strings */ + lua_pushliteral(L, ""); /* dummy string */ + lua_pushvalue(L, -2); + lua_setmetatable(L, -2); /* set string metatable */ + lua_pop(L, 1); /* pop dummy string */ + lua_pushvalue(L, -2); /* string library... */ + lua_setfield(L, -2, "__index"); /* ...is the __index metamethod */ + lua_pop(L, 1); /* pop metatable */ +} + + +/* +** Open string library +*/ +LUALIB_API int luaopen_string (lua_State *L) { + luaL_register(L, LUA_STRLIBNAME, strlib); +#if defined(LUA_COMPAT_GFIND) + lua_getfield(L, -1, "gmatch"); + lua_setfield(L, -2, "gfind"); +#endif + createmetatable(L); + return 1; +} + diff --git a/src/lua/ltable.c b/src/lua/ltable.c new file mode 100644 index 0000000..352856c --- /dev/null +++ b/src/lua/ltable.c @@ -0,0 +1,596 @@ +/* +** $Id: ltable.c,v 2.32.1.2 2007/12/28 15:32:23 roberto Exp $ +** Lua tables (hash) +** See Copyright Notice in lua.h +*/ + + +/* +** Implementation of tables (aka arrays, objects, or hash tables). +** Tables keep its elements in two parts: an array part and a hash part. +** Non-negative integer keys are all candidates to be kept in the array +** part. The actual size of the array is the largest `n' such that at +** least half the slots between 0 and n are in use. +** Hash uses a mix of chained scatter table with Brent's variation. +** A main invariant of these tables is that, if an element is not +** in its main position (i.e. the `original' position that its hash gives +** to it), then the colliding element is in its own main position. +** Hence even when the load factor reaches 100%, performance remains good. +*/ + +#include +#include + +#define ltable_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lgc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstate.h" +#include "ltable.h" + + +/* +** max size of array part is 2^MAXBITS +*/ +#if LUAI_BITSINT > 26 +#define MAXBITS 26 +#else +#define MAXBITS (LUAI_BITSINT-2) +#endif + +#define MAXASIZE (1 << MAXBITS) + + +#define hashpow2(t,n) (gnode(t, lmod((n), sizenode(t)))) + +#define hashstr(t,str) hashpow2(t, (str)->tsv.hash) +#define hashboolean(t,p) hashpow2(t, p) + + +/* +** for some types, it is better to avoid modulus by power of 2, as +** they tend to have many 2 factors. +*/ +#define hashmod(t,n) (gnode(t, ((n) % ((sizenode(t)-1)|1)))) + + +#define hashpointer(t,p) hashmod(t, IntPoint(p)) + + +/* +** number of ints inside a lua_Number +*/ +#define numints cast_int(sizeof(lua_Number)/sizeof(int)) + + + +#define dummynode (&dummynode_) + +static const Node dummynode_ = { + {{NULL}, LUA_TNIL}, /* value */ + {{{NULL}, LUA_TNIL, NULL}} /* key */ +}; + + +/* +** hash for lua_Numbers +*/ +static Node *hashnum (const Table *t, lua_Number n) { + unsigned int a[numints]; + int i; + if (luai_numeq(n, 0)) /* avoid problems with -0 */ + return gnode(t, 0); + memcpy(a, &n, sizeof(a)); + for (i = 1; i < numints; i++) a[0] += a[i]; + return hashmod(t, a[0]); +} + + + +/* +** returns the `main' position of an element in a table (that is, the index +** of its hash value) +*/ +static Node *mainposition (const Table *t, const TValue *key) { + switch (ttype(key)) { + case LUA_TNUMBER: + return hashnum(t, nvalue(key)); + case LUA_TSTRING: + return hashstr(t, rawtsvalue(key)); + case LUA_TBOOLEAN: + return hashboolean(t, bvalue(key)); + case LUA_TLIGHTUSERDATA: + return hashpointer(t, pvalue(key)); + default: + return hashpointer(t, gcvalue(key)); + } +} + + +/* +** returns the index for `key' if `key' is an appropriate key to live in +** the array part of the table, -1 otherwise. +*/ +static int arrayindex (const TValue *key) { + if (ttisnumber(key)) { + lua_Number n = nvalue(key); + int k; + lua_number2int(k, n); + if (luai_numeq(cast_num(k), n)) + return k; + } + return -1; /* `key' did not match some condition */ +} + + +/* +** returns the index of a `key' for table traversals. First goes all +** elements in the array part, then elements in the hash part. The +** beginning of a traversal is signalled by -1. +*/ +static int findindex (lua_State *L, Table *t, StkId key) { + int i; + if (ttisnil(key)) return -1; /* first iteration */ + i = arrayindex(key); + if (0 < i && i <= t->sizearray) /* is `key' inside array part? */ + return i-1; /* yes; that's the index (corrected to C) */ + else { + Node *n = mainposition(t, key); + do { /* check whether `key' is somewhere in the chain */ + /* key may be dead already, but it is ok to use it in `next' */ + if (luaO_rawequalObj(key2tval(n), key) || + (ttype(gkey(n)) == LUA_TDEADKEY && iscollectable(key) && + gcvalue(gkey(n)) == gcvalue(key))) { + i = cast_int(n - gnode(t, 0)); /* key index in hash table */ + /* hash elements are numbered after array ones */ + return i + t->sizearray; + } + else n = gnext(n); + } while (n); + luaG_runerror(L, "invalid key to " LUA_QL("next")); /* key not found */ + return 0; /* to avoid warnings */ + } +} + + +int luaH_next (lua_State *L, Table *t, StkId key) { + int i = findindex(L, t, key); /* find original element */ + for (i++; i < t->sizearray; i++) { /* try first array part */ + if (!ttisnil(&t->array[i])) { /* a non-nil value? */ + setnvalue(key, cast_num(i+1)); + setobj2s(L, key+1, &t->array[i]); + return 1; + } + } + for (i -= t->sizearray; i < sizenode(t); i++) { /* then hash part */ + if (!ttisnil(gval(gnode(t, i)))) { /* a non-nil value? */ + setobj2s(L, key, key2tval(gnode(t, i))); + setobj2s(L, key+1, gval(gnode(t, i))); + return 1; + } + } + return 0; /* no more elements */ +} + + +/* +** {============================================================= +** Rehash +** ============================================================== +*/ + + +static int computesizes (int nums[], int *narray) { + int i; + int twotoi; /* 2^i */ + int a = 0; /* number of elements smaller than 2^i */ + int na = 0; /* number of elements to go to array part */ + int n = 0; /* optimal size for array part */ + for (i = 0, twotoi = 1; twotoi/2 < *narray; i++, twotoi *= 2) { + if (nums[i] > 0) { + a += nums[i]; + if (a > twotoi/2) { /* more than half elements present? */ + n = twotoi; /* optimal size (till now) */ + na = a; /* all elements smaller than n will go to array part */ + } + } + if (a == *narray) break; /* all elements already counted */ + } + *narray = n; + lua_assert(*narray/2 <= na && na <= *narray); + return na; +} + + +static int countint (const TValue *key, int *nums) { + int k = arrayindex(key); + if (0 < k && k <= MAXASIZE) { /* is `key' an appropriate array index? */ + nums[ceillog2(k)]++; /* count as such */ + return 1; + } + else + return 0; +} + + +static int numusearray (const Table *t, int *nums) { + int lg; + int ttlg; /* 2^lg */ + int ause = 0; /* summation of `nums' */ + int i = 1; /* count to traverse all array keys */ + for (lg=0, ttlg=1; lg<=MAXBITS; lg++, ttlg*=2) { /* for each slice */ + int lc = 0; /* counter */ + int lim = ttlg; + if (lim > t->sizearray) { + lim = t->sizearray; /* adjust upper limit */ + if (i > lim) + break; /* no more elements to count */ + } + /* count elements in range (2^(lg-1), 2^lg] */ + for (; i <= lim; i++) { + if (!ttisnil(&t->array[i-1])) + lc++; + } + nums[lg] += lc; + ause += lc; + } + return ause; +} + + +static int numusehash (const Table *t, int *nums, int *pnasize) { + int totaluse = 0; /* total number of elements */ + int ause = 0; /* summation of `nums' */ + int i = sizenode(t); + while (i--) { + Node *n = &t->node[i]; + if (!ttisnil(gval(n))) { + ause += countint(key2tval(n), nums); + totaluse++; + } + } + *pnasize += ause; + return totaluse; +} + + +static void setarrayvector (lua_State *L, Table *t, int size) { + int i; + luaM_reallocvector(L, t->array, t->sizearray, size, TValue); + for (i=t->sizearray; iarray[i]); + t->sizearray = size; +} + + +static void setnodevector (lua_State *L, Table *t, int size) { + int lsize; + if (size == 0) { /* no elements to hash part? */ + t->node = cast(Node *, dummynode); /* use common `dummynode' */ + lsize = 0; + } + else { + int i; + lsize = ceillog2(size); + if (lsize > MAXBITS) + luaG_runerror(L, "table overflow"); + size = twoto(lsize); + t->node = luaM_newvector(L, size, Node); + for (i=0; ilsizenode = cast_byte(lsize); + t->lastfree = gnode(t, size); /* all positions are free */ +} + + +static void resize (lua_State *L, Table *t, int nasize, int nhsize) { + int i; + int oldasize = t->sizearray; + int oldhsize = t->lsizenode; + Node *nold = t->node; /* save old hash ... */ + if (nasize > oldasize) /* array part must grow? */ + setarrayvector(L, t, nasize); + /* create new hash part with appropriate size */ + setnodevector(L, t, nhsize); + if (nasize < oldasize) { /* array part must shrink? */ + t->sizearray = nasize; + /* re-insert elements from vanishing slice */ + for (i=nasize; iarray[i])) + setobjt2t(L, luaH_setnum(L, t, i+1), &t->array[i]); + } + /* shrink array */ + luaM_reallocvector(L, t->array, oldasize, nasize, TValue); + } + /* re-insert elements from hash part */ + for (i = twoto(oldhsize) - 1; i >= 0; i--) { + Node *old = nold+i; + if (!ttisnil(gval(old))) + setobjt2t(L, luaH_set(L, t, key2tval(old)), gval(old)); + } + if (nold != dummynode) + luaM_freearray(L, nold, (size_t)twoto(oldhsize), Node); /* free old array */ +} + + +void luaH_resizearray (lua_State *L, Table *t, int nasize) { + int nsize = (t->node == dummynode) ? 0 : sizenode(t); + resize(L, t, nasize, nsize); +} + + +static void rehash (lua_State *L, Table *t, const TValue *ek) { + int nasize, na; + int nums[MAXBITS+1]; /* nums[i] = number of keys between 2^(i-1) and 2^i */ + int i; + int totaluse; + for (i=0; i<=MAXBITS; i++) nums[i] = 0; /* reset counts */ + nasize = numusearray(t, nums); /* count keys in array part */ + totaluse = nasize; /* all those keys are integer keys */ + totaluse += numusehash(t, nums, &nasize); /* count keys in hash part */ + /* count extra key */ + nasize += countint(ek, nums); + totaluse++; + /* compute new size for array part */ + na = computesizes(nums, &nasize); + /* resize the table to new computed sizes */ + resize(L, t, nasize, totaluse - na); +} + + + +/* +** }============================================================= +*/ + + +Table *luaH_new (lua_State *L, int narray, int nhash) { + Table *t = luaM_new(L, Table); + luaC_link(L, obj2gco(t), LUA_TTABLE); + t->metatable = NULL; + t->flags = cast_byte(~0); + /* temporary values (kept only if some malloc fails) */ + t->array = NULL; + t->sizearray = 0; + t->lsizenode = 0; + t->node = cast(Node *, dummynode); + setarrayvector(L, t, narray); + setnodevector(L, t, nhash); + return t; +} + + +void luaH_free (lua_State *L, Table *t) { + if (t->node != dummynode) + luaM_freearray(L, t->node, (size_t)sizenode(t), Node); + luaM_freearray(L, t->array, t->sizearray, TValue); + luaM_free(L, t); +} + + +static Node *getfreepos (Table *t) { + while (t->lastfree-- > t->node) { + if (ttisnil(gkey(t->lastfree))) + return t->lastfree; + } + return NULL; /* could not find a free place */ +} + + + +/* +** inserts a new key into a hash table; first, check whether key's main +** position is free. If not, check whether colliding node is in its main +** position or not: if it is not, move colliding node to an empty place and +** put new key in its main position; otherwise (colliding node is in its main +** position), new key goes to an empty position. +*/ +static TValue *newkey (lua_State *L, Table *t, const TValue *key) { + Node *mp = mainposition(t, key); + if (!ttisnil(gval(mp)) || mp == dummynode) { + Node *othern; + Node *n = getfreepos(t); /* get a free place */ + if (n == NULL) { /* cannot find a free place? */ + rehash(L, t, key); /* grow table */ + return luaH_set(L, t, key); /* re-insert key into grown table */ + } + lua_assert(n != dummynode); + othern = mainposition(t, key2tval(mp)); + if (othern != mp) { /* is colliding node out of its main position? */ + /* yes; move colliding node into free position */ + while (gnext(othern) != mp) othern = gnext(othern); /* find previous */ + gnext(othern) = n; /* redo the chain with `n' in place of `mp' */ + *n = *mp; /* copy colliding node into free pos. (mp->next also goes) */ + gnext(mp) = NULL; /* now `mp' is free */ + setnilvalue(gval(mp)); + } + else { /* colliding node is in its own main position */ + /* new node will go into free position */ + gnext(n) = gnext(mp); /* chain new position */ + gnext(mp) = n; + mp = n; + } + } + gkey(mp)->value = key->value; gkey(mp)->tt = key->tt; + luaC_barriert(L, t, key); + lua_assert(ttisnil(gval(mp))); + return gval(mp); +} + + +/* +** search function for integers +*/ +const TValue *luaH_getnum (Table *t, int key) { + /* (1 <= key && key <= t->sizearray) */ + if (cast(unsigned int, key-1) < cast(unsigned int, t->sizearray)) + return &t->array[key-1]; + else { + lua_Number nk = cast_num(key); + Node *n = hashnum(t, nk); + do { /* check whether `key' is somewhere in the chain */ + if (ttisnumber(gkey(n)) && luai_numeq(nvalue(gkey(n)), nk)) + return gval(n); /* that's it */ + else n = gnext(n); + } while (n); + return luaO_nilobject; + } +} + + +/* +** search function for strings +*/ +const TValue *luaH_getstr (Table *t, TString *key) { + Node *n = hashstr(t, key); + do { /* check whether `key' is somewhere in the chain */ + if (ttisstring(gkey(n)) && rawtsvalue(gkey(n)) == key) + return gval(n); /* that's it */ + else n = gnext(n); + } while (n); + return luaO_nilobject; +} + + +/* +** main search function +*/ +const TValue *luaH_get (Table *t, const TValue *key) { + switch (ttype(key)) { + case LUA_TNIL: return luaO_nilobject; + case LUA_TSTRING: return luaH_getstr(t, rawtsvalue(key)); + case LUA_TNUMBER: { + int k; + lua_Number n = nvalue(key); + lua_number2int(k, n); + if (luai_numeq(cast_num(k), nvalue(key))) /* index is int? */ + return luaH_getnum(t, k); /* use specialized version */ + /* else go through */ + } + /* FALLTHRU */ + default: { + Node *n = mainposition(t, key); + do { /* check whether `key' is somewhere in the chain */ + if (luaO_rawequalObj(key2tval(n), key)) + return gval(n); /* that's it */ + else n = gnext(n); + } while (n); + return luaO_nilobject; + } + } +} + + +TValue *luaH_set (lua_State *L, Table *t, const TValue *key) { + const TValue *p = luaH_get(t, key); + t->flags = 0; + if (p != luaO_nilobject) + return cast(TValue *, p); + else { + if (ttisnil(key)) luaG_runerror(L, "table index is nil"); + else if (ttisnumber(key) && luai_numisnan(nvalue(key))) + luaG_runerror(L, "table index is NaN"); + return newkey(L, t, key); + } +} + + +TValue *luaH_setnum (lua_State *L, Table *t, int key) { + const TValue *p = luaH_getnum(t, key); + if (p != luaO_nilobject) + return cast(TValue *, p); + else { + TValue k; + setnvalue(&k, cast_num(key)); + return newkey(L, t, &k); + } +} + + +TValue *luaH_setstr (lua_State *L, Table *t, TString *key) { + const TValue *p = luaH_getstr(t, key); + if (p != luaO_nilobject) + return cast(TValue *, p); + else { + TValue k; + setsvalue(L, &k, key); + return newkey(L, t, &k); + } +} + + +static int unbound_search (Table *t, unsigned int j) { + unsigned int i = j; /* i is zero or a present index */ + j++; + /* find `i' and `j' such that i is present and j is not */ + while (!ttisnil(luaH_getnum(t, j))) { + i = j; + j *= 2; + if (j > cast(unsigned int, MAX_INT)) { /* overflow? */ + /* table was built with bad purposes: resort to linear search */ + i = 1; + while (!ttisnil(luaH_getnum(t, i))) i++; + return i - 1; + } + } + /* now do a binary search between them */ + while (j - i > 1) { + unsigned int m = (i+j)/2; + if (ttisnil(luaH_getnum(t, m))) j = m; + else i = m; + } + return i; +} + + +/* +** Try to find a boundary in table `t'. A `boundary' is an integer index +** such that t[i] is non-nil and t[i+1] is nil (and 0 if t[1] is nil). +*/ +int luaH_getn (Table *t) { + unsigned int j = t->sizearray; + if (j > 0 && ttisnil(&t->array[j - 1])) { + /* there is a boundary in the array part: (binary) search for it */ + unsigned int i = 0; + while (j - i > 1) { + unsigned int m = (i+j)/2; + if (ttisnil(&t->array[m - 1])) j = m; + else i = m; + } + return i; + } + /* else must find a boundary in hash part */ + else if (t->node == dummynode) /* hash part is empty? */ + return j; /* that is easy... */ + else return unbound_search(t, j); +} + + +int luaH_type (Table *t) { + if (t->sizearray == 0) { + return t->node == dummynode ? LUA_TTEMPTY : LUA_TTHASH; + } + return t->node == dummynode ? LUA_TTARRAY : LUA_TTMIXED; +} + + +#if defined(LUA_DEBUG) + +Node *luaH_mainposition (const Table *t, const TValue *key) { + return mainposition(t, key); +} + +int luaH_isdummy (Node *n) { return n == dummynode; } + +#endif diff --git a/src/lua/ltable.h b/src/lua/ltable.h new file mode 100644 index 0000000..d896ae4 --- /dev/null +++ b/src/lua/ltable.h @@ -0,0 +1,41 @@ +/* +** $Id: ltable.h,v 2.10.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua tables (hash) +** See Copyright Notice in lua.h +*/ + +#ifndef ltable_h +#define ltable_h + +#include "lobject.h" + + +#define gnode(t,i) (&(t)->node[i]) +#define gkey(n) (&(n)->i_key.nk) +#define gval(n) (&(n)->i_val) +#define gnext(n) ((n)->i_key.nk.next) + +#define key2tval(n) (&(n)->i_key.tvk) + + +LUAI_FUNC const TValue *luaH_getnum (Table *t, int key); +LUAI_FUNC TValue *luaH_setnum (lua_State *L, Table *t, int key); +LUAI_FUNC const TValue *luaH_getstr (Table *t, TString *key); +LUAI_FUNC TValue *luaH_setstr (lua_State *L, Table *t, TString *key); +LUAI_FUNC const TValue *luaH_get (Table *t, const TValue *key); +LUAI_FUNC TValue *luaH_set (lua_State *L, Table *t, const TValue *key); +LUAI_FUNC Table *luaH_new (lua_State *L, int narray, int lnhash); +LUAI_FUNC void luaH_resizearray (lua_State *L, Table *t, int nasize); +LUAI_FUNC void luaH_free (lua_State *L, Table *t); +LUAI_FUNC int luaH_next (lua_State *L, Table *t, StkId key); +LUAI_FUNC int luaH_getn (Table *t); +LUAI_FUNC int luaH_type (Table *t); + + +#if defined(LUA_DEBUG) +LUAI_FUNC Node *luaH_mainposition (const Table *t, const TValue *key); +LUAI_FUNC int luaH_isdummy (Node *n); +#endif + + +#endif diff --git a/src/lua/ltablib.c b/src/lua/ltablib.c new file mode 100644 index 0000000..14eca90 --- /dev/null +++ b/src/lua/ltablib.c @@ -0,0 +1,287 @@ +/* +** $Id: ltablib.c,v 1.38.1.3 2008/02/14 16:46:58 roberto Exp $ +** Library for Table Manipulation +** See Copyright Notice in lua.h +*/ + + +#include + +#define ltablib_c +#define LUA_LIB + +#include "lua.h" + +#include "lauxlib.h" +#include "lualib.h" + + +#define aux_getn(L,n) (luaL_checktype(L, n, LUA_TTABLE), luaL_getn(L, n)) + + +static int foreachi (lua_State *L) { + int i; + int n = aux_getn(L, 1); + luaL_checktype(L, 2, LUA_TFUNCTION); + for (i=1; i <= n; i++) { + lua_pushvalue(L, 2); /* function */ + lua_pushinteger(L, i); /* 1st argument */ + lua_rawgeti(L, 1, i); /* 2nd argument */ + lua_call(L, 2, 1); + if (!lua_isnil(L, -1)) + return 1; + lua_pop(L, 1); /* remove nil result */ + } + return 0; +} + + +static int foreach (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); + luaL_checktype(L, 2, LUA_TFUNCTION); + lua_pushnil(L); /* first key */ + while (lua_next(L, 1)) { + lua_pushvalue(L, 2); /* function */ + lua_pushvalue(L, -3); /* key */ + lua_pushvalue(L, -3); /* value */ + lua_call(L, 2, 1); + if (!lua_isnil(L, -1)) + return 1; + lua_pop(L, 2); /* remove value and result */ + } + return 0; +} + + +static int maxn (lua_State *L) { + lua_Number max = 0; + luaL_checktype(L, 1, LUA_TTABLE); + lua_pushnil(L); /* first key */ + while (lua_next(L, 1)) { + lua_pop(L, 1); /* remove value */ + if (lua_type(L, -1) == LUA_TNUMBER) { + lua_Number v = lua_tonumber(L, -1); + if (v > max) max = v; + } + } + lua_pushnumber(L, max); + return 1; +} + + +static int getn (lua_State *L) { + lua_pushinteger(L, aux_getn(L, 1)); + return 1; +} + + +static int setn (lua_State *L) { + luaL_checktype(L, 1, LUA_TTABLE); +#ifndef luaL_setn + luaL_setn(L, 1, luaL_checkint(L, 2)); +#else + luaL_error(L, LUA_QL("setn") " is obsolete"); +#endif + lua_pushvalue(L, 1); + return 1; +} + + +static int tinsert (lua_State *L) { + int e = aux_getn(L, 1) + 1; /* first empty element */ + int pos; /* where to insert new element */ + switch (lua_gettop(L)) { + case 2: { /* called with only 2 arguments */ + pos = e; /* insert new element at the end */ + break; + } + case 3: { + int i; + pos = luaL_checkint(L, 2); /* 2nd argument is the position */ + if (pos > e) e = pos; /* `grow' array if necessary */ + for (i = e; i > pos; i--) { /* move up elements */ + lua_rawgeti(L, 1, i-1); + lua_rawseti(L, 1, i); /* t[i] = t[i-1] */ + } + break; + } + default: { + return luaL_error(L, "wrong number of arguments to " LUA_QL("insert")); + } + } + luaL_setn(L, 1, e); /* new size */ + lua_rawseti(L, 1, pos); /* t[pos] = v */ + return 0; +} + + +static int tremove (lua_State *L) { + int e = aux_getn(L, 1); + int pos = luaL_optint(L, 2, e); + if (!(1 <= pos && pos <= e)) /* position is outside bounds? */ + return 0; /* nothing to remove */ + luaL_setn(L, 1, e - 1); /* t.n = n-1 */ + lua_rawgeti(L, 1, pos); /* result = t[pos] */ + for ( ;pos= P */ + while (lua_rawgeti(L, 1, ++i), sort_comp(L, -1, -2)) { + if (i>u) luaL_error(L, "invalid order function for sorting"); + lua_pop(L, 1); /* remove a[i] */ + } + /* repeat --j until a[j] <= P */ + while (lua_rawgeti(L, 1, --j), sort_comp(L, -3, -1)) { + if (j + +#define ltm_c +#define LUA_CORE + +#include "lua.h" + +#include "lobject.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" + + + +const char *const luaT_typenames[] = { + "nil", "boolean", "userdata", "number", + "string", "table", "function", "userdata", "thread", + "proto", "upval" +}; + + +void luaT_init (lua_State *L) { + static const char *const luaT_eventname[] = { /* ORDER TM */ + "__index", "__newindex", + "__gc", "__mode", "__eq", + "__add", "__sub", "__mul", "__div", "__mod", + "__pow", "__unm", "__len", "__lt", "__le", + "__concat", "__call" + }; + int i; + for (i=0; itmname[i] = luaS_new(L, luaT_eventname[i]); + luaS_fix(G(L)->tmname[i]); /* never collect these names */ + } +} + + +/* +** function to be used with macro "fasttm": optimized for absence of +** tag methods +*/ +const TValue *luaT_gettm (Table *events, TMS event, TString *ename) { + const TValue *tm = luaH_getstr(events, ename); + lua_assert(event <= TM_EQ); + if (ttisnil(tm)) { /* no tag method? */ + events->flags |= cast_byte(1u<metatable; + break; + case LUA_TUSERDATA: + mt = uvalue(o)->metatable; + break; + default: + mt = G(L)->mt[ttype(o)]; + } + return (mt ? luaH_getstr(mt, G(L)->tmname[event]) : luaO_nilobject); +} + diff --git a/src/lua/ltm.h b/src/lua/ltm.h new file mode 100644 index 0000000..64343b7 --- /dev/null +++ b/src/lua/ltm.h @@ -0,0 +1,54 @@ +/* +** $Id: ltm.h,v 2.6.1.1 2007/12/27 13:02:25 roberto Exp $ +** Tag methods +** See Copyright Notice in lua.h +*/ + +#ifndef ltm_h +#define ltm_h + + +#include "lobject.h" + + +/* +* WARNING: if you change the order of this enumeration, +* grep "ORDER TM" +*/ +typedef enum { + TM_INDEX, + TM_NEWINDEX, + TM_GC, + TM_MODE, + TM_EQ, /* last tag method with `fast' access */ + TM_ADD, + TM_SUB, + TM_MUL, + TM_DIV, + TM_MOD, + TM_POW, + TM_UNM, + TM_LEN, + TM_LT, + TM_LE, + TM_CONCAT, + TM_CALL, + TM_N /* number of elements in the enum */ +} TMS; + + + +#define gfasttm(g,et,e) ((et) == NULL ? NULL : \ + ((et)->flags & (1u<<(e))) ? NULL : luaT_gettm(et, e, (g)->tmname[e])) + +#define fasttm(l,et,e) gfasttm(G(l), et, e) + +LUAI_DATA const char *const luaT_typenames[]; + + +LUAI_FUNC const TValue *luaT_gettm (Table *events, TMS event, TString *ename); +LUAI_FUNC const TValue *luaT_gettmbyobj (lua_State *L, const TValue *o, + TMS event); +LUAI_FUNC void luaT_init (lua_State *L); + +#endif diff --git a/src/lua/lundump.c b/src/lua/lundump.c new file mode 100644 index 0000000..8010a45 --- /dev/null +++ b/src/lua/lundump.c @@ -0,0 +1,227 @@ +/* +** $Id: lundump.c,v 2.7.1.4 2008/04/04 19:51:41 roberto Exp $ +** load precompiled Lua chunks +** See Copyright Notice in lua.h +*/ + +#include + +#define lundump_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lmem.h" +#include "lobject.h" +#include "lstring.h" +#include "lundump.h" +#include "lzio.h" + +typedef struct { + lua_State* L; + ZIO* Z; + Mbuffer* b; + const char* name; +} LoadState; + +#ifdef LUAC_TRUST_BINARIES +#define IF(c,s) +#define error(S,s) +#else +#define IF(c,s) if (c) error(S,s) + +static void error(LoadState* S, const char* why) +{ + luaO_pushfstring(S->L,"%s: %s in precompiled chunk",S->name,why); + luaD_throw(S->L,LUA_ERRSYNTAX); +} +#endif + +#define LoadMem(S,b,n,size) LoadBlock(S,b,(n)*(size)) +#define LoadByte(S) (lu_byte)LoadChar(S) +#define LoadVar(S,x) LoadMem(S,&x,1,sizeof(x)) +#define LoadVector(S,b,n,size) LoadMem(S,b,n,size) + +static void LoadBlock(LoadState* S, void* b, size_t size) +{ + size_t r=luaZ_read(S->Z,b,size); + IF (r!=0, "unexpected end"); +} + +static int LoadChar(LoadState* S) +{ + char x; + LoadVar(S,x); + return x; +} + +static int LoadInt(LoadState* S) +{ + int x; + LoadVar(S,x); + IF (x<0, "bad integer"); + return x; +} + +static lua_Number LoadNumber(LoadState* S) +{ + lua_Number x; + LoadVar(S,x); + return x; +} + +static TString* LoadString(LoadState* S) +{ + size_t size; + LoadVar(S,size); + if (size==0) + return NULL; + else + { + char* s=luaZ_openspace(S->L,S->b,size); + LoadBlock(S,s,size); + return luaS_newlstr(S->L,s,size-1); /* remove trailing '\0' */ + } +} + +static void LoadCode(LoadState* S, Proto* f) +{ + int n=LoadInt(S); + f->code=luaM_newvector(S->L,n,Instruction); + f->sizecode=n; + LoadVector(S,f->code,n,sizeof(Instruction)); +} + +static Proto* LoadFunction(LoadState* S, TString* p); + +static void LoadConstants(LoadState* S, Proto* f) +{ + int i,n; + n=LoadInt(S); + f->k=luaM_newvector(S->L,n,TValue); + f->sizek=n; + for (i=0; ik[i]); + for (i=0; ik[i]; + int t=LoadChar(S); + switch (t) + { + case LUA_TNIL: + setnilvalue(o); + break; + case LUA_TBOOLEAN: + setbvalue(o,LoadChar(S)!=0); + break; + case LUA_TNUMBER: + setnvalue(o,LoadNumber(S)); + break; + case LUA_TSTRING: + setsvalue2n(S->L,o,LoadString(S)); + break; + default: + error(S,"bad constant"); + break; + } + } + n=LoadInt(S); + f->p=luaM_newvector(S->L,n,Proto*); + f->sizep=n; + for (i=0; ip[i]=NULL; + for (i=0; ip[i]=LoadFunction(S,f->source); +} + +static void LoadDebug(LoadState* S, Proto* f) +{ + int i,n; + n=LoadInt(S); + f->lineinfo=luaM_newvector(S->L,n,int); + f->sizelineinfo=n; + LoadVector(S,f->lineinfo,n,sizeof(int)); + n=LoadInt(S); + f->locvars=luaM_newvector(S->L,n,LocVar); + f->sizelocvars=n; + for (i=0; ilocvars[i].varname=NULL; + for (i=0; ilocvars[i].varname=LoadString(S); + f->locvars[i].startpc=LoadInt(S); + f->locvars[i].endpc=LoadInt(S); + } + n=LoadInt(S); + f->upvalues=luaM_newvector(S->L,n,TString*); + f->sizeupvalues=n; + for (i=0; iupvalues[i]=NULL; + for (i=0; iupvalues[i]=LoadString(S); +} + +static Proto* LoadFunction(LoadState* S, TString* p) +{ + Proto* f; + if (++S->L->nCcalls > LUAI_MAXCCALLS) error(S,"code too deep"); + f=luaF_newproto(S->L); + setptvalue2s(S->L,S->L->top,f); incr_top(S->L); + f->source=LoadString(S); if (f->source==NULL) f->source=p; + f->linedefined=LoadInt(S); + f->lastlinedefined=LoadInt(S); + f->nups=LoadByte(S); + f->numparams=LoadByte(S); + f->is_vararg=LoadByte(S); + f->maxstacksize=LoadByte(S); + LoadCode(S,f); + LoadConstants(S,f); + LoadDebug(S,f); + IF (!luaG_checkcode(f), "bad code"); + S->L->top--; + S->L->nCcalls--; + return f; +} + +static void LoadHeader(LoadState* S) +{ + char h[LUAC_HEADERSIZE]; + char s[LUAC_HEADERSIZE]; + luaU_header(h); + LoadBlock(S,s,LUAC_HEADERSIZE); + IF (memcmp(h,s,LUAC_HEADERSIZE)!=0, "bad header"); +} + +/* +** load precompiled chunk +*/ +Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name) +{ + LoadState S; + if (*name=='@' || *name=='=') + S.name=name+1; + else if (*name==LUA_SIGNATURE[0]) + S.name="binary string"; + else + S.name=name; + S.L=L; + S.Z=Z; + S.b=buff; + LoadHeader(&S); + return LoadFunction(&S,luaS_newliteral(L,"=?")); +} + +/* +* make header +*/ +void luaU_header (char* h) +{ + int x=1; + memcpy(h,LUA_SIGNATURE,sizeof(LUA_SIGNATURE)-1); + h+=sizeof(LUA_SIGNATURE)-1; + *h++=(char)LUAC_VERSION; + *h++=(char)LUAC_FORMAT; + *h++=(char)*(char*)&x; /* endianness */ + *h++=(char)sizeof(int); + *h++=(char)sizeof(size_t); + *h++=(char)sizeof(Instruction); + *h++=(char)sizeof(lua_Number); + *h++=(char)(((lua_Number)0.5)==0); /* is lua_Number integral? */ +} diff --git a/src/lua/lundump.h b/src/lua/lundump.h new file mode 100644 index 0000000..c80189d --- /dev/null +++ b/src/lua/lundump.h @@ -0,0 +1,36 @@ +/* +** $Id: lundump.h,v 1.37.1.1 2007/12/27 13:02:25 roberto Exp $ +** load precompiled Lua chunks +** See Copyright Notice in lua.h +*/ + +#ifndef lundump_h +#define lundump_h + +#include "lobject.h" +#include "lzio.h" + +/* load one chunk; from lundump.c */ +LUAI_FUNC Proto* luaU_undump (lua_State* L, ZIO* Z, Mbuffer* buff, const char* name); + +/* make header; from lundump.c */ +LUAI_FUNC void luaU_header (char* h); + +/* dump one chunk; from ldump.c */ +LUAI_FUNC int luaU_dump (lua_State* L, const Proto* f, lua_Writer w, void* data, int strip); + +#ifdef luac_c +/* print one chunk; from print.c */ +LUAI_FUNC void luaU_print (const Proto* f, int full); +#endif + +/* for header of binary files -- this is Lua 5.1 */ +#define LUAC_VERSION 0x51 + +/* for header of binary files -- this is the official format */ +#define LUAC_FORMAT 0 + +/* size of header of binary files */ +#define LUAC_HEADERSIZE 12 + +#endif diff --git a/src/lua/lvm.c b/src/lua/lvm.c new file mode 100644 index 0000000..e0a0cd8 --- /dev/null +++ b/src/lua/lvm.c @@ -0,0 +1,767 @@ +/* +** $Id: lvm.c,v 2.63.1.5 2011/08/17 20:43:11 roberto Exp $ +** Lua virtual machine +** See Copyright Notice in lua.h +*/ + + +#include +#include +#include + +#define lvm_c +#define LUA_CORE + +#include "lua.h" + +#include "ldebug.h" +#include "ldo.h" +#include "lfunc.h" +#include "lgc.h" +#include "lobject.h" +#include "lopcodes.h" +#include "lstate.h" +#include "lstring.h" +#include "ltable.h" +#include "ltm.h" +#include "lvm.h" + + + +/* limit for table tag-method chains (to avoid loops) */ +#define MAXTAGLOOP 100 + + +const TValue *luaV_tonumber (const TValue *obj, TValue *n) { + lua_Number num; + if (ttisnumber(obj)) return obj; + if (ttisstring(obj) && luaO_str2d(svalue(obj), &num)) { + setnvalue(n, num); + return n; + } + else + return NULL; +} + + +int luaV_tostring (lua_State *L, StkId obj) { + if (!ttisnumber(obj)) + return 0; + else { + char s[LUAI_MAXNUMBER2STR]; + lua_Number n = nvalue(obj); + lua_number2str(s, n); + setsvalue2s(L, obj, luaS_new(L, s)); + return 1; + } +} + + +static void traceexec (lua_State *L, const Instruction *pc) { + lu_byte mask = L->hookmask; + const Instruction *oldpc = L->savedpc; + L->savedpc = pc; + if ((mask & LUA_MASKCOUNT) && L->hookcount == 0) { + resethookcount(L); + luaD_callhook(L, LUA_HOOKCOUNT, -1); + } + if (mask & LUA_MASKLINE) { + Proto *p = ci_func(L->ci)->l.p; + int npc = pcRel(pc, p); + int newline = getline(p, npc); + /* call linehook when enter a new function, when jump back (loop), + or when enter a new line */ + if (npc == 0 || pc <= oldpc || newline != getline(p, pcRel(oldpc, p))) + luaD_callhook(L, LUA_HOOKLINE, newline); + } +} + + +static void callTMres (lua_State *L, StkId res, const TValue *f, + const TValue *p1, const TValue *p2) { + ptrdiff_t result = savestack(L, res); + setobj2s(L, L->top, f); /* push function */ + setobj2s(L, L->top+1, p1); /* 1st argument */ + setobj2s(L, L->top+2, p2); /* 2nd argument */ + luaD_checkstack(L, 3); + L->top += 3; + luaD_call(L, L->top - 3, 1); + res = restorestack(L, result); + L->top--; + setobjs2s(L, res, L->top); +} + + + +static void callTM (lua_State *L, const TValue *f, const TValue *p1, + const TValue *p2, const TValue *p3) { + setobj2s(L, L->top, f); /* push function */ + setobj2s(L, L->top+1, p1); /* 1st argument */ + setobj2s(L, L->top+2, p2); /* 2nd argument */ + setobj2s(L, L->top+3, p3); /* 3th argument */ + luaD_checkstack(L, 4); + L->top += 4; + luaD_call(L, L->top - 4, 0); +} + + +void luaV_gettable (lua_State *L, const TValue *t, TValue *key, StkId val) { + int loop; + for (loop = 0; loop < MAXTAGLOOP; loop++) { + const TValue *tm; + if (ttistable(t)) { /* `t' is a table? */ + Table *h = hvalue(t); + const TValue *res = luaH_get(h, key); /* do a primitive get */ + if (!ttisnil(res) || /* result is no nil? */ + (tm = fasttm(L, h->metatable, TM_INDEX)) == NULL) { /* or no TM? */ + setobj2s(L, val, res); + return; + } + /* else will try the tag method */ + } + else if (ttisnil(tm = luaT_gettmbyobj(L, t, TM_INDEX))) + luaG_typeerror(L, t, "index"); + if (ttisfunction(tm)) { + callTMres(L, val, tm, t, key); + return; + } + t = tm; /* else repeat with `tm' */ + } + luaG_runerror(L, "loop in gettable"); +} + + +void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { + int loop; + TValue temp; + for (loop = 0; loop < MAXTAGLOOP; loop++) { + const TValue *tm; + if (ttistable(t)) { /* `t' is a table? */ + Table *h = hvalue(t); + TValue *oldval = luaH_set(L, h, key); /* do a primitive set */ + if (!ttisnil(oldval) || /* result is no nil? */ + (tm = fasttm(L, h->metatable, TM_NEWINDEX)) == NULL) { /* or no TM? */ + setobj2t(L, oldval, val); + h->flags = 0; + luaC_barriert(L, h, val); + return; + } + /* else will try the tag method */ + } + else if (ttisnil(tm = luaT_gettmbyobj(L, t, TM_NEWINDEX))) + luaG_typeerror(L, t, "index"); + if (ttisfunction(tm)) { + callTM(L, tm, t, key, val); + return; + } + /* else repeat with `tm' */ + setobj(L, &temp, tm); /* avoid pointing inside table (may rehash) */ + t = &temp; + } + luaG_runerror(L, "loop in settable"); +} + + +static int call_binTM (lua_State *L, const TValue *p1, const TValue *p2, + StkId res, TMS event) { + const TValue *tm = luaT_gettmbyobj(L, p1, event); /* try first operand */ + if (ttisnil(tm)) + tm = luaT_gettmbyobj(L, p2, event); /* try second operand */ + if (ttisnil(tm)) return 0; + callTMres(L, res, tm, p1, p2); + return 1; +} + + +static const TValue *get_compTM (lua_State *L, Table *mt1, Table *mt2, + TMS event) { + const TValue *tm1 = fasttm(L, mt1, event); + const TValue *tm2; + if (tm1 == NULL) return NULL; /* no metamethod */ + if (mt1 == mt2) return tm1; /* same metatables => same metamethods */ + tm2 = fasttm(L, mt2, event); + if (tm2 == NULL) return NULL; /* no metamethod */ + if (luaO_rawequalObj(tm1, tm2)) /* same metamethods? */ + return tm1; + return NULL; +} + + +static int call_orderTM (lua_State *L, const TValue *p1, const TValue *p2, + TMS event) { + const TValue *tm1 = luaT_gettmbyobj(L, p1, event); + const TValue *tm2; + if (ttisnil(tm1)) return -1; /* no metamethod? */ + tm2 = luaT_gettmbyobj(L, p2, event); + if (!luaO_rawequalObj(tm1, tm2)) /* different metamethods? */ + return -1; + callTMres(L, L->top, tm1, p1, p2); + return !l_isfalse(L->top); +} + + +static int l_strcmp (const TString *ls, const TString *rs) { + const char *l = getstr(ls); + size_t ll = ls->tsv.len; + const char *r = getstr(rs); + size_t lr = rs->tsv.len; + for (;;) { + int temp = strcoll(l, r); + if (temp != 0) return temp; + else { /* strings are equal up to a `\0' */ + size_t len = strlen(l); /* index of first `\0' in both strings */ + if (len == lr) /* r is finished? */ + return (len == ll) ? 0 : 1; + else if (len == ll) /* l is finished? */ + return -1; /* l is smaller than r (because r is not finished) */ + /* both strings longer than `len'; go on comparing (after the `\0') */ + len++; + l += len; ll -= len; r += len; lr -= len; + } + } +} + + +int luaV_lessthan (lua_State *L, const TValue *l, const TValue *r) { + int res; + if (ttype(l) != ttype(r)) + return luaG_ordererror(L, l, r); + else if (ttisnumber(l)) + return luai_numlt(nvalue(l), nvalue(r)); + else if (ttisstring(l)) + return l_strcmp(rawtsvalue(l), rawtsvalue(r)) < 0; + else if ((res = call_orderTM(L, l, r, TM_LT)) != -1) + return res; + return luaG_ordererror(L, l, r); +} + + +static int lessequal (lua_State *L, const TValue *l, const TValue *r) { + int res; + if (ttype(l) != ttype(r)) + return luaG_ordererror(L, l, r); + else if (ttisnumber(l)) + return luai_numle(nvalue(l), nvalue(r)); + else if (ttisstring(l)) + return l_strcmp(rawtsvalue(l), rawtsvalue(r)) <= 0; + else if ((res = call_orderTM(L, l, r, TM_LE)) != -1) /* first try `le' */ + return res; + else if ((res = call_orderTM(L, r, l, TM_LT)) != -1) /* else try `lt' */ + return !res; + return luaG_ordererror(L, l, r); +} + + +int luaV_equalval (lua_State *L, const TValue *t1, const TValue *t2) { + const TValue *tm; + lua_assert(ttype(t1) == ttype(t2)); + switch (ttype(t1)) { + case LUA_TNIL: return 1; + case LUA_TNUMBER: return luai_numeq(nvalue(t1), nvalue(t2)); + case LUA_TBOOLEAN: return bvalue(t1) == bvalue(t2); /* true must be 1 !! */ + case LUA_TLIGHTUSERDATA: return pvalue(t1) == pvalue(t2); + case LUA_TUSERDATA: { + if (uvalue(t1) == uvalue(t2)) return 1; + tm = get_compTM(L, uvalue(t1)->metatable, uvalue(t2)->metatable, + TM_EQ); + break; /* will try TM */ + } + case LUA_TTABLE: { + if (hvalue(t1) == hvalue(t2)) return 1; + tm = get_compTM(L, hvalue(t1)->metatable, hvalue(t2)->metatable, TM_EQ); + break; /* will try TM */ + } + default: return gcvalue(t1) == gcvalue(t2); + } + if (tm == NULL) return 0; /* no TM? */ + callTMres(L, L->top, tm, t1, t2); /* call TM */ + return !l_isfalse(L->top); +} + + +void luaV_concat (lua_State *L, int total, int last) { + do { + StkId top = L->base + last + 1; + int n = 2; /* number of elements handled in this pass (at least 2) */ + if (!(ttisstring(top-2) || ttisnumber(top-2)) || !tostring(L, top-1)) { + if (!call_binTM(L, top-2, top-1, top-2, TM_CONCAT)) + luaG_concaterror(L, top-2, top-1); + } else if (tsvalue(top-1)->len == 0) /* second op is empty? */ + (void)tostring(L, top - 2); /* result is first op (as string) */ + else { + /* at least two string values; get as many as possible */ + size_t tl = tsvalue(top-1)->len; + char *buffer; + int i; + /* collect total length */ + for (n = 1; n < total && tostring(L, top-n-1); n++) { + size_t l = tsvalue(top-n-1)->len; + if (l >= MAX_SIZET - tl) luaG_runerror(L, "string length overflow"); + tl += l; + } + buffer = luaZ_openspace(L, &G(L)->buff, tl); + tl = 0; + for (i=n; i>0; i--) { /* concat all strings */ + size_t l = tsvalue(top-i)->len; + memcpy(buffer+tl, svalue(top-i), l); + tl += l; + } + setsvalue2s(L, top-n, luaS_newlstr(L, buffer, tl)); + } + total -= n-1; /* got `n' strings to create 1 new */ + last -= n-1; + } while (total > 1); /* repeat until only 1 result left */ +} + + +static void Arith (lua_State *L, StkId ra, const TValue *rb, + const TValue *rc, TMS op) { + TValue tempb, tempc; + const TValue *b, *c; + if ((b = luaV_tonumber(rb, &tempb)) != NULL && + (c = luaV_tonumber(rc, &tempc)) != NULL) { + lua_Number nb = nvalue(b), nc = nvalue(c); + switch (op) { + case TM_ADD: setnvalue(ra, luai_numadd(nb, nc)); break; + case TM_SUB: setnvalue(ra, luai_numsub(nb, nc)); break; + case TM_MUL: setnvalue(ra, luai_nummul(nb, nc)); break; + case TM_DIV: setnvalue(ra, luai_numdiv(nb, nc)); break; + case TM_MOD: setnvalue(ra, luai_nummod(nb, nc)); break; + case TM_POW: setnvalue(ra, luai_numpow(nb, nc)); break; + case TM_UNM: setnvalue(ra, luai_numunm(nb)); break; + default: lua_assert(0); break; + } + } + else if (!call_binTM(L, rb, rc, ra, op)) + luaG_aritherror(L, rb, rc); +} + + + +/* +** some macros for common tasks in `luaV_execute' +*/ + +#define runtime_check(L, c) { if (!(c)) break; } + +#define RA(i) (base+GETARG_A(i)) +/* to be used after possible stack reallocation */ +#define RB(i) check_exp(getBMode(GET_OPCODE(i)) == OpArgR, base+GETARG_B(i)) +#define RC(i) check_exp(getCMode(GET_OPCODE(i)) == OpArgR, base+GETARG_C(i)) +#define RKB(i) check_exp(getBMode(GET_OPCODE(i)) == OpArgK, \ + ISK(GETARG_B(i)) ? k+INDEXK(GETARG_B(i)) : base+GETARG_B(i)) +#define RKC(i) check_exp(getCMode(GET_OPCODE(i)) == OpArgK, \ + ISK(GETARG_C(i)) ? k+INDEXK(GETARG_C(i)) : base+GETARG_C(i)) +#define KBx(i) check_exp(getBMode(GET_OPCODE(i)) == OpArgK, k+GETARG_Bx(i)) + + +#define dojump(L,pc,i) {(pc) += (i); luai_threadyield(L);} + + +#define Protect(x) { L->savedpc = pc; {x;}; base = L->base; } + + +#define arith_op(op,tm) { \ + TValue *rb = RKB(i); \ + TValue *rc = RKC(i); \ + if (ttisnumber(rb) && ttisnumber(rc)) { \ + lua_Number nb = nvalue(rb), nc = nvalue(rc); \ + setnvalue(ra, op(nb, nc)); \ + } \ + else \ + Protect(Arith(L, ra, rb, rc, tm)); \ + } + + + +void luaV_execute (lua_State *L, int nexeccalls) { + LClosure *cl; + StkId base; + TValue *k; + const Instruction *pc; + reentry: /* entry point */ + lua_assert(isLua(L->ci)); + pc = L->savedpc; + cl = &clvalue(L->ci->func)->l; + base = L->base; + k = cl->p->k; + /* main loop of interpreter */ + for (;;) { + const Instruction i = *pc++; + StkId ra; + if ((L->hookmask & (LUA_MASKLINE | LUA_MASKCOUNT)) && + (--L->hookcount == 0 || L->hookmask & LUA_MASKLINE)) { + traceexec(L, pc); + if (L->status == LUA_YIELD) { /* did hook yield? */ + L->savedpc = pc - 1; + return; + } + base = L->base; + } + /* warning!! several calls may realloc the stack and invalidate `ra' */ + ra = RA(i); + lua_assert(base == L->base && L->base == L->ci->base); + lua_assert(base <= L->top && L->top <= L->stack + L->stacksize); + lua_assert(L->top == L->ci->top || luaG_checkopenop(i)); + switch (GET_OPCODE(i)) { + case OP_MOVE: { + setobjs2s(L, ra, RB(i)); + continue; + } + case OP_LOADK: { + setobj2s(L, ra, KBx(i)); + continue; + } + case OP_LOADBOOL: { + setbvalue(ra, GETARG_B(i)); + if (GETARG_C(i)) pc++; /* skip next instruction (if C) */ + continue; + } + case OP_LOADNIL: { + TValue *rb = RB(i); + do { + setnilvalue(rb--); + } while (rb >= ra); + continue; + } + case OP_GETUPVAL: { + int b = GETARG_B(i); + setobj2s(L, ra, cl->upvals[b]->v); + continue; + } + case OP_GETGLOBAL: { + TValue g; + TValue *rb = KBx(i); + sethvalue(L, &g, cl->env); + lua_assert(ttisstring(rb)); + Protect(luaV_gettable(L, &g, rb, ra)); + continue; + } + case OP_GETTABLE: { + Protect(luaV_gettable(L, RB(i), RKC(i), ra)); + continue; + } + case OP_SETGLOBAL: { + TValue g; + sethvalue(L, &g, cl->env); + lua_assert(ttisstring(KBx(i))); + Protect(luaV_settable(L, &g, KBx(i), ra)); + continue; + } + case OP_SETUPVAL: { + UpVal *uv = cl->upvals[GETARG_B(i)]; + setobj(L, uv->v, ra); + luaC_barrier(L, uv, ra); + continue; + } + case OP_SETTABLE: { + Protect(luaV_settable(L, ra, RKB(i), RKC(i))); + continue; + } + case OP_NEWTABLE: { + int b = GETARG_B(i); + int c = GETARG_C(i); + sethvalue(L, ra, luaH_new(L, luaO_fb2int(b), luaO_fb2int(c))); + Protect(luaC_checkGC(L)); + continue; + } + case OP_SELF: { + StkId rb = RB(i); + setobjs2s(L, ra+1, rb); + Protect(luaV_gettable(L, rb, RKC(i), ra)); + continue; + } + case OP_ADD: { + arith_op(luai_numadd, TM_ADD); + continue; + } + case OP_SUB: { + arith_op(luai_numsub, TM_SUB); + continue; + } + case OP_MUL: { + arith_op(luai_nummul, TM_MUL); + continue; + } + case OP_DIV: { + arith_op(luai_numdiv, TM_DIV); + continue; + } + case OP_MOD: { + arith_op(luai_nummod, TM_MOD); + continue; + } + case OP_POW: { + arith_op(luai_numpow, TM_POW); + continue; + } + case OP_UNM: { + TValue *rb = RB(i); + if (ttisnumber(rb)) { + lua_Number nb = nvalue(rb); + setnvalue(ra, luai_numunm(nb)); + } + else { + Protect(Arith(L, ra, rb, rb, TM_UNM)); + } + continue; + } + case OP_NOT: { + int res = l_isfalse(RB(i)); /* next assignment may change this value */ + setbvalue(ra, res); + continue; + } + case OP_LEN: { + const TValue *rb = RB(i); + switch (ttype(rb)) { + case LUA_TTABLE: { + setnvalue(ra, cast_num(luaH_getn(hvalue(rb)))); + break; + } + case LUA_TSTRING: { + setnvalue(ra, cast_num(tsvalue(rb)->len)); + break; + } + default: { /* try metamethod */ + Protect( + if (!call_binTM(L, rb, luaO_nilobject, ra, TM_LEN)) + luaG_typeerror(L, rb, "get length of"); + ) + } + } + continue; + } + case OP_CONCAT: { + int b = GETARG_B(i); + int c = GETARG_C(i); + Protect(luaV_concat(L, c-b+1, c); luaC_checkGC(L)); + setobjs2s(L, RA(i), base+b); + continue; + } + case OP_JMP: { + dojump(L, pc, GETARG_sBx(i)); + continue; + } + case OP_EQ: { + TValue *rb = RKB(i); + TValue *rc = RKC(i); + Protect( + if (equalobj(L, rb, rc) == GETARG_A(i)) + dojump(L, pc, GETARG_sBx(*pc)); + ) + pc++; + continue; + } + case OP_LT: { + Protect( + if (luaV_lessthan(L, RKB(i), RKC(i)) == GETARG_A(i)) + dojump(L, pc, GETARG_sBx(*pc)); + ) + pc++; + continue; + } + case OP_LE: { + Protect( + if (lessequal(L, RKB(i), RKC(i)) == GETARG_A(i)) + dojump(L, pc, GETARG_sBx(*pc)); + ) + pc++; + continue; + } + case OP_TEST: { + if (l_isfalse(ra) != GETARG_C(i)) + dojump(L, pc, GETARG_sBx(*pc)); + pc++; + continue; + } + case OP_TESTSET: { + TValue *rb = RB(i); + if (l_isfalse(rb) != GETARG_C(i)) { + setobjs2s(L, ra, rb); + dojump(L, pc, GETARG_sBx(*pc)); + } + pc++; + continue; + } + case OP_CALL: { + int b = GETARG_B(i); + int nresults = GETARG_C(i) - 1; + if (b != 0) L->top = ra+b; /* else previous instruction set top */ + L->savedpc = pc; + switch (luaD_precall(L, ra, nresults)) { + case PCRLUA: { + nexeccalls++; + goto reentry; /* restart luaV_execute over new Lua function */ + } + case PCRC: { + /* it was a C function (`precall' called it); adjust results */ + if (nresults >= 0) L->top = L->ci->top; + base = L->base; + continue; + } + default: { + return; /* yield */ + } + } + } + case OP_TAILCALL: { + int b = GETARG_B(i); + if (b != 0) L->top = ra+b; /* else previous instruction set top */ + L->savedpc = pc; + lua_assert(GETARG_C(i) - 1 == LUA_MULTRET); + switch (luaD_precall(L, ra, LUA_MULTRET)) { + case PCRLUA: { + /* tail call: put new frame in place of previous one */ + CallInfo *ci = L->ci - 1; /* previous frame */ + int aux; + StkId func = ci->func; + StkId pfunc = (ci+1)->func; /* previous function index */ + if (L->openupval) luaF_close(L, ci->base); + L->base = ci->base = ci->func + ((ci+1)->base - pfunc); + for (aux = 0; pfunc+aux < L->top; aux++) /* move frame down */ + setobjs2s(L, func+aux, pfunc+aux); + ci->top = L->top = func+aux; /* correct top */ + lua_assert(L->top == L->base + clvalue(func)->l.p->maxstacksize); + ci->savedpc = L->savedpc; + ci->tailcalls++; /* one more call lost */ + L->ci--; /* remove new frame */ + goto reentry; + } + case PCRC: { /* it was a C function (`precall' called it) */ + base = L->base; + continue; + } + default: { + return; /* yield */ + } + } + } + case OP_RETURN: { + int b = GETARG_B(i); + if (b != 0) L->top = ra+b-1; + if (L->openupval) luaF_close(L, base); + L->savedpc = pc; + b = luaD_poscall(L, ra); + if (--nexeccalls == 0) /* was previous function running `here'? */ + return; /* no: return */ + else { /* yes: continue its execution */ + if (b) L->top = L->ci->top; + lua_assert(isLua(L->ci)); + lua_assert(GET_OPCODE(*((L->ci)->savedpc - 1)) == OP_CALL); + goto reentry; + } + } + case OP_FORLOOP: { + lua_Number step = nvalue(ra+2); + lua_Number idx = luai_numadd(nvalue(ra), step); /* increment index */ + lua_Number limit = nvalue(ra+1); + if (luai_numlt(0, step) ? luai_numle(idx, limit) + : luai_numle(limit, idx)) { + dojump(L, pc, GETARG_sBx(i)); /* jump back */ + setnvalue(ra, idx); /* update internal index... */ + setnvalue(ra+3, idx); /* ...and external index */ + } + continue; + } + case OP_FORPREP: { + const TValue *init = ra; + const TValue *plimit = ra+1; + const TValue *pstep = ra+2; + L->savedpc = pc; /* next steps may throw errors */ + if (!tonumber(init, ra)) + luaG_runerror(L, LUA_QL("for") " initial value must be a number"); + else if (!tonumber(plimit, ra+1)) + luaG_runerror(L, LUA_QL("for") " limit must be a number"); + else if (!tonumber(pstep, ra+2)) + luaG_runerror(L, LUA_QL("for") " step must be a number"); + setnvalue(ra, luai_numsub(nvalue(ra), nvalue(pstep))); + dojump(L, pc, GETARG_sBx(i)); + continue; + } + case OP_TFORLOOP: { + StkId cb = ra + 3; /* call base */ + setobjs2s(L, cb+2, ra+2); + setobjs2s(L, cb+1, ra+1); + setobjs2s(L, cb, ra); + L->top = cb+3; /* func. + 2 args (state and index) */ + Protect(luaD_call(L, cb, GETARG_C(i))); + L->top = L->ci->top; + cb = RA(i) + 3; /* previous call may change the stack */ + if (!ttisnil(cb)) { /* continue loop? */ + setobjs2s(L, cb-1, cb); /* save control variable */ + dojump(L, pc, GETARG_sBx(*pc)); /* jump back */ + } + pc++; + continue; + } + case OP_SETLIST: { + int n = GETARG_B(i); + int c = GETARG_C(i); + int last; + Table *h; + if (n == 0) { + n = cast_int(L->top - ra) - 1; + L->top = L->ci->top; + } + if (c == 0) c = cast_int(*pc++); + runtime_check(L, ttistable(ra)); + h = hvalue(ra); + last = ((c-1)*LFIELDS_PER_FLUSH) + n; + if (last > h->sizearray) /* needs more space? */ + luaH_resizearray(L, h, last); /* pre-alloc it at once */ + for (; n > 0; n--) { + TValue *val = ra+n; + setobj2t(L, luaH_setnum(L, h, last--), val); + luaC_barriert(L, h, val); + } + continue; + } + case OP_CLOSE: { + luaF_close(L, ra); + continue; + } + case OP_CLOSURE: { + Proto *p; + Closure *ncl; + int nup, j; + p = cl->p->p[GETARG_Bx(i)]; + nup = p->nups; + ncl = luaF_newLclosure(L, nup, cl->env); + ncl->l.p = p; + for (j=0; jl.upvals[j] = cl->upvals[GETARG_B(*pc)]; + else { + lua_assert(GET_OPCODE(*pc) == OP_MOVE); + ncl->l.upvals[j] = luaF_findupval(L, base + GETARG_B(*pc)); + } + } + setclvalue(L, ra, ncl); + Protect(luaC_checkGC(L)); + continue; + } + case OP_VARARG: { + int b = GETARG_B(i) - 1; + int j; + CallInfo *ci = L->ci; + int n = cast_int(ci->base - ci->func) - cl->p->numparams - 1; + if (b == LUA_MULTRET) { + Protect(luaD_checkstack(L, n)); + ra = RA(i); /* previous call may change the stack */ + b = n; + L->top = ra + n; + } + for (j = 0; j < b; j++) { + if (j < n) { + setobjs2s(L, ra + j, ci->base - n + j); + } + else { + setnilvalue(ra + j); + } + } + continue; + } + } + } +} + diff --git a/src/lua/lvm.h b/src/lua/lvm.h new file mode 100644 index 0000000..bfe4f56 --- /dev/null +++ b/src/lua/lvm.h @@ -0,0 +1,36 @@ +/* +** $Id: lvm.h,v 2.5.1.1 2007/12/27 13:02:25 roberto Exp $ +** Lua virtual machine +** See Copyright Notice in lua.h +*/ + +#ifndef lvm_h +#define lvm_h + + +#include "ldo.h" +#include "lobject.h" +#include "ltm.h" + + +#define tostring(L,o) ((ttype(o) == LUA_TSTRING) || (luaV_tostring(L, o))) + +#define tonumber(o,n) (ttype(o) == LUA_TNUMBER || \ + (((o) = luaV_tonumber(o,n)) != NULL)) + +#define equalobj(L,o1,o2) \ + (ttype(o1) == ttype(o2) && luaV_equalval(L, o1, o2)) + + +LUAI_FUNC int luaV_lessthan (lua_State *L, const TValue *l, const TValue *r); +LUAI_FUNC int luaV_equalval (lua_State *L, const TValue *t1, const TValue *t2); +LUAI_FUNC const TValue *luaV_tonumber (const TValue *obj, TValue *n); +LUAI_FUNC int luaV_tostring (lua_State *L, StkId obj); +LUAI_FUNC void luaV_gettable (lua_State *L, const TValue *t, TValue *key, + StkId val); +LUAI_FUNC void luaV_settable (lua_State *L, const TValue *t, TValue *key, + StkId val); +LUAI_FUNC void luaV_execute (lua_State *L, int nexeccalls); +LUAI_FUNC void luaV_concat (lua_State *L, int total, int last); + +#endif diff --git a/src/lua/lzio.c b/src/lua/lzio.c new file mode 100644 index 0000000..293edd5 --- /dev/null +++ b/src/lua/lzio.c @@ -0,0 +1,82 @@ +/* +** $Id: lzio.c,v 1.31.1.1 2007/12/27 13:02:25 roberto Exp $ +** a generic input stream interface +** See Copyright Notice in lua.h +*/ + + +#include + +#define lzio_c +#define LUA_CORE + +#include "lua.h" + +#include "llimits.h" +#include "lmem.h" +#include "lstate.h" +#include "lzio.h" + + +int luaZ_fill (ZIO *z) { + size_t size; + lua_State *L = z->L; + const char *buff; + lua_unlock(L); + buff = z->reader(L, z->data, &size); + lua_lock(L); + if (buff == NULL || size == 0) return EOZ; + z->n = size - 1; + z->p = buff; + return char2int(*(z->p++)); +} + + +int luaZ_lookahead (ZIO *z) { + if (z->n == 0) { + if (luaZ_fill(z) == EOZ) + return EOZ; + else { + z->n++; /* luaZ_fill removed first byte; put back it */ + z->p--; + } + } + return char2int(*z->p); +} + + +void luaZ_init (lua_State *L, ZIO *z, lua_Reader reader, void *data) { + z->L = L; + z->reader = reader; + z->data = data; + z->n = 0; + z->p = NULL; +} + + +/* --------------------------------------------------------------- read --- */ +size_t luaZ_read (ZIO *z, void *b, size_t n) { + while (n) { + size_t m; + if (luaZ_lookahead(z) == EOZ) + return n; /* return number of missing bytes */ + m = (n <= z->n) ? n : z->n; /* min. between n and z->n */ + memcpy(b, z->p, m); + z->n -= m; + z->p += m; + b = (char *)b + m; + n -= m; + } + return 0; +} + +/* ------------------------------------------------------------------------ */ +char *luaZ_openspace (lua_State *L, Mbuffer *buff, size_t n) { + if (n > buff->buffsize) { + if (n < LUA_MINBUFFER) n = LUA_MINBUFFER; + luaZ_resizebuffer(L, buff, n); + } + return buff->buffer; +} + + diff --git a/src/lua/lzio.h b/src/lua/lzio.h new file mode 100644 index 0000000..51d695d --- /dev/null +++ b/src/lua/lzio.h @@ -0,0 +1,67 @@ +/* +** $Id: lzio.h,v 1.21.1.1 2007/12/27 13:02:25 roberto Exp $ +** Buffered streams +** See Copyright Notice in lua.h +*/ + + +#ifndef lzio_h +#define lzio_h + +#include "lua.h" + +#include "lmem.h" + + +#define EOZ (-1) /* end of stream */ + +typedef struct Zio ZIO; + +#define char2int(c) cast(int, cast(unsigned char, (c))) + +#define zgetc(z) (((z)->n--)>0 ? char2int(*(z)->p++) : luaZ_fill(z)) + +typedef struct Mbuffer { + char *buffer; + size_t n; + size_t buffsize; +} Mbuffer; + +#define luaZ_initbuffer(L, buff) ((buff)->buffer = NULL, (buff)->buffsize = 0) + +#define luaZ_buffer(buff) ((buff)->buffer) +#define luaZ_sizebuffer(buff) ((buff)->buffsize) +#define luaZ_bufflen(buff) ((buff)->n) + +#define luaZ_resetbuffer(buff) ((buff)->n = 0) + + +#define luaZ_resizebuffer(L, buff, size) \ + (luaM_reallocvector(L, (buff)->buffer, (buff)->buffsize, size, char), \ + (buff)->buffsize = size) + +#define luaZ_freebuffer(L, buff) luaZ_resizebuffer(L, buff, 0) + + +LUAI_FUNC char *luaZ_openspace (lua_State *L, Mbuffer *buff, size_t n); +LUAI_FUNC void luaZ_init (lua_State *L, ZIO *z, lua_Reader reader, + void *data); +LUAI_FUNC size_t luaZ_read (ZIO* z, void* b, size_t n); /* read next n bytes */ +LUAI_FUNC int luaZ_lookahead (ZIO *z); + + + +/* --------- Private Part ------------------ */ + +struct Zio { + size_t n; /* bytes still unread */ + const char *p; /* current position in buffer */ + lua_Reader reader; + void* data; /* additional data */ + lua_State *L; /* Lua state (for reader) */ +}; + + +LUAI_FUNC int luaZ_fill (ZIO *z); + +#endif diff --git a/src/lua_bloom_filter.c b/src/lua_bloom_filter.c deleted file mode 100644 index 2a36474..0000000 --- a/src/lua_bloom_filter.c +++ /dev/null @@ -1,207 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua bloom_filter implementation @file */ - -#include -#include -#include -#include -#include - -#include "lua_bloom_filter.h" -#include "lua_serialize.h" -#include "xxhash.h" - -const char* lsb_bloom_filter = "lsb.bloom_filter"; -const char* lsb_bloom_filter_table = "bloom_filter"; - -struct bloom_filter -{ - size_t items; - size_t bytes; - size_t bits; - unsigned int hashes; - double probability; - char data[]; -}; - - -static int bloom_filter_new(lua_State* lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n == 2, 0, "incorrect number of arguments"); - int items = luaL_checkint(lua, 1); - luaL_argcheck(lua, 1 < items, 1, "items must be > 1"); - double probability = luaL_checknumber(lua, 2); - luaL_argcheck(lua, 0 < probability && 1 > probability, 2, "probability must be between 0 and 1"); - - size_t bits = (size_t)ceil(items * log(probability) / log(1 / pow(2, log(2)))); - size_t bytes = (size_t)ceil((double)bits / CHAR_BIT); - unsigned int hashes = (unsigned int)round(log(2) * bits/items); - - // subtract 1 for the byte already included in the struct - size_t nbytes = sizeof(bloom_filter) + bytes; - bloom_filter* bf = (bloom_filter*)lua_newuserdata(lua, nbytes); - bf->items = items; - bf->bits = bits; - bf->bytes = bytes; - bf->hashes = hashes; - bf->probability = probability; - memset(bf->data, 0, bf->bytes); - - luaL_getmetatable(lua, lsb_bloom_filter); - lua_setmetatable(lua, -2); - - return 1; -} - - -static bloom_filter* check_bloom_filter(lua_State* lua, int args) -{ - void* ud = luaL_checkudata(lua, 1, lsb_bloom_filter); - luaL_argcheck(lua, ud != NULL, 1, "invalid userdata type"); - luaL_argcheck(lua, args == lua_gettop(lua), 0, - "incorrect number of arguments"); - return (bloom_filter*)ud; -} - - -static int bloom_filter_add(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 2); - size_t len = 0; - double val = 0; - void *key = NULL; - switch (lua_type(lua, 2)) { - case LUA_TSTRING: - key = (void*) lua_tolstring(lua, 2, &len); - break; - case LUA_TNUMBER: - val = lua_tonumber(lua, 2); - len = sizeof(double); - key = &val; - break; - default: - luaL_argerror(lua, 2, "must be a string or number"); - break; - } - unsigned int bit = 0; - int added = 0; - - for (unsigned int i = 0; i < bf->hashes; ++i) { - bit = XXH32(key, (int)len, i) % bf->bits; - if (!(bf->data[bit/CHAR_BIT] & 1<<(bit%CHAR_BIT))) { - bf->data[bit/CHAR_BIT] |= 1<<(bit%CHAR_BIT); - added = 1; - } - } - - lua_pushboolean(lua, added); - return 1; -} - - -static int bloom_filter_query(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 2); - size_t len = 0; - double val = 0; - void *key = NULL; - switch (lua_type(lua, 2)) { - case LUA_TSTRING: - key = (void*) lua_tolstring(lua, 2, &len); - break; - case LUA_TNUMBER: - val = lua_tonumber(lua, 2); - len = sizeof(double); - key = &val; - break; - default: - luaL_argerror(lua, 2, "must be a string or number"); - break; - } - unsigned int bit = 0; - int found = 1; - - for (unsigned int i = 0; i < bf->hashes && found; ++i) { - bit = XXH32(key, (int)len, i) % bf->bits; - found = bf->data[bit/CHAR_BIT] & 1<<(bit%CHAR_BIT); - } - - lua_pushboolean(lua, found); - return 1; -} - - -static int bloom_filter_clear(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 1); - memset(bf->data, 0, bf->bytes); - return 0; -} - -static int bloom_filter_fromstring(lua_State* lua) -{ - bloom_filter* bf = check_bloom_filter(lua, 2); - size_t len = 0; - const char* values = luaL_checklstring(lua, 2, &len); - if (len != bf->bytes) { - luaL_error(lua, "fromstring() bytes found: %d, expected %d", len, bf->bytes); - } - memcpy(bf->data, values, len); - return 0; -} - - -int serialize_bloom_filter(const char* key, bloom_filter* bf, output_data* output) -{ - output->pos = 0; - if (appendf(output, - "if %s == nil then %s = bloom_filter.new(%d, %g) end\n", - key, - key, - bf->items, - bf->probability)) { - return 1; - } - - if (appendf(output, "%s:fromstring(\"", key)) { - return 1; - } - if (serialize_binary(bf->data, bf->bytes, output)) return 1; - if (appends(output, "\")\n")) { - return 1; - } - return 0; -} - - -static const struct luaL_reg bloom_filterlib_f[] = -{ - { "new", bloom_filter_new } - , { NULL, NULL } -}; - -static const struct luaL_reg bloom_filterlib_m[] = -{ - { "add", bloom_filter_add } - , { "query", bloom_filter_query } - , { "clear", bloom_filter_clear } - , { "fromstring", bloom_filter_fromstring } // used for data restoration - , { NULL, NULL } -}; - - -int luaopen_bloom_filter(lua_State* lua) -{ - luaL_newmetatable(lua, lsb_bloom_filter); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, bloom_filterlib_m); - luaL_register(lua, lsb_bloom_filter_table, bloom_filterlib_f); - return 1; -} diff --git a/src/lua_bloom_filter.h b/src/lua_bloom_filter.h deleted file mode 100644 index 9b380c1..0000000 --- a/src/lua_bloom_filter.h +++ /dev/null @@ -1,43 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua bloom_filter filter @file */ - -#ifndef lua_bloom_filter_h_ -#define lua_bloom_filter_h_ - -#include -#include "lua_sandbox_private.h" - -extern const char* lsb_bloom_filter; -extern const char* lsb_bloom_filter_table; -typedef struct bloom_filter bloom_filter; - -/** - * Serialize bloom_filter data - * - * @param key Lua variable name. - * @param b Bloom userdata object. - * @param output Output stream where the data is written. - * @return Zero on success - * - */ -int serialize_bloom_filter(const char* key, bloom_filter* bf, - output_data* output); - - -/** - * Bloom library loader - * - * @param lua Lua state. - * - * @return 1 on success - * - */ -int luaopen_bloom_filter(lua_State* lua); - - -#endif diff --git a/src/lua_circular_buffer.c b/src/lua_circular_buffer.c deleted file mode 100644 index 297933d..0000000 --- a/src/lua_circular_buffer.c +++ /dev/null @@ -1,1091 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// @brief Lua circular buffer implementation @file - -#include "cephes.h" -#include "lua_circular_buffer.h" -#include "lua_serialize.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -const char* lsb_circular_buffer = "lsb.circular_buffer"; -const char* lsb_circular_buffer_table = "circular_buffer"; - -#define COLUMN_NAME_SIZE 16 -#define UNIT_LABEL_SIZE 8 - -static const char* column_aggregation_methods[] = { "sum", "min", "max", "none", - "none", NULL }; -static const char* default_unit = "count"; - -typedef enum { - AGGREGATION_SUM = 0, - AGGREGATION_MIN = 1, - AGGREGATION_MAX = 2, - AGGREGATION_UNUSED = 3, - AGGREGATION_NONE = 4, - - MAX_AGGREGATION -} COLUMN_AGGREGATION; - -typedef enum { - OUTPUT_CBUF = 0, - OUTPUT_CBUFD = 1, - - LSB_OUTPUT_FORMAT -} OUTPUT_FORMAT; - -typedef struct -{ - char name[COLUMN_NAME_SIZE]; - char unit[UNIT_LABEL_SIZE]; - COLUMN_AGGREGATION aggregation; -} header_info; - -struct circular_buffer -{ - time_t current_time; - unsigned seconds_per_row; - unsigned current_row; - unsigned rows; - unsigned columns; - header_info* headers; - double* values; - int delta; - OUTPUT_FORMAT format; - int ref; - char bytes[1]; -}; - - -static time_t get_start_time(circular_buffer* cb) -{ - return cb->current_time - (cb->seconds_per_row * (cb->rows - 1)); -} - - -static void copy_cleared_row(circular_buffer* cb, double* cleared, size_t rows) -{ - size_t pool = 1; - size_t ask; - - while (rows > 0) { - if (rows >= pool) { - ask = pool; - } else { - ask = rows; - } - memcpy(cleared + (pool * cb->columns), cleared, sizeof(double) * cb->columns * ask); - rows -= ask; - pool += ask; - } -} - - -static void clear_rows(circular_buffer* cb, unsigned num_rows) -{ - if (num_rows >= cb->rows) { - num_rows = cb->rows; - } - unsigned row = cb->current_row; - ++row; - if (row >= cb->rows) {row = 0;} - for (unsigned c = 0; c < cb->columns; ++c) { - cb->values[(row * cb->columns) + c] = NAN; - } - double* cleared = &cb->values[row * cb->columns]; - if (row + num_rows - 1 >= cb->rows) { - copy_cleared_row(cb, cleared, cb->rows - row - 1); - for (unsigned c = 0; c < cb->columns; ++c) { - cb->values[c] = NAN; - } - copy_cleared_row(cb, cb->values, row + num_rows - 1 - cb->rows); - } else { - copy_cleared_row(cb, cleared, num_rows - 1); - } -} - - -static int circular_buffer_new(lua_State* lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n >= 3 && n <= 4, 0, "incorrect number of arguments"); - int rows = luaL_checkint(lua, 1); - luaL_argcheck(lua, 1 < rows, 1, "rows must be > 1"); - int columns = luaL_checkint(lua, 2); - luaL_argcheck(lua, 0 < columns, 2, "columns must be > 0"); - int seconds_per_row = luaL_checkint(lua, 3); - luaL_argcheck(lua, 0 < seconds_per_row, 3, "seconds_per_row is out of range"); - int delta = 0; - if (4 == n) { - delta = lua_toboolean(lua, 4); - } - - size_t header_bytes = sizeof(header_info) * columns; - size_t buffer_bytes = sizeof(double) * rows * columns; - size_t struct_bytes = sizeof(circular_buffer) - 1; // subtract 1 for the - // byte already included - // in the struct - - size_t nbytes = header_bytes + buffer_bytes + struct_bytes; - circular_buffer* cb = (circular_buffer*)lua_newuserdata(lua, nbytes); - cb->ref = LUA_NOREF; - cb->delta = delta; - cb->format = OUTPUT_CBUF; - cb->headers = (header_info*)&cb->bytes[0]; - cb->values = (double*)&cb->bytes[header_bytes]; - - luaL_getmetatable(lua, lsb_circular_buffer); - lua_setmetatable(lua, -2); - - cb->current_time = seconds_per_row * (rows - 1); - cb->current_row = rows - 1; - cb->rows = rows; - cb->columns = columns; - cb->seconds_per_row = seconds_per_row; - memset(cb->bytes, 0, header_bytes); - for (unsigned column_idx = 0; column_idx < cb->columns; ++column_idx) { - snprintf(cb->headers[column_idx].name, COLUMN_NAME_SIZE, - "Column_%d", column_idx + 1); - strncpy(cb->headers[column_idx].unit, default_unit, - UNIT_LABEL_SIZE - 1); - } - clear_rows(cb, rows); - return 1; -} - - -static circular_buffer* check_circular_buffer(lua_State* lua, int min_args) -{ - void* ud = luaL_checkudata(lua, 1, lsb_circular_buffer); - luaL_argcheck(lua, ud != NULL, 1, "invalid userdata type"); - luaL_argcheck(lua, min_args <= lua_gettop(lua), 0, - "incorrect number of arguments"); - return (circular_buffer*)ud; -} - - -static int check_row(circular_buffer* cb, double ns, int advance) -{ - time_t t = (time_t)(ns / 1e9); - t = t - (t % cb->seconds_per_row); - - int current_row = (int)(cb->current_time / cb->seconds_per_row); - int requested_row = (int)(t / cb->seconds_per_row); - int row_delta = requested_row - current_row; - int row = requested_row % cb->rows; - - if (row_delta > 0 && advance) { - clear_rows(cb, row_delta); - cb->current_time = t; - cb->current_row = row; - } else if (requested_row > current_row - || abs(row_delta) >= (int)cb->rows) { - return -1; - } - return row; -} - - -static int check_column(lua_State* lua, circular_buffer* cb, int arg) -{ - unsigned column = luaL_checkint(lua, arg); - luaL_argcheck(lua, 1 <= column && column <= cb->columns, arg, - "column out of range"); - --column; // make zero based - return column; -} - - -static void circular_buffer_add_delta(lua_State* lua, circular_buffer* cb, - double ns, int column, double value) -{ - // Storing the deltas in a Lua table allows the sandbox to account for the - // memory usage. todo: if too inefficient use a C data struct and report - // memory usage back to the sandbox - time_t t = (time_t)(ns / 1e9); - t = t - (t % cb->seconds_per_row); - lua_getglobal(lua, lsb_circular_buffer_table); - if (lua_istable(lua, -1)) { - if (cb->ref == LUA_NOREF) { - lua_newtable(lua); - cb->ref = luaL_ref(lua, -2); - } - // get the delta table for this cbuf - lua_rawgeti(lua, -1, cb->ref); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 2); // remove bogus table and cbuf table - return; - } - - // get the delta row using the timestamp - lua_rawgeti(lua, -1, (int)t); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 1); // remove non table entry - lua_newtable(lua); - lua_rawseti(lua, -2, (int)t); - lua_rawgeti(lua, -1, (int)t); - } - - // get the previous delta value - lua_rawgeti(lua, -1, column); - value += lua_tonumber(lua, -1); - lua_pop(lua, 1); // remove the old value - - // push the new delta - lua_pushnumber(lua, value); - lua_rawseti(lua, -2, column); - - lua_pop(lua, 2); // remove ref table, timestamped row - } else { - luaL_error(lua, "Could not find table %s", lsb_circular_buffer_table); - } - lua_pop(lua, 1); // remove the circular buffer table or failed nil - return; -} - - -static int circular_buffer_add(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 4); - double ns = luaL_checknumber(lua, 2); - int row = check_row(cb, - ns, - 1); // advance the buffer forward if - // necessary - int column = check_column(lua, cb, 3); - double value = luaL_checknumber(lua, 4); - if (row != -1) { - int i = (row * cb->columns) + column; - if (isnan(cb->values[i])) { - cb->values[i] = value; - } else { - cb->values[i] += value; - } - lua_pushnumber(lua, cb->values[i]); - if (cb->delta && value != 0) { - if (cb->headers[column].aggregation != AGGREGATION_SUM) { - value = cb->values[i]; - } - circular_buffer_add_delta(lua, cb, ns, column, value); - } - } else { - lua_pushnil(lua); - } - return 1; -} - - -static int circular_buffer_get(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 3); - int row = check_row(cb, - luaL_checknumber(lua, 2), - 0); - int column = check_column(lua, cb, 3); - - if (row != -1) { - lua_pushnumber(lua, cb->values[(row * cb->columns) + column]); - } else { - lua_pushnil(lua); - } - return 1; -} - - -static int circular_buffer_get_configuration(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 1); - - lua_pushnumber(lua, cb->rows); - lua_pushnumber(lua, cb->columns); - lua_pushnumber(lua, cb->seconds_per_row); - return 3; -} - - -static int circular_buffer_set(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 4); - double ns = luaL_checknumber(lua, 2); - int row = check_row(cb, ns, 1); // advance the buffer forward if - // necessary - int column = check_column(lua, cb, 3); - double value = luaL_checknumber(lua, 4); - - if (row != -1) { - int i = (row * cb->columns) + column; - double old = cb->values[i]; - switch (cb->headers[column].aggregation) { - case AGGREGATION_MIN: - if (isnan(cb->values[i]) || value < cb->values[i]) { - cb->values[i] = value; - if (cb->delta) { - circular_buffer_add_delta(lua, cb, ns, column, value); - } - } - break; - case AGGREGATION_MAX: - if (isnan(cb->values[i]) || value > cb->values[i]) { - cb->values[i] = value; - if (cb->delta) { - circular_buffer_add_delta(lua, cb, ns, column, value); - } - } - break; - default: - cb->values[i] = value; - if (cb->delta) { - if (!isnan(old)) { - value -= old; - } - circular_buffer_add_delta(lua, cb, ns, column, value); - } - break; - } - lua_pushnumber(lua, cb->values[i]); - } else { - lua_pushnil(lua); - } - return 1; -} - - -static int circular_buffer_set_header(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 3); - int column = check_column(lua, cb, 2); - const char* name = luaL_checkstring(lua, 3); - const char* unit = luaL_optstring(lua, 4, default_unit); - cb->headers[column].aggregation = luaL_checkoption(lua, 5, "sum", - column_aggregation_methods); - - strncpy(cb->headers[column].name, name, COLUMN_NAME_SIZE - 1); - char* n = cb->headers[column].name; - for (int j = 0; n[j] != 0; ++j) { - if (!isalnum(n[j])) { - n[j] = '_'; - } - } - strncpy(cb->headers[column].unit, unit, UNIT_LABEL_SIZE - 1); - n = cb->headers[column].unit; - for (int j = 0; n[j] != 0; ++j) { - if (n[j] != '/' && n[j] != '*' && !isalnum(n[j])) { - n[j] = '_'; - } - } - - lua_pushinteger(lua, column + 1); // return the 1 based Lua column - return 1; -} - - -static int circular_buffer_get_header(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 2); - int column = check_column(lua, cb, 2); - - lua_pushstring(lua, cb->headers[column].name); - lua_pushstring(lua, cb->headers[column].unit); - lua_pushstring(lua, - column_aggregation_methods[cb->headers[column].aggregation]); - return 3; -} - - -static double compute_sum(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double value = 0; - double result = 0; - unsigned row = start_row; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (isnan(value)) { - continue; - } - ++(*active_rows); - result += value; - } - while (row++ != end_row); - return result; -} - - -static double compute_avg(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double value = 0; - double result = 0; - unsigned row = start_row; - unsigned row_count = 0; - - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - result += value; - ++row_count; - } - } - while (row++ != end_row); - *active_rows = row_count; - return result / row_count; -} - - -static double compute_variance(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double avg = compute_avg(cb, column, start_row, end_row, active_rows); - if (isnan(avg)) return avg; - - double sum_squares = 0; - double value = 0; - unsigned row = start_row; - unsigned row_count = 0; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - value -= avg; - sum_squares += value * value; - ++row_count; - } - } - while (row++ != end_row); - return sum_squares / row_count; -} - - -static double compute_sd(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - return sqrt(compute_variance(cb, column, start_row, end_row, active_rows)); -} - - -static double compute_min(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double result = DBL_MAX; - double value = 0; - unsigned row = start_row; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - ++(*active_rows); - if (value < result) { - result = value; - } - } - } - while (row++ != end_row); - if (result == DBL_MAX) { - result = NAN; - } - return result; -} - - -static double compute_max(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - unsigned* active_rows) -{ - double result = DBL_MIN; - double value = 0; - unsigned row = start_row; - do { - if (row == cb->rows) { - row = 0; - } - value = cb->values[(row * cb->columns) + column]; - if (!isnan(value)) { - ++(*active_rows); - if (value > result) { - result = value; - } - } - } - while (row++ != end_row); - if (result == DBL_MIN) { - result = NAN; - } - return result; -} - - -static int circular_buffer_compute(lua_State* lua) -{ - static const char* functions[] = { "sum", "avg", "sd", "min", "max", - "variance", NULL }; - circular_buffer* cb = check_circular_buffer(lua, 3); - int function = luaL_checkoption(lua, 2, NULL, functions); - int column = check_column(lua, cb, 3); - - // optional range arguments - double start_ns = luaL_optnumber(lua, 4, get_start_time(cb) * 1e9); - double end_ns = luaL_optnumber(lua, 5, cb->current_time * 1e9); - luaL_argcheck(lua, end_ns >= start_ns, 5, "end must be >= start"); - - unsigned active_rows = 0; - int start_row = check_row(cb, start_ns, 0); - int end_row = check_row(cb, end_ns, 0); - if (-1 == start_row || -1 == end_row) { - lua_pushnil(lua); - lua_pushinteger(lua, active_rows); - return 2; - } - - double result = 0; - switch (function) { - case 0: - result = compute_sum(cb, column, start_row, end_row, &active_rows); - break; - case 1: - result = compute_avg(cb, column, start_row, end_row, &active_rows); - break; - case 2: - result = compute_sd(cb, column, start_row, end_row, &active_rows); - break; - case 3: - result = compute_min(cb, column, start_row, end_row, &active_rows); - break; - case 4: - result = compute_max(cb, column, start_row, end_row, &active_rows); - break; - case 5: - result = compute_variance(cb, column, start_row, end_row, &active_rows); - break; - } - - lua_pushnumber(lua, result); - lua_pushinteger(lua, active_rows); - return 2; -} - - -static void append_values(circular_buffer* cb, unsigned column, - unsigned start_row, unsigned end_row, - double ranked[]) -{ - unsigned row = start_row; - unsigned x = 0; - do { - if (row == cb->rows) { - row = 0; - } - ranked[x++] = cb->values[(row * cb->columns) + column]; - } - while (row++ != end_row); -} - - -static double rank_data(double* sorted[], size_t ranked_size) -{ - size_t next = 0, dupe_count = 0; - double tie_correction = 0; - for (size_t i = 0; i < ranked_size; ++i) { - next = i + 1; - if (i == ranked_size - 1 || (!(isnan(*sorted[i]) && isnan(*sorted[next])) - && *sorted[i] != *sorted[next])) { - if (dupe_count) { - double tie_rank = next - 0.5 * dupe_count; - for (size_t j = i - dupe_count; j < next; ++j) { - *sorted[j] = tie_rank; - } - ++dupe_count; - tie_correction += pow((double)dupe_count, 3.0) - dupe_count; - dupe_count = 0; - } else { - *sorted[i] = (double)next; - } - } else { - ++dupe_count; - } - } - tie_correction = 1.0 - tie_correction / (pow((double)ranked_size, 3.0) - ranked_size); - return tie_correction; -} - - -static int double_pp_compare(const void* a, const void* b) -{ - double* d1 = *(double**)a; - double* d2 = *(double**)b; - - if (isnan(*d1) && isnan(*d2)) return 0; - if (isnan(*d1)) return -1; - if (isnan(*d2)) return 1; - - if (*d1 < *d2) return -1; - if (*d1 == *d2) return 0; - return 1; -} - -// http://en.wikipedia.org/wiki/Mann-Whitney_U_test -static int circular_buffer_mannwhitneyu(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 6); - int n = lua_gettop(lua); - luaL_argcheck(lua, n <= 7, 0, "too many arguments"); - int column = check_column(lua, cb, 2); - - double start_1 = luaL_checknumber(lua, 3); - double end_1 = luaL_checknumber(lua, 4); - luaL_argcheck(lua, end_1 >= start_1, 4, "end_1 must be >= start_1"); - - double start_2 = luaL_checknumber(lua, 5); - double end_2 = luaL_checknumber(lua, 6); - luaL_argcheck(lua, end_2 >= start_2, 6, "end_2 must be >= start_2"); - luaL_argcheck(lua, end_1 < start_2 || end_2 < start_1, 4, "ranges must not overlap"); - - int use_continuity = 1; // optional argument - if (7 == n) { - luaL_argcheck(lua, lua_isboolean(lua, n), n, "use_continuity must be a boolean"); - use_continuity = lua_toboolean(lua, n); - } - - int start_1_row = check_row(cb, start_1, 0); - int end_1_row = check_row(cb, end_1, 0); - if (-1 == start_1_row || -1 == end_1_row) { - return 0; - } - int start_2_row = check_row(cb, start_2, 0); - int end_2_row = check_row(cb, end_2, 0); - if (-1 == start_2_row || -1 == end_2_row) { - return 0; - } - - size_t n1 = (size_t)((end_1 - start_1) / 1e9 / cb->seconds_per_row) + 1; - size_t n2 = (size_t)((end_2 - start_2) / 1e9 / cb->seconds_per_row) + 1; - size_t ranked_size = n1 + n2; - // note: user could temporarily exceed the sandbox memory limit here without - // detection - double* ranked = (double*)malloc(sizeof(double) * ranked_size); - if (!ranked) { - return 0; - } - append_values(cb, column, start_1_row, end_1_row, ranked); - append_values(cb, column, start_2_row, end_2_row, ranked + n1); - - double** sorted = (double**)malloc(sizeof(double*) * ranked_size); - if (!sorted) { - free(ranked); - return 0; - } - for (size_t i = 0; i < ranked_size; ++i) { - sorted[i] = ranked + i; - } - - qsort(sorted, ranked_size, sizeof(double*), double_pp_compare); - double tie_correction = rank_data(sorted, ranked_size); - free(sorted); - if (!tie_correction) { // data set values are all identical - free(ranked); - return 0; - } - - double sum = 0; - for (size_t i = 0; i < n1; ++i) { - if (!isnan(ranked[i])) { - sum += ranked[i]; - } - } - free(ranked); - - double u1 = sum - (n1 * (n1 + 1)) / 2.0; - double u2 = n1 * n2 - u1; - double lu = u1 > u2 ? u1 : u2; - - double z = 0; - double sd = sqrt(tie_correction * n1 * n2 * (n1 + n2 + 1) / 12.0); - if (use_continuity) { - // normal approximation for prob calc with continuity correction - z = fabs((lu - 0.5 - n1 * n2 / 2.0) / sd); - } else { - // normal approximation for prob calc - z = fabs((lu - n1 * n2 / 2.0) / sd); - } - - lua_pushnumber(lua, u1); - lua_pushnumber(lua, ndtr(-z)); - return 2; -} - - -static int circular_buffer_current_time(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 0); - lua_pushnumber(lua, cb->current_time * 1e9); - return 1; // return the current time -} - -static int circular_buffer_format(lua_State* lua) -{ - static const char* output_types[] = { "cbuf", "cbufd", NULL }; - circular_buffer* cb = check_circular_buffer(lua, 2); - luaL_argcheck(lua, 2 == lua_gettop(lua), 0, - "incorrect number of arguments"); - - cb->format = luaL_checkoption(lua, 2, NULL, output_types); - lua_pop(lua, 1); // remove the format - return 1; // return the circular buffer object -} - - -const char* get_output_format(circular_buffer* cb) -{ - switch (cb->format) { - case OUTPUT_CBUFD: - return "cbufd"; - default: - return "cbuf"; - } -} - - -static void read_time_row(char** p, circular_buffer* cb) -{ - cb->current_time = (time_t)strtoll(*p, &*p, 10); - cb->current_row = strtoul(*p, &*p, 10); -} - - -static int read_double(char** p, double* value) -{ - while (**p != 0 && isspace(**p)) { - ++*p; - } - if (0 == **p) return 0; - - if (**p == not_a_number[0]) { - ++*p; - if (0 == **p || **p != not_a_number[1]) return 0; - - ++*p; - if (0 == **p || **p != not_a_number[2]) return 0; - - ++*p; - *value = NAN; - } else { - *value = strtod(*p, &*p); - } - return 1; -} - - -static void circular_buffer_delta_fromstring(lua_State* lua, - circular_buffer* cb, - char** p) -{ - double value, ns = 0; - size_t pos = 0; - while (read_double(&*p, &value)) { - if (pos == 0) { // new row, starts with a time_t - ns = value * 1e9; - } else { - circular_buffer_add_delta(lua, cb, ns, (int)(pos - 1), value); - } - if (pos == cb->columns) { - pos = 0; - } else { - ++pos; - } - } - if (pos != 0) { - lua_pushstring(lua, "fromstring() invalid delta"); - lua_error(lua); - } - return; -} - - -static int circular_buffer_fromstring(lua_State* lua) -{ - circular_buffer* cb = check_circular_buffer(lua, 2); - const char* values = luaL_checkstring(lua, 2); - - char* p = (char*)values; - read_time_row(&p, cb); - - size_t pos = 0; - size_t len = cb->rows * cb->columns; - double value; - while (pos < len && read_double(&p, &value)) { - cb->values[pos++] = value; - } - if (pos == len) { - if (cb->delta) { - circular_buffer_delta_fromstring(lua, cb, &p); - } - } else { - luaL_error(lua, "fromstring() too few values: %d, expected %d", pos, len); - } - if (read_double(&p, &value)) { - luaL_error(lua, "fromstring() too many values, more than: %d", len); - } - return 0; -} - - -int output_circular_buffer_full(circular_buffer* cb, output_data* output) -{ - unsigned column_idx; - unsigned row_idx = cb->current_row + 1; - for (unsigned i = 0; i < cb->rows; ++i, ++row_idx) { - if (row_idx >= cb->rows) row_idx = 0; - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (column_idx != 0) { - if (appendc(output, '\t')) return 1; - } - if (serialize_double(output, - cb->values[(row_idx * cb->columns) + column_idx])) { - return 1; - } - } - if (appendc(output, '\n')) return 1; - } - return 0; -} - - -int output_circular_buffer_cbufd(lua_State* lua, circular_buffer* cb, - output_data* output) -{ - lua_getglobal(lua, lsb_circular_buffer_table); - if (lua_istable(lua, -1)) { - // get the delta table for this cbuf - lua_rawgeti(lua, -1, cb->ref); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 2); // remove bogus table and cbuf table - luaL_error(lua, "Could not find the delta table"); - } - lua_pushnil(lua); - while (lua_next(lua, -2) != 0) { - if (!lua_istable(lua, -1)) { - luaL_error(lua, "Invalid delta table structure"); - } - if (serialize_double(output, lua_tonumber(lua, -2))) return 1; - for (unsigned column_idx = 0; column_idx < cb->columns; - ++column_idx) { - if (appendc(output, '\t')) return 1; - lua_rawgeti(lua, -1, column_idx); - if (LUA_TNIL == lua_type(lua, -1)) { - if (appends(output, not_a_number)) return 1; - } else { - if (serialize_double(output, lua_tonumber(lua, -1))) return 1; - } - lua_pop(lua, 1); // remove the number - } - if (appendc(output, '\n')) return 1; - lua_pop(lua, 1); // remove the value, keep the key - } - lua_pop(lua, 1); // remove the delta table - - // delete the delta table - luaL_unref(lua, -1, cb->ref); - cb->ref = LUA_NOREF; - } else { - luaL_error(lua, "Could not find table %s", lsb_circular_buffer_table); - } - lua_pop(lua, 1); // remove the circular buffer table or failed nil - return 0; -} - - -int output_circular_buffer(lua_State* lua, circular_buffer* cb, - output_data* output) -{ - if (OUTPUT_CBUFD == cb->format) { - if (cb->ref == LUA_NOREF) return 0; - } - - if (appendf(output, - "{\"time\":%lld,\"rows\":%d,\"columns\":%d,\"seconds_per_row\":%d,\"column_info\":[", - (long long)get_start_time(cb), - cb->rows, - cb->columns, - cb->seconds_per_row)) { - return 1; - } - - unsigned column_idx; - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (column_idx != 0) { - if (appendc(output, ',')) return 1; - } - if (appendf(output, "{\"name\":\"%s\",\"unit\":\"%s\",\"aggregation\":\"%s\"}", - cb->headers[column_idx].name, - cb->headers[column_idx].unit, - column_aggregation_methods[cb->headers[column_idx].aggregation])) { - return 1; - } - } - if (appends(output, "]}\n")) return 1; - - if (OUTPUT_CBUFD == cb->format) { - return output_circular_buffer_cbufd(lua, cb, output); - } - return output_circular_buffer_full(cb, output); -} - - -int serialize_circular_buffer_delta(lua_State* lua, circular_buffer* cb, - output_data* output) -{ - if (cb->ref == LUA_NOREF) return 0; - lua_getglobal(lua, lsb_circular_buffer_table); - if (lua_istable(lua, -1)) { - // get the delta table for this cbuf - lua_rawgeti(lua, -1, cb->ref); - if (!lua_istable(lua, -1)) { - lua_pop(lua, 2); // remove bogus table and cbuf table - luaL_error(lua, "Could not find the delta table"); - } - lua_pushnil(lua); - while (lua_next(lua, -2) != 0) { - if (!lua_istable(lua, -1)) { - luaL_error(lua, "Invalid delta table structure"); - } - if (appendc(output, ' ')) return 1; - if (serialize_double(output, lua_tonumber(lua, -2))) return 1; - - for (unsigned column_idx = 0; column_idx < cb->columns; - ++column_idx) { - if (appends(output, " ")) return 1; - lua_rawgeti(lua, -1, column_idx); - if (serialize_double(output, lua_tonumber(lua, -1))) return 1; - lua_pop(lua, 1); // remove the number - } - lua_pop(lua, 1); // remove the value, keep the key - } - lua_pop(lua, 1); // remove the delta table - - // delete the delta table - lua_pushnil(lua); - lua_rawseti(lua, -2, cb->ref); - cb->ref = LUA_NOREF; - } else { - luaL_error(lua, "Could not find table %s", lsb_circular_buffer_table); - } - lua_pop(lua, 1); // remove the circular buffer table or failed nil - return 0; -} - - -int serialize_circular_buffer(lua_State* lua, const char* key, - circular_buffer* cb, output_data* output) -{ - output->pos = 0; - char* delta = ""; - if (cb->delta) { - delta = ", true"; - } - if (appendf(output, - "if %s == nil then %s = circular_buffer.new(%d, %d, %d%s) end\n", - key, - key, - cb->rows, - cb->columns, - cb->seconds_per_row, - delta)) { - return 1; - } - - unsigned column_idx; - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appendf(output, "%s:set_header(%d, \"%s\", \"%s\", \"%s\")\n", - key, - column_idx + 1, - cb->headers[column_idx].name, - cb->headers[column_idx].unit, - column_aggregation_methods[cb->headers[column_idx].aggregation])) { - return 1; - } - } - - if (appendf(output, "%s:fromstring(\"%lld %d", - key, - (long long)cb->current_time, - cb->current_row)) { - return 1; - } - for (unsigned row_idx = 0; row_idx < cb->rows; ++row_idx) { - for (column_idx = 0; column_idx < cb->columns; ++column_idx) { - if (appendc(output, ' ')) return 1; - if (serialize_double(output, - cb->values[(row_idx * cb->columns) + column_idx])) { - return 1; - } - } - } - if (serialize_circular_buffer_delta(lua, cb, output)) { - return 1; - } - if (appends(output, "\")\n")) { - return 1; - } - return 0; -} - - -static const struct luaL_reg circular_bufferlib_f[] = -{ - { "new", circular_buffer_new } - , { NULL, NULL } -}; - -static const struct luaL_reg circular_bufferlib_m[] = -{ - { "add", circular_buffer_add } - , { "get", circular_buffer_get } - , { "get_configuration", circular_buffer_get_configuration } - , { "set", circular_buffer_set } - , { "set_header", circular_buffer_set_header } - , { "get_header", circular_buffer_get_header } - , { "compute", circular_buffer_compute } - , { "mannwhitneyu", circular_buffer_mannwhitneyu } - , { "current_time", circular_buffer_current_time } - , { "format", circular_buffer_format } - , { "fromstring", circular_buffer_fromstring } // used for data restoration - , { NULL, NULL } -}; - - -int luaopen_circular_buffer(lua_State* lua) -{ - luaL_newmetatable(lua, lsb_circular_buffer); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, circular_bufferlib_m); - luaL_register(lua, lsb_circular_buffer_table, circular_bufferlib_f); - return 1; -} diff --git a/src/lua_circular_buffer.h b/src/lua_circular_buffer.h deleted file mode 100644 index 21beed9..0000000 --- a/src/lua_circular_buffer.h +++ /dev/null @@ -1,63 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// @brief Lua circular buffer - time series data store for sandboxes @file -#ifndef lua_circular_buffer_h_ -#define lua_circular_buffer_h_ - -#include -#include "lua_sandbox_private.h" - -extern const char* lsb_circular_buffer; -extern const char* lsb_circular_buffer_table; -typedef struct circular_buffer circular_buffer; - -/** - * Output the circular buffer user data - * - * @param lua Lua state. - * @param cb Circular buffer userdata object. - * @param output Output stream where the data is written. - * @return Zero on success - * - */ -int output_circular_buffer(lua_State* lua, circular_buffer* cb, - output_data* output); - -/** - * Get the current output format string. - * - * @param cb Circular buffer userdata object. - * - * @return const char* - */ -const char* get_output_format(circular_buffer* cb); - -/** - * Serialize the circular buffer user data - * - * @param lua Lua state. - * @param key Lua variable name. - * @param cb Circular buffer userdata object. - * @param output Output stream where the data is written. - * @return Zero on success - * - */ -int serialize_circular_buffer(lua_State* lua, const char* key, - circular_buffer* cb, output_data* output); - -/** - * Circular buffer library loader - * - * @param lua Lua state. - * - * @return 1 on success - * - */ -int luaopen_circular_buffer(lua_State* lua); - - -#endif diff --git a/src/lua_hyperloglog.c b/src/lua_hyperloglog.c deleted file mode 100644 index a00bf26..0000000 --- a/src/lua_hyperloglog.c +++ /dev/null @@ -1,193 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua hyperloglog implementation @file */ - -#include -#include -#include -#include - -#include "lua_hyperloglog.h" -#include "lua_serialize.h" - -const char* lsb_hyperloglog = "lsb.hyperloglog"; -const char* lsb_hyperloglog_table = "hyperloglog"; - -static const char* hll_magic = "HYLL"; - -/* The cached cardinality MSB is used to signal validity of the cached value. */ -#define HLL_INVALIDATE_CACHE(hll) (hll)->card[7] |= (1<<7) -#define HLL_VALID_CACHE(hll) (((hll)->card[7] & (1<<7)) == 0) - -static int hyperloglog_new(lua_State* lua) -{ - int n = lua_gettop(lua); - luaL_argcheck(lua, n == 0, 0, "incorrect number of arguments"); - - size_t nbytes = sizeof(hyperloglog); - hyperloglog* hll = (hyperloglog*)lua_newuserdata(lua, nbytes); - memcpy(hll->magic, hll_magic, sizeof(hll->magic)); - hll->encoding = HLL_DENSE; - HLL_INVALIDATE_CACHE(hll); - memset(hll->notused, 0, sizeof(hll->notused)); - memset(hll->registers, 0, HLL_REGISTERS_SIZE); - - luaL_getmetatable(lua, lsb_hyperloglog); - lua_setmetatable(lua, -2); - - return 1; -} - - -static hyperloglog* check_hyperloglog(lua_State* lua, int args) -{ - void* ud = luaL_checkudata(lua, 1, lsb_hyperloglog); - luaL_argcheck(lua, ud != NULL, 1, "invalid userdata type"); - luaL_argcheck(lua, args == lua_gettop(lua), 0, - "incorrect number of arguments"); - return (hyperloglog*)ud; -} - - -static int hyperloglog_add(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 2); - size_t len = 0; - double val = 0; - void* key = NULL; - switch (lua_type(lua, 2)) { - case LUA_TSTRING: - key = (void*)lua_tolstring(lua, 2, &len); - break; - case LUA_TNUMBER: - val = lua_tonumber(lua, 2); - len = sizeof(double); - key = &val; - break; - default: - luaL_argerror(lua, 2, "must be a string or number"); - break; - } - - int altered = 0; - if (1 == hllDenseAdd(hll->registers, (unsigned char*)key, len)) { - HLL_INVALIDATE_CACHE(hll); - altered = 1; - } - - lua_pushboolean(lua, altered); - return 1; -} - - -static int hyperloglog_count(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 1); - uint64_t card; - /* Check if the cached cardinality is valid. */ - if (HLL_VALID_CACHE(hll)) { - /* Just return the cached value. */ - card = (uint64_t)hll->card[0]; - card |= (uint64_t)hll->card[1] << 8; - card |= (uint64_t)hll->card[2] << 16; - card |= (uint64_t)hll->card[3] << 24; - card |= (uint64_t)hll->card[4] << 32; - card |= (uint64_t)hll->card[5] << 40; - card |= (uint64_t)hll->card[6] << 48; - card |= (uint64_t)hll->card[7] << 56; - } else { - /* Recompute it and update the cached value. */ - card = hllCount(hll); - hll->card[0] = card & 0xff; - hll->card[1] = (card >> 8) & 0xff; - hll->card[2] = (card >> 16) & 0xff; - hll->card[3] = (card >> 24) & 0xff; - hll->card[4] = (card >> 32) & 0xff; - hll->card[5] = (card >> 40) & 0xff; - hll->card[6] = (card >> 48) & 0xff; - hll->card[7] = (card >> 56) & 0xff; - } - - lua_pushnumber(lua, (double)card); - return 1; -} - - -static int hyperloglog_clear(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 1); - memset(hll->registers, 0, HLL_REGISTERS_SIZE); - HLL_INVALIDATE_CACHE(hll); - return 0; -} - - -static int hyperloglog_fromstring(lua_State* lua) -{ - hyperloglog* hll = check_hyperloglog(lua, 2); - size_t len = 0; - const char* values = luaL_checklstring(lua, 2, &len); - if (len != sizeof(hyperloglog) - 1) { - luaL_error(lua, "fromstring() bytes found: %d, expected %d", - len, sizeof(hyperloglog) - 1); - } - if (memcmp(values, hll_magic, sizeof(hll->magic)) != 0) { - luaL_error(lua, "fromstring() HYLL header not found"); - } - if (values[5] != HLL_DENSE) { - luaL_error(lua, "fromstring() invalid encoding"); - } - memcpy(hll, values, sizeof(hyperloglog) - 1); - return 0; -} - - -int serialize_hyperloglog(const char* key, hyperloglog* hll, output_data* output) -{ - output->pos = 0; - if (appendf(output, - "if %s == nil then %s = hyperloglog.new() end\n", key, key)) { - return 1; - } - - if (appendf(output, "%s:fromstring(\"", key)) { - return 1; - } - if (serialize_binary(hll, sizeof(hyperloglog) - 1, output)) return 1; - if (appends(output, "\")\n")) { - return 1; - } - return 0; -} - - -static const struct luaL_reg hyperlogloglib_f[] = -{ - { "new", hyperloglog_new } - , { NULL, NULL } -}; - - -static const struct luaL_reg hyperlogloglib_m[] = -{ - { "add", hyperloglog_add } - , { "count", hyperloglog_count } - , { "clear", hyperloglog_clear } - , { "fromstring", hyperloglog_fromstring } // used for data restoration - , { NULL, NULL } -}; - - -int luaopen_hyperloglog(lua_State* lua) -{ - luaL_newmetatable(lua, lsb_hyperloglog); - lua_pushvalue(lua, -1); - lua_setfield(lua, -2, "__index"); - luaL_register(lua, NULL, hyperlogloglib_m); - luaL_register(lua, lsb_hyperloglog_table, hyperlogloglib_f); - return 1; -} diff --git a/src/lua_hyperloglog.h b/src/lua_hyperloglog.h deleted file mode 100644 index f07b8cd..0000000 --- a/src/lua_hyperloglog.h +++ /dev/null @@ -1,84 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua HyperLogLog probabilistic cardinality approximation leveraging - * the Redis HLL dense represention/implementation @file */ - -#ifndef lua_hyperloglog_h_ -#define lua_hyperloglog_h_ - -#include -#include -#include "lua_sandbox_private.h" - -extern const char* lsb_hyperloglog; -extern const char* lsb_hyperloglog_table; - -#define HLL_P 14 /* The greater is P, the smaller the error. */ -#define HLL_REGISTERS (1< -#include -#include -#include -#include -#include -#include -#include -#include "lua_sandbox_private.h" -#include "lua_serialize.h" -#include "lua_serialize_protobuf.h" -#include "lua_circular_buffer.h" - -static const char* disable_base_functions[] = { "collectgarbage", "coroutine", - "dofile", "load", "loadfile", "loadstring", "module", "print", "require", NULL }; - -static jmp_buf g_jbuf; - - -lua_sandbox* lsb_create(void* parent, - const char* lua_file, - const char* require_path, - unsigned memory_limit, - unsigned instruction_limit, - unsigned output_limit) -{ - if (!lua_file) { - return NULL; - } - -#if _WIN32 - if (_putenv("TZ=UTC") != 0) { - return NULL; - } -#else - if (setenv("TZ", "UTC", 1) != 0) { - return NULL; - } -#endif - - lua_sandbox* lsb = malloc(sizeof(lua_sandbox)); - memset(lsb->usage, 0, sizeof(lsb->usage)); - if (!lsb) { - return NULL; - } -#ifdef LUA_JIT - lsb->lua = luaL_newstate(); -#else - lsb->lua = lua_newstate(memory_manager, lsb); -#endif - - if (!lsb->lua) { - free(lsb); - return NULL; - } - - lsb->parent = parent; - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = memory_limit; - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] = instruction_limit; - lsb->usage[LSB_UT_OUTPUT][LSB_US_LIMIT] = output_limit; - lsb->state = LSB_UNKNOWN; - lsb->error_message[0] = 0; - lsb->output.pos = 0; - lsb->output.maxsize = output_limit; - if (output_limit && output_limit < OUTPUT_SIZE) { - lsb->output.size = output_limit; - } else { - lsb->output.size = OUTPUT_SIZE; - } - lsb->output.data = malloc(lsb->output.size); - size_t len = strlen(lua_file); - lsb->lua_file = malloc(len + 1); - lsb->require_path = NULL; - if (require_path) { - len = strlen(require_path); - lsb->require_path = malloc(len + 1); - } - if (!lsb->output.data || !lsb->lua_file - || (require_path && !lsb->require_path)) { - free(lsb); - free(lsb->lua_file); - free(lsb->require_path); - lua_close(lsb->lua); - lsb->lua = NULL; - return NULL; - } - strcpy(lsb->lua_file, lua_file); - if (lsb->require_path) { - strcpy(lsb->require_path, require_path); - } - srand((unsigned int)time(NULL)); - return lsb; -} - - -int unprotected_panic(lua_State* lua) -{ - (void)lua; - longjmp(g_jbuf, 1); - return 0; -} - - -int lsb_init(lua_sandbox* lsb, const char* data_file) -{ - if (!lsb) { - return 0; - } -#ifndef LUA_JIT - unsigned mem_limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif - - load_library(lsb->lua, "", luaopen_base, disable_base_functions); - lua_pop(lsb->lua, 1); - - // Create a simple package cache - lua_createtable(lsb->lua, 0, 1); - lua_pushvalue(lsb->lua, -1); - lua_setglobal(lsb->lua, package_table); - // Add empty metatable to prevent serialization - lua_newtable(lsb->lua); - lua_setmetatable(lsb->lua, -2); - // add the loaded table - lua_newtable(lsb->lua); - lua_setfield(lsb->lua, -2, loaded_table); - lua_pop(lsb->lua, 1); // remove the package table - - lua_pushlightuserdata(lsb->lua, (void*)lsb); - lua_pushcclosure(lsb->lua, &require_library, 1); - lua_setglobal(lsb->lua, "require"); - - lua_pushlightuserdata(lsb->lua, (void*)lsb); - lua_pushcclosure(lsb->lua, &output, 1); - lua_setglobal(lsb->lua, "output"); - - lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - (int)lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); -#else - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = mem_limit; -#endif - lua_CFunction pf = lua_atpanic(lsb->lua, unprotected_panic); - int jump = setjmp(g_jbuf); - if (jump || luaL_dofile(lsb->lua, lsb->lua_file) != 0) { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "%s", - lua_tostring(lsb->lua, -1)); - if (len >= LSB_ERROR_SIZE || len < 0) { - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - } - sandbox_terminate(lsb); - return 2; - } else { - lua_gc(lsb->lua, LUA_GCCOLLECT, 0); - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = - (unsigned)instruction_usage(lsb); - if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] - > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; - } - lsb->state = LSB_RUNNING; - if (data_file != NULL && strlen(data_file) > 0) { - if (restore_global_data(lsb, data_file)) return 3; - } - } - lua_atpanic(lsb->lua, pf); - return 0; -} - - -char* lsb_destroy(lua_sandbox* lsb, const char* data_file) -{ - char* err = NULL; - if (!lsb) { - return err; - } - - if (lsb->lua && data_file && strlen(data_file) > 0) { - if (preserve_global_data(lsb, data_file) != 0) { - size_t len = strlen(lsb->error_message); - err = malloc(len + 1); - if (err != NULL) { - strcpy(err, lsb->error_message); - } - } - } - sandbox_terminate(lsb); - free(lsb->output.data); - free(lsb->lua_file); - free(lsb->require_path); - free(lsb); - return err; -} - - -unsigned lsb_usage(lua_sandbox* lsb, lsb_usage_type utype, - lsb_usage_stat ustat) -{ - if (!lsb || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { - return 0; - } -#ifdef LUA_JIT - if (lsb->lua && utype == LSB_UT_MEMORY) { - switch (ustat) { - case LSB_US_CURRENT: - return lua_gc(lsb->lua, LUA_GCCOUNT, 0) * 1024 - + lua_gc(lsb->lua, LUA_GCCOUNTB, 0); - case LSB_US_MAXIMUM: - return lua_gc(lsb->lua, LUA_GCMAXCOUNT, 0) * 1024 - + lua_gc(lsb->lua, LUA_GCMAXCOUNTB, 0); - default: - break; - } - } -#endif - return lsb->usage[utype][ustat]; -} - - -const char* lsb_get_error(lua_sandbox* lsb) -{ - if (lsb) { - return lsb->error_message; - } - return ""; -} - - -void lsb_set_error(lua_sandbox* lsb, const char* err) -{ - if (lsb) { - if (err) { - strncpy(lsb->error_message, err, LSB_ERROR_SIZE); - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - } else { - lsb->error_message[0] = 0; - } - } -} - - -lua_State* lsb_get_lua(lua_sandbox* lsb) -{ - return lsb->lua; -} - - -void* lsb_get_parent(lua_sandbox* lsb) -{ - return lsb->parent; -} - - -lsb_state lsb_get_state(lua_sandbox* lsb) -{ - if (lsb) { - return lsb->state; - } - return LSB_UNKNOWN; -} - - -void lsb_add_function(lua_sandbox* lsb, lua_CFunction func, - const char* func_name) -{ - lua_pushlightuserdata(lsb->lua, (void*)lsb); - lua_pushcclosure(lsb->lua, func, 1); - lua_setglobal(lsb->lua, func_name); -} - - -const char* lsb_get_output(lua_sandbox* lsb, size_t* len) -{ - if (len) { - *len = lsb->output.pos; - } - if (lsb->output.pos == 0) return ""; - - lsb->output.pos = 0; - return lsb->output.data; -} - - -void lsb_output(lua_sandbox* lsb, int start, int end, int append) -{ - if (!append) { - lsb->output.pos = 0; - } - - int result = 0; - void* ud = NULL; - for (int i = start; result == 0 && i <= end; ++i) { - switch (lua_type(lsb->lua, i)) { - case LUA_TNUMBER: - if (serialize_double(&lsb->output, lua_tonumber(lsb->lua, i))) { - result = 1; - } - break; - case LUA_TSTRING: - if (appendf(&lsb->output, "%s", lua_tostring(lsb->lua, i))) { - result = 1; - } - break; - case LUA_TNIL: - if (appends(&lsb->output, "nil")) { - result = 1; - } - break; - case LUA_TBOOLEAN: - if (appendf(&lsb->output, "%s", - lua_toboolean(lsb->lua, i) - ? "true" : "false")) { - result = 1; - } - break; - case LUA_TUSERDATA: - ud = userdata_type(lsb->lua, i, lsb_circular_buffer); - if (ud) { - if (output_circular_buffer(lsb->lua, (circular_buffer*)ud, - &lsb->output)) { - result = 1; - } - } else { - luaL_argerror(lsb->lua, i, "unknown userdata type"); - } - break; - default: - luaL_argerror(lsb->lua, i, "unsuported type"); - break; - } - } - update_output_stats(lsb); - if (result != 0) { - if (lsb->error_message[0] == 0) { - luaL_error(lsb->lua, "output_limit exceeded"); - } - luaL_error(lsb->lua, lsb->error_message); - } -} - - -int lsb_output_protobuf(lua_sandbox* lsb, int index, int append) -{ - if (!append) { - lsb->output.pos = 0; - } - - size_t last_pos = lsb->output.pos; - if (serialize_table_as_pb(lsb, index) != 0) { - lsb->output.pos = last_pos; - return 1; - } - - update_output_stats(lsb); - return 0; -} - - -int lsb_pcall_setup(lua_sandbox* lsb, const char* func_name) -{ - - lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); - lua_getglobal(lsb->lua, func_name); - if (!lua_isfunction(lsb->lua, -1)) { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "%s() function was not found", - func_name); - if (len >= LSB_ERROR_SIZE || len < 0) { - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - } - return 1; - } - return 0; -} - - -void lsb_pcall_teardown(lua_sandbox* lsb) -{ - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = - (unsigned)instruction_usage(lsb); - if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] - > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; - } -} - - -void lsb_terminate(lua_sandbox* lsb, const char* err) -{ - strncpy(lsb->error_message, err, LSB_ERROR_SIZE); - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - sandbox_terminate(lsb); -} diff --git a/src/lua_sandbox_private.c b/src/lua_sandbox_private.c deleted file mode 100644 index a15b0c5..0000000 --- a/src/lua_sandbox_private.c +++ /dev/null @@ -1,337 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua sandbox private implementation @file */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "lua_bloom_filter.h" -#include "lua_circular_buffer.h" -#include "lua_hyperloglog.h" -#include "lua_sandbox_private.h" -#include "lua_serialize.h" -#include "lua_serialize_protobuf.h" - -#ifdef _WIN32 -#define PATH_DELIMITER '\\' -#else -#define PATH_DELIMITER '/' -#endif - -#ifndef MAX_PATH -#define MAX_PATH 260 -#endif - -const char* disable_none[] = { NULL }; -const char* package_table = "package"; -const char* loaded_table = "loaded"; - - -void load_library(lua_State* lua, const char* table, lua_CFunction f, - const char** disable) -{ - lua_pushcfunction(lua, f); - lua_call(lua, 0, 1); - - if (strlen(table) == 0) { // Handle the special "" base table. - for (int i = 0; disable[i]; ++i) { - lua_pushnil(lua); - lua_setfield(lua, LUA_GLOBALSINDEX, disable[i]); - } - } else { - for (int i = 0; disable[i]; ++i) { - lua_pushnil(lua); - lua_setfield(lua, -2, disable[i]); - } - // Add an empty metatable to identify core libraries during - // preservation. - lua_newtable(lua); - lua_setmetatable(lua, -2); - } -} - -#ifndef LUA_JIT -void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize) -{ - lua_sandbox* lsb = (lua_sandbox*)ud; - - void* nptr = NULL; - if (nsize == 0) { - free(ptr); - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= (unsigned)osize; - } else { - unsigned new_state_memory = - (unsigned)(lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize); - if (0 == lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] - || new_state_memory - <= lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]) { - nptr = realloc(ptr, nsize); - if (nptr != NULL) { - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = - new_state_memory; - if (lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] - > lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; - } - } - } - } - return nptr; -} -#endif - -void instruction_manager(lua_State* lua, lua_Debug* ar) -{ - if (LUA_HOOKCOUNT == ar->event) { - luaL_error(lua, "instruction_limit exceeded"); - } -} - - -size_t instruction_usage(lua_sandbox* lsb) -{ - return lua_gethookcount(lsb->lua) - lua_gethookcountremaining(lsb->lua); -} - - -void sandbox_terminate(lua_sandbox* lsb) -{ - if (lsb->lua) { - lua_close(lsb->lua); - lsb->lua = NULL; - } - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = 0; - lsb->state = LSB_TERMINATED; -} - - -void update_output_stats(lua_sandbox* lsb) -{ - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = (unsigned)lsb->output.pos; - if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] - > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { - lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; - } -} - - -int appendf(output_data* output, const char* fmt, ...) -{ - va_list args; - int result = 0; - int remaining = 0; - char* ptr = NULL, *old_ptr = NULL; - do { - ptr = output->data + output->pos; - remaining = (int)(output->size - output->pos); - va_start(args, fmt); - int needed = vsnprintf(ptr, remaining, fmt, args); - va_end(args); - if (needed == -1) { - // Windows and Unix have different return values for this function - // -1 on Unix is a format error - // -1 on Windows means the buffer is too small and the required len - // is not returned - needed = remaining; - } - if (needed >= remaining) { - if (output->maxsize - && (output->size >= output->maxsize - || output->pos + needed >= output->maxsize)) { - return 1; - } - size_t newsize = output->size * 2; - while ((size_t)needed >= newsize - output->pos) { - newsize *= 2; - } - if (output->maxsize && newsize > output->maxsize) { - newsize = output->maxsize; - } - void* p = malloc(newsize); - if (p != NULL) { - memcpy(p, output->data, output->pos); - old_ptr = output->data; - output->data = p; - output->size = newsize; - } else { - return 1; // Out of memory condition. - } - } else { - output->pos += needed; - break; - } - } - while (1); - free(old_ptr); - return result; -} - - -int realloc_output(output_data* output, size_t needed) -{ - if (output->maxsize && needed + output->pos > output->maxsize) { - return 1; - } - size_t newsize = output->size * 2; - while (needed >= newsize - output->pos) { - newsize *= 2; - } - if (output->maxsize && newsize > output->maxsize) { - newsize = output->maxsize; - } - - void* ptr = realloc(output->data, newsize); - if (!ptr) return 1; - output->data = ptr; - output->size = newsize; - return 0; -} - - -int appends(output_data* output, const char* str) -{ - size_t needed = strlen(str) + 1; - if (output->size - output->pos < needed) { - if (realloc_output(output, needed)) return 1; - } - memcpy(output->data + output->pos, str, needed); - output->pos += needed - 1; - return 0; -} - - -int appendc(output_data* output, char ch) -{ - size_t needed = 2; - if (output->size - output->pos < needed) { - if (realloc_output(output, needed)) return 1; - } - output->data[output->pos++] = ch; - output->data[output->pos] = 0; - return 0; -} - - -int output(lua_State* lua) -{ - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "output() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - int n = lua_gettop(lua); - if (n == 0) { - return luaL_argerror(lsb->lua, 0, "must have at least one argument"); - } - lsb_output(lsb, 1, n, 1); - return 0; -} - -LUALIB_API int luaopen_cjson(lua_State* L); -LUALIB_API int luaopen_lpeg(lua_State* L); -LUALIB_API int luaopen_struct(lua_State* L); -int set_encode_max_buffer(lua_State* L, int index, unsigned maxsize); - -int require_library(lua_State* lua) -{ - const char* name = luaL_checkstring(lua, 1); - lua_getglobal(lua, package_table); - if (!lua_istable(lua, -1)) { - return luaL_error(lua, "%s table is missing", package_table); - } - lua_getfield(lua, -1, loaded_table); - if (!lua_istable(lua, -1)) { - return luaL_error(lua, "%s.%s table is missing", package_table, loaded_table); - } - lua_getfield(lua, -1, name); - if (!lua_isnil(lua, -1)) { - return 1; // returned the cache copy - } - lua_pop(lua, 1); // remove the nil - int pos = lua_gettop(lua); - lua_pushboolean(lua, 1); - lua_setfield(lua, pos, name); // mark it as loaded to prevent a dependency loop - - if (strcmp(name, LUA_STRLIBNAME) == 0) { - load_library(lua, name, luaopen_string, disable_none); - } else if (strcmp(name, LUA_MATHLIBNAME) == 0) { - load_library(lua, name, luaopen_math, disable_none); - } else if (strcmp(name, LUA_TABLIBNAME) == 0) { - load_library(lua, name, luaopen_table, disable_none); - } else if (strcmp(name, LUA_OSLIBNAME) == 0) { - const char* disable[] = { "execute", "exit", "remove", "rename", - "setlocale", "tmpname", NULL }; - load_library(lua, name, luaopen_os, disable); - } else if (strcmp(name, lsb_circular_buffer_table) == 0) { - load_library(lua, name, luaopen_circular_buffer, disable_none); - } else if (strcmp(name, lsb_bloom_filter_table) == 0) { - load_library(lua, name, luaopen_bloom_filter, disable_none); - } else if (strcmp(name, lsb_hyperloglog_table) == 0) { - load_library(lua, name, luaopen_hyperloglog, disable_none); - } else if (strcmp(name, "lpeg") == 0) { - load_library(lua, name, luaopen_lpeg, disable_none); - } else if (strcmp(name, "cjson") == 0) { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "require_library() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - const char* disable[] = { "new", "encode_keep_buffer", NULL }; - load_library(lua, name, luaopen_cjson, disable); - if (set_encode_max_buffer(lua, -1, (unsigned)lsb->output.maxsize)) { - return luaL_error(lua, "cjson encode buffer could not be configured"); - } - lua_pushvalue(lua, -1); - lua_setglobal(lua, name); - } else if (strcmp(name, "struct") == 0) { - load_library(lua, name, luaopen_struct, disable_none); - } else { - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - return luaL_error(lua, "require_library() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - if (!lsb->require_path) { - return luaL_error(lua, "require_library() external modules are disabled"); - } - - int i = 0; - while (name[i]) { - if (!isalnum(name[i]) && name[i] != '_') { - return luaL_error(lua, "invalid module name '%s'", name); - } - ++i; - } - char fn[MAX_PATH]; - i = snprintf(fn, MAX_PATH, "%s%c%s.lua", lsb->require_path, PATH_DELIMITER, - name); - if (i < 0 || i >= MAX_PATH) { - return luaL_error(lua, "require_path exceeded %d", MAX_PATH); - } - - if (luaL_dofile(lua, fn) != 0) { - return luaL_error(lua, "%s", lua_tostring(lua, -1)); - } - // Add an empty metatable to identify the library during preservation. - lua_newtable(lua); - lua_setmetatable(lua, -2); - } - lua_pushvalue(lua, -1); - lua_setfield(lua, pos, name); - return 1; -} diff --git a/src/lua_sandbox_private.h b/src/lua_sandbox_private.h deleted file mode 100644 index 2eaaf9a..0000000 --- a/src/lua_sandbox_private.h +++ /dev/null @@ -1,167 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// Lua sandbox private functions @file -#ifndef lua_sandbox_private_h_ -#define lua_sandbox_private_h_ - -#include -#include -#include "lua_sandbox.h" - -#define OUTPUT_SIZE 1024 - -#ifdef _WIN32 -#define snprintf _snprintf -#endif - -typedef struct -{ - size_t maxsize; - size_t size; - size_t pos; - char* data; -} output_data; - -struct lua_sandbox { - lua_State* lua; - void* parent; - lsb_state state; - output_data output; - char* lua_file; - char* require_path; - unsigned usage[LSB_UT_MAX][LSB_US_MAX]; - char error_message[LSB_ERROR_SIZE]; -}; - -extern const char* disable_none[]; -extern const char* package_table; -extern const char* loaded_table; - -//////////////////////////////////////////////////////////////////////////////// -/// Sandbox management functions. -//////////////////////////////////////////////////////////////////////////////// -/** - * Performs the library load and secures the sandbox environment for use. - * - * @param lua Pointer to the Lua state. - * @param table Name of the table being loaded. - * @param f Pointer to the table load function. - * @param disable Array of function names to disable in the loaded table. - */ -void load_library(lua_State* lua, const char* table, lua_CFunction f, - const char** disable); -#ifndef LUA_JIT -/** -* Implementation of the memory allocator for the Lua state. -* -* See: http://www.lua.org/manual/5.1/manual.html#lua_Alloc -* -* @param ud Pointer to the lua_sandbox -* @param ptr Pointer to the memory block being allocated/reallocated/freed. -* @param osize The original size of the memory block. -* @param nsize The new size of the memory block. -* -* @return void* A pointer to the memory block. -*/ -void* memory_manager(void* ud, void* ptr, size_t osize, size_t nsize); -#endif - - -/** - * Lua hook to monitor the instruction usage of the sandbox. - * - * @param lua Pointer to the Lua state. - * @param ar Pointer to the Lua debug interface. - */ -void instruction_manager(lua_State* lua, lua_Debug* ar); - -/** - * Extracts the current instruction usage count from the Lua state. - * - * @param lsb Pointer to the sandbox. - * - * @return size_t The number of instructions used in the last function call - */ -size_t instruction_usage(lua_sandbox* lsb); - -/** - * Tears down the sandbox on error. - * - * @param lsb Pointer to the sandbox. - */ -void sandbox_terminate(lua_sandbox* lsb); - -/** - * Helper function to update the output statistics. - * - * @param lsb Pointer to the sandbox. - */ -void update_output_stats(lua_sandbox* lsb); - -/** - * Append formatted string to the output stream. - * - * @param output Pointer the output collector. - * @param fmt Printf format specifier. - * - * @return int Zero on success, non-zero if out of memory. - */ -int appendf(output_data* output, const char* fmt, ...); - -/** - * Resize the output buffer when more space is needed. - * - * @param output Output buffer to resize. - * @param needed Number of additional bytes needed. - * - * @return int Zero on success, non-zero on failure. - */ -int realloc_output(output_data* output, size_t needed); - -/** - * Append a fixed string to the output stream. - * - * @param output Pointer the output collector. - * @param str String to append to the output. - * - * @return int Zero on success, non-zero if out of memory. - */ -int appends(output_data* output, const char* str); - -/** - * Append a character to the output stream. - * - * @param output Pointer the output collector. - * @param ch Character to append to the output. - * - * @return int Zero on success, non-zero if out of memory. - */ -int appendc(output_data* output, char ch); - -//////////////////////////////////////////////////////////////////////////////// -/// Lua to C function interface -//////////////////////////////////////////////////////////////////////////////// -/** - * Collect sandbox output into an in memory buffer. - * - * @param lua Pointer to the Lua state. - * - * @return int Returns zero values on the stack. - */ -int output(lua_State* lua); - -/** - * Overridden 'require' used to load optional sandbox libraries in global space. - * - * @param lua Pointer to the Lua state. - * - * @return int Returns 1 value on the stack (for the standard modules a table - * for the LPEG grammars, userdata). - */ -int require_library(lua_State* lua); - -#endif diff --git a/src/lua_serialize.c b/src/lua_serialize.c deleted file mode 100644 index ac779ec..0000000 --- a/src/lua_serialize.c +++ /dev/null @@ -1,577 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Sandbox serialization implementation @file */ - -#include -#include -#include -#include -#include -#include "lua_serialize.h" -#include "lua_circular_buffer.h" -#include "lua_bloom_filter.h" -#include "lua_hyperloglog.h" - -const char* not_a_number = "nan"; -static const char* preservation_version = "_PRESERVATION_VERSION"; - -static int get_preservation_version(lua_State* lua) -{ - int ver = 0; - lua_getglobal(lua, preservation_version); - int t = lua_type(lua, -1); - if (t == LUA_TNUMBER) { - ver = (int)lua_tointeger(lua, -1); - } - lua_pop(lua, 1); // remove the version from the stack - - if (t != LUA_TNIL) { // remove the version from the data preservation - lua_pushnil(lua); - lua_setglobal(lua, preservation_version); - } - return ver; -} - -int preserve_global_data(lua_sandbox* lsb, const char* data_file) -{ - static const char* G = "_G"; - - // make sure the string library is loaded before we start - lua_getglobal(lsb->lua, LUA_STRLIBNAME); - if (!lua_istable(lsb->lua, -1)) { - load_library(lsb->lua, LUA_STRLIBNAME, luaopen_string, disable_none); - } - lua_pop(lsb->lua, 1); // Remove string table. - - lua_getglobal(lsb->lua, G); - if (!lua_istable(lsb->lua, -1)) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve_global_data cannot access the global table"); - return 1; - } - - FILE* fh = fopen(data_file, "wb"); - if (fh == NULL) { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve_global_data could not open: %s", data_file); - if (len >= LSB_ERROR_SIZE || len < 0) { - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - } - return 1; - } - - int result = 0; - serialization_data data; - data.fh = fh; - data.keys.maxsize = 0; - -// Clear the sandbox limits during preservation. -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); -#else -// unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif - lua_sethook(lsb->lua, instruction_manager, 0, 0); -// size_t cur_output_size = lsb->output.size; -// size_t max_output_size = lsb->output.maxsize; - lsb->output.maxsize = 0; -// end clear - - data.keys.size = OUTPUT_SIZE; - data.keys.pos = 0; - data.keys.data = malloc(data.keys.size); - data.tables.size = 64; - data.tables.pos = 0; - data.tables.array = malloc(data.tables.size * sizeof(table_ref)); - if (data.tables.array == NULL || data.keys.data == NULL) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve_global_data out of memory"); - result = 1; - } else { - fprintf(data.fh, "if %s and %s ~= %d then return end\n", - preservation_version, - preservation_version, - get_preservation_version(lsb->lua)); - appendf(&data.keys, "%s", G); - data.keys.pos += 1; - data.globals = lua_topointer(lsb->lua, -1); - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - result = serialize_kvp(lsb, &data, 0); - lua_pop(lsb->lua, 1); - } - lua_pop(lsb->lua, lua_gettop(lsb->lua)); - // Wipe the entire Lua stack. Since incremental cleanup on failure - // was added the stack should only contain table _G. - } - free(data.tables.array); - free(data.keys.data); - fclose(fh); - if (result != 0) { - remove(data_file); - } - -// Uncomment if we start preserving state when not destroying the sandbox -/* -// Restore the sandbox limits after preservation -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); -#else - lua_gc(lsb->lua, LUA_GCCOLLECT, 0); - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; - lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; -#endif - lsb->output.maxsize = max_output_size; - lsb->output.pos = 0; - if (lsb->output.size > cur_output_size) { - void* ptr = realloc(lsb->output.data, cur_output_size); - if (!ptr) return 1; - lsb->output.data = ptr; - lsb->output.size = cur_output_size; - } -// end restore -*/ - - return result; -} - - -int serialize_double(output_data* output, double d) -{ - if (isnan(d)) { - return appends(output, not_a_number); - } - if (d > INT_MAX) { - return appendf(output, "%0.17g", d); - } - - const int precision = 8; - const unsigned magnitude = 100000000; - char buffer[20]; - char* p = buffer; - int negative = 0; - - if (d < 0) { - negative = 1; - d = -d; - } - - int number = (int)d; - double tmp = (d - number) * magnitude; - unsigned fraction = (unsigned)tmp; - double diff = tmp - fraction; - - if (diff > 0.5) { - ++fraction; - if (fraction >= magnitude) { - fraction = 0; - ++number; - } - } else if (diff == 0.5 && ((fraction == 0) || (fraction & 1))) { - // bankers rounding - ++fraction; - } - - // output decimal fraction - if (fraction != 0) { - int nodigits = 1; - char c = 0; - for (int x = 0; x < precision; ++x) { - c = fraction % 10; - if (!(c == 0 && nodigits)) { - *p++ = c + '0'; - nodigits = 0; - } - fraction /= 10; - } - *p++ = '.'; - } - - // output number - do { - *p++ = (number % 10) + '0'; - number /= 10; - } - while (number > 0); - - size_t remaining = output->size - output->pos; - size_t needed = (p - buffer) + negative; - if (needed >= remaining) { - if (realloc_output(output, needed)) return 1; - } - - if (negative) { - output->data[output->pos++] = '-'; - } - do { - --p; - output->data[output->pos++] = *p; - } - while (p != buffer); - output->data[output->pos] = 0; - - return 0; -} - - -int serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent) -{ - int result = 0; - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - result = serialize_kvp(lsb, data, parent); - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. - } - return result; -} - - -int serialize_data(lua_sandbox* lsb, int index, output_data* output) -{ - output->pos = 0; - switch (lua_type(lsb->lua, index)) { - case LUA_TNUMBER: - if (serialize_double(output, lua_tonumber(lsb->lua, index))) { - return 1; - } - break; - case LUA_TSTRING: - // The stack is cleaned up on failure by preserve_global_data - // but for clarity it is incrementally cleaned up anyway. - lua_checkstack(lsb->lua, 4); - lua_getglobal(lsb->lua, LUA_STRLIBNAME); - lua_getfield(lsb->lua, -1, "format"); - if (!lua_isfunction(lsb->lua, -1)) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_data cannot access the string format function"); - lua_pop(lsb->lua, 2); // Remove the bogus format function and - // string table. - return 1; - } - lua_pushstring(lsb->lua, "%q"); - lua_pushvalue(lsb->lua, index - 3); - if (lua_pcall(lsb->lua, 2, 1, 0) == 0) { - if (appendf(output, "%s", lua_tostring(lsb->lua, -1))) { - lua_pop(lsb->lua, 1); // Remove the string table. - return 1; - } - } else { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_data '%s'", - lua_tostring(lsb->lua, -1)); - if (len >= LSB_ERROR_SIZE || len < 0) { - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - } - lua_pop(lsb->lua, 2); // Remove the error message and the string - // table. - return 1; - } - lua_pop(lsb->lua, 2); // Remove the pcall result and the string table. - break; - case LUA_TBOOLEAN: - if (appendf(output, "%s", - lua_toboolean(lsb->lua, index) - ? "true" : "false")) { - return 1; - } - break; - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_data cannot preserve type '%s'", - lua_typename(lsb->lua, lua_type(lsb->lua, index))); - return 1; - } - return 0; -} - - -void* userdata_type(lua_State* lua, int index, const char* tname) -{ - void* ud = lua_touserdata(lua, index); - if (ud != NULL) { - if (lua_getmetatable(lua, index)) { - lua_getfield(lua, LUA_REGISTRYINDEX, tname); - if (!lua_rawequal(lua, -1, -2)) { - ud = NULL; - } - } - } - lua_pop(lua, 2); // metatable and field - return ud; -} - - -int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent) -{ - int kindex = -2, vindex = -1, result = 0; - - if (ignore_value_type(lsb, data, vindex)) { - return 0; - } - - if (serialize_data(lsb, kindex, &lsb->output)) { - return 1; - } - - size_t pos = data->keys.pos; - if (appendf(&data->keys, "%s[%s]", data->keys.data + parent, - lsb->output.data)) { - return 1; - } - - if (lua_type(lsb->lua, vindex) == LUA_TTABLE) { - const void* ptr = lua_topointer(lsb->lua, vindex); - table_ref* seen = find_table_ref(&data->tables, ptr); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ptr, pos); - if (seen != NULL) { - data->keys.pos += 1; - fprintf(data->fh, "%s = {}\n", data->keys.data + pos); - result = serialize_table(lsb, data, pos); - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "preserve table out of memory"); - return 1; - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + seen->name_pos); - } - } else if (lua_type(lsb->lua, vindex) == LUA_TUSERDATA) { - void* ud = NULL; - if ((ud = userdata_type(lsb->lua, vindex, lsb_circular_buffer))) { - table_ref* seen = find_table_ref(&data->tables, ud); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ud, pos); - if (seen != NULL) { - data->keys.pos += 1; - result = serialize_circular_buffer(lsb->lua, - data->keys.data + pos, - (circular_buffer*)ud, - &lsb->output); - if (result == 0) { - fprintf(data->fh, "%s", lsb->output.data); - } - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_circular_buffer out of memory"); - return 1; - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + - seen->name_pos); - } - } else if ((ud = userdata_type(lsb->lua, vindex, lsb_bloom_filter))) { - table_ref* seen = find_table_ref(&data->tables, ud); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ud, pos); - if (seen != NULL) { - data->keys.pos += 1; - result = serialize_bloom_filter(data->keys.data + pos, - (bloom_filter*)ud, - &lsb->output); - if (result == 0) { - size_t n = fwrite(lsb->output.data, 1, lsb->output.pos, data->fh); - if (n != lsb->output.pos) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_bloom_filter failed"); - return 1; - } - } - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_bloom_filter out of memory"); - return 1; - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + - seen->name_pos); - } - } else if ((ud = userdata_type(lsb->lua, vindex, lsb_hyperloglog))) { - table_ref* seen = find_table_ref(&data->tables, ud); - if (seen == NULL) { - seen = add_table_ref(&data->tables, ud, pos); - if (seen != NULL) { - data->keys.pos += 1; - result = serialize_hyperloglog(data->keys.data + pos, - (hyperloglog*)ud, - &lsb->output); - if (result == 0) { - size_t n = fwrite(lsb->output.data, 1, lsb->output.pos, data->fh); - if (n != lsb->output.pos) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_hyperloglog failed"); - return 1; - } - } - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "serialize_hyperloglog out of memory"); - return 1; - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - fprintf(data->fh, "%s\n", data->keys.data + - seen->name_pos); - } - } - } else { - fprintf(data->fh, "%s = ", data->keys.data + pos); - data->keys.pos = pos; - result = serialize_data(lsb, vindex, &lsb->output); - if (result == 0) { - fprintf(data->fh, "%s\n", lsb->output.data); - } - } - return result; -} - - -int ignore_key(lua_sandbox* lsb, int index) -{ - if (lua_type(lsb->lua, index) == LUA_TSTRING) { - const char* key = lua_tostring(lsb->lua, index); - if (key[0] == '_') { - return 1; - } - } - return 0; -} - - -table_ref* find_table_ref(table_ref_array* tra, const void* ptr) -{ - for (size_t i = 0; i < tra->pos; ++i) { - if (ptr == tra->array[i].ptr) { - return &tra->array[i]; - } - } - return NULL; -} - - -table_ref* add_table_ref(table_ref_array* tra, const void* ptr, size_t name_pos) -{ - if (tra->pos == tra->size) { - size_t newsize = tra->size * 2; - void* p = realloc(tra->array, newsize * sizeof(table_ref)); - if (p != NULL) { - tra->array = p; - tra->size = newsize; - } else { - return NULL; - } - } - tra->array[tra->pos].ptr = ptr; - tra->array[tra->pos].name_pos = name_pos; - return &tra->array[tra->pos++]; -} - - -int ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index) -{ - switch (lua_type(lsb->lua, index)) { - case LUA_TTABLE: - if (lua_getmetatable(lsb->lua, index) != 0) { - lua_pop(lsb->lua, 1); // Remove the metatable. - return 1; - } - if (lua_topointer(lsb->lua, index) == data->globals) { - return 1; - } - break; - case LUA_TUSERDATA: - if (!userdata_type(lsb->lua, index, lsb_circular_buffer) && - !userdata_type(lsb->lua, index, lsb_bloom_filter) && - !userdata_type(lsb->lua, index, lsb_hyperloglog)) { - return 1; - } - break; - case LUA_TNONE: - case LUA_TFUNCTION: - case LUA_TTHREAD: - case LUA_TLIGHTUSERDATA: - case LUA_TNIL: - return 1; - } - return 0; -} - - -int restore_global_data(lua_sandbox* lsb, const char* data_file) -{ - // Clear the sandbox limits during restoration. -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, 0); -#else - unsigned limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; -#endif - lua_sethook(lsb->lua, instruction_manager, 0, 0); - - int err = 0; - if ((err = luaL_dofile(lsb->lua, data_file)) != 0) { - if (LUA_ERRFILE != err) { - int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, - "restore_global_data %s", - lua_tostring(lsb->lua, -1)); - if (len >= LSB_ERROR_SIZE || len < 0) { - lsb->error_message[LSB_ERROR_SIZE - 1] = 0; - } - sandbox_terminate(lsb); - return 1; - } - } -#ifdef LUA_JIT - lua_gc(lsb->lua, LUA_GCSETMEMLIMIT, - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]); // reinstate limit -#else - lua_gc(lsb->lua, LUA_GCCOLLECT, 0); - lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; - lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = - lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; -#endif - return 0; -} - - -int serialize_binary(const void* src, size_t len, output_data* output) -{ - const char* uc = (const char*)src; - for (unsigned i = 0; i < len; ++i) { - switch (uc[i]) { - case '\n': - if (appends(output, "\\n")) return 1; - break; - case '\r': - if (appends(output, "\\r")) return 1; - break; - case '"': - if (appends(output, "\\\"")) return 1; - break; - case '\\': - if (appends(output, "\\\\")) return 1; - break; - default: - if (appendc(output, uc[i])) return 1; - break; - } - } - return 0; -} diff --git a/src/lua_serialize.h b/src/lua_serialize.h deleted file mode 100644 index ea2f5da..0000000 --- a/src/lua_serialize.h +++ /dev/null @@ -1,168 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** Lua sandbox serialization @file */ - -#ifndef lua_serialize_h_ -#define lua_serialize_h_ - -#include -#include "lua_sandbox_private.h" - -extern const char* not_a_number; - -typedef struct -{ - const void* ptr; - size_t name_pos; -} table_ref; - -typedef struct -{ - size_t size; - size_t pos; - table_ref* array; -} table_ref_array; - -typedef struct -{ - FILE* fh; - output_data keys; - table_ref_array tables; - const void* globals; -} serialization_data; - -/** - * Serialize all user global data to disk. - * - * @param lsb Pointer to the sandbox. - * @param data_file Filename where the data will be written (create/overwrite) - * - * @return int Zero on success, non-zero on failure. - */ -int preserve_global_data(lua_sandbox* lsb, const char* data_file); - -/** - * More efficient serialization of a double to a string - * - * @param output Pointer the output collector. - * @param d Double value to convert to a string. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_double(output_data* output, double d); - -/** - * Serializes a Lua table structure. - * - * @param lsb Pointer to the sandbox. - * @param data Pointer to the serialization state data. - * @param parent Index pointing to the parent's name in the table array. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_table(lua_sandbox* lsb, serialization_data* data, size_t parent); - -/** - * Serializes a Lua data value. - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index where the data resides. - * @param output Pointer the output collector. - * - * @return int - */ -int serialize_data(lua_sandbox* lsb, int index, output_data* output); - -/** - * Determines the name of the userdata type - * - * @param lua Lua State - * @param index Index on the stack where the userdata pointer resides - * @param tname userdata table name - * - * @return const char* NULL if not found - */ -void *userdata_type(lua_State* lua, int index, const char *tname); - -/** - * Serializes a table key value pair. - * - * @param lsb Pointer to the sandbox. - * @param data Pointer to the serialization state data. - * @param parent Index pointing to the parent's name in the table array. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_kvp(lua_sandbox* lsb, serialization_data* data, size_t parent); - -/** - * Checks to see a string key starts with a '_' in which case it will be - * ignored. - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index where the key resides. - * - * @return int True if the key should not be serialized. - */ -int ignore_key(lua_sandbox* lsb, int index); - - -/** - * Looks for a table to see if it has already been processed. - * - * @param tra Pointer to the table references. - * @param ptr Pointer value of the table. - * - * @return table_ref* NULL if not found. - */ -table_ref* find_table_ref(table_ref_array* tra, const void* ptr); - -/** - * Adds a table to the processed array. - * - * @param tra Pointer to the table references. - * @param ptr Pointer value of the table. - * @param name_pos Index pointing to name in the table array. - * - * @return table_ref* Pointer to the table reference or NULL if out of memory. - */ -table_ref* add_table_ref(table_ref_array* tra, const void* ptr, - size_t name_pos); - -/** - * Helper function to determine what data should not be serialized. - * - * @param lsb Pointer to the sandbox. - * @param data Pointer to the serialization state data. - * @param index Lua stack index where the data resides. - * - * @return int - */ -int ignore_value_type(lua_sandbox* lsb, serialization_data* data, int index); - -/** - * Restores previously serialized data from disk. - * - * @param lsb Pointer to the sandbox. - * @param data_file Filename from where the data will be read. - * - * @return int Zero on success, non-zero on failure. - */ -int restore_global_data(lua_sandbox* lsb, const char* data_file); - -/** - * Serializes a binary data to a Lua string. - * - * @param src Pointer to the binary data. - * @param len Length in bytes of the data to output. - * @param output Pointer the output collector. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_binary(const void *src, size_t len, output_data* output); - -#endif diff --git a/src/lua_serialize_protobuf.c b/src/lua_serialize_protobuf.c deleted file mode 100644 index de78aa8..0000000 --- a/src/lua_serialize_protobuf.c +++ /dev/null @@ -1,360 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// @brief Lua sandbox Heka protobuf serialization @file - -#include "lua_serialize_protobuf.h" - -#include -#include -#include - - -int serialize_table_as_pb(lua_sandbox* lsb, int index) -{ - output_data* d = &lsb->output; - d->pos = 0; - size_t needed = 18; - if (needed > d->size) { - if (realloc_output(d, needed)) return 1; - } - - // create a type 4 uuid - d->data[d->pos++] = 2 | (1 << 3); - d->data[d->pos++] = 16; - for (int x = 0; x < 16; ++x) { - d->data[d->pos++] = rand() % 255; - } - d->data[8] = (d->data[8] & 0x0F) | 0x40; - d->data[10] = (d->data[10] & 0x0F) | 0xA0; - - // use existing or create a timestamp - lua_getfield(lsb->lua, index, "Timestamp"); - long long ts; - if (lua_isnumber(lsb->lua, -1)) { - ts = (long long)lua_tonumber(lsb->lua, -1); - } else { - ts = (long long)(time(NULL) * 1e9); - } - lua_pop(lsb->lua, 1); - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, ts)) return 1; - - if (encode_string(lsb, d, 3, "Type", index)) return 1; - if (encode_string(lsb, d, 4, "Logger", index)) return 1; - if (encode_int(lsb, d, 5, "Severity", index)) return 1; - if (encode_string(lsb, d, 6, "Payload", index)) return 1; - if (encode_string(lsb, d, 7, "EnvVersion", index)) return 1; - if (encode_int(lsb, d, 8, "Pid", index)) return 1; - if (encode_string(lsb, d, 9, "Hostname", index)) return 1; - if (encode_fields(lsb, d, 10, "Fields", index)) return 1; - // if we go above 15 pb_write_tag will need to start varint encoding - needed = 1; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - d->data[d->pos] = 0; // NULL terminate incase someone tries to treat this - // as a string - - return 0; -} - - -int pb_write_varint(output_data* d, long long i) -{ - size_t needed = 10; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - - if (i == 0) { - d->data[d->pos++] = 0; - return 0; - } - - while (i) { - d->data[d->pos++] = (i & 0x7F) | 0x80; - i >>= 7; - } - d->data[d->pos - 1] &= 0x7F; // end the varint - return 0; -} - - -int pb_write_double(output_data* d, double i) -{ - size_t needed = sizeof(double); - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - - // todo add big endian support if necessary - memcpy(&d->data[d->pos], &i, needed); - d->pos += needed; - return 0; -} - - -int pb_write_bool(output_data* d, int i) -{ - size_t needed = 1; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - - if (i) { - d->data[d->pos++] = 1; - } else { - d->data[d->pos++] = 0; - } - return 0; -} - - -int pb_write_tag(output_data* d, char id, char wire_type) -{ - size_t needed = 1; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - - d->data[d->pos++] = wire_type | (id << 3); - return 0; -} - - -int pb_write_string(output_data* d, char id, const char* s, size_t len) -{ - - if (pb_write_tag(d, id, 2)) { - return 1; - } - if (pb_write_varint(d, len)) { - return 1; - } - - size_t needed = len; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - memcpy(&d->data[d->pos], s, len); - d->pos += len; - return 0; -} - - -int encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index) -{ - int result = 0; - lua_getfield(lsb->lua, index, name); - if (lua_isstring(lsb->lua, -1)) { - size_t len; - const char* s = lua_tolstring(lsb->lua, -1, &len); - result = pb_write_string(d, id, s, len); - } - lua_pop(lsb->lua, 1); - return result; -} - - -int encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index) -{ - int result = 0; - lua_getfield(lsb->lua, index, name); - if (lua_isnumber(lsb->lua, -1)) { - long long i = (long long)lua_tonumber(lsb->lua, -1); - if (!(result = pb_write_tag(d, id, 0))) { - result = pb_write_varint(d, i); - } - } - lua_pop(lsb->lua, 1); - return result; -} - - -int encode_field_array(lua_sandbox* lsb, output_data* d, int t, - const char* representation) -{ - int result = 0, first = (int)lua_objlen(lsb->lua, -1); - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - if (lua_type(lsb->lua, -1) != t) { - snprintf(lsb->error_message, LSB_ERROR_SIZE, "array has mixed types"); - return 1; - } - result = encode_field_value(lsb, d, first, representation); - first = 0; - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. - } - return result; -} - - -int encode_field_object(lua_sandbox* lsb, output_data* d) -{ - int result = 0; - const char* representation = NULL; - lua_getfield(lsb->lua, -1, "representation"); - if (lua_isstring(lsb->lua, -1)) { - representation = lua_tostring(lsb->lua, -1); - } - lua_getfield(lsb->lua, -2, "value"); - result = encode_field_value(lsb, d, 1, representation); - lua_pop(lsb->lua, 2); // remove representation and value - return result; -} - - -int encode_field_value(lua_sandbox* lsb, output_data* d, int first, - const char* representation) -{ - int result = 1; - size_t len; - const char* s; - - int t = lua_type(lsb->lua, -1); - switch (t) { - case LUA_TSTRING: - if (first && representation) { // this uglyness keeps the protobuf - // fields in order without additional - // lookups - if (pb_write_string(d, 3, representation, strlen(representation))) { - return 1; - } - } - s = lua_tolstring(lsb->lua, -1, &len); - result = pb_write_string(d, 4, s, len); - break; - case LUA_TNUMBER: - if (first) { - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, 3)) return 1; - if (representation) { - if (pb_write_string(d, 3, representation, - strlen(representation))) { - return 1; - } - } - if (1 == first) { - if (pb_write_tag(d, 7, 1)) return 1; - } else { - if (pb_write_tag(d, 7, 2)) return 1; - if (pb_write_varint(d, first * sizeof(double))) return 1; - } - } - result = pb_write_double(d, lua_tonumber(lsb->lua, -1)); - break; - case LUA_TBOOLEAN: - if (first) { - if (pb_write_tag(d, 2, 0)) return 1; - if (pb_write_varint(d, 4)) return 1; - if (representation) { - if (pb_write_string(d, 3, representation, - strlen(representation))) { - return 1; - } - } - if (1 == first) { - if (pb_write_tag(d, 8, 0)) return 1; - } else { - if (pb_write_tag(d, 8, 2)) return 1; - if (pb_write_varint(d, first)) return 1; - } - } - result = pb_write_bool(d, lua_toboolean(lsb->lua, -1)); - break; - case LUA_TTABLE: - { - lua_rawgeti(lsb->lua, -1, 1); - int t = lua_type(lsb->lua, -1); - lua_pop(lsb->lua, 1); // remove the array test value - switch (t) { - case LUA_TNIL: - result = encode_field_object(lsb, d); - break; - case LUA_TNUMBER: - case LUA_TSTRING: - case LUA_TBOOLEAN: - result = encode_field_array(lsb, d, t, representation); - break; - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "unsupported array type: %s", lua_typename(lsb->lua, t)); - result = 1; - break; - } - } - break; - default: - snprintf(lsb->error_message, LSB_ERROR_SIZE, "unsupported type: %s", - lua_typename(lsb->lua, t)); - result = 1; - } - return result; -} - - -int update_field_length(output_data* d, size_t len_pos) -{ - size_t len = d->pos - len_pos - 1; - if (len < 128) { - d->data[len_pos] = (char)len; - return 0; - } - size_t l = len, cnt = 0; - while (l) { - l >>= 7; - ++cnt; // compute the number of bytes needed for the varint length - } - size_t needed = cnt - 1; - if (needed > d->size - d->pos) { - if (realloc_output(d, needed)) return 1; - } - size_t end_pos = d->pos + needed; - memmove(&d->data[len_pos + cnt], &d->data[len_pos + 1], len); - d->pos = len_pos; - if (pb_write_varint(d, len)) return 1; - d->pos = end_pos; - return 0; -} - - -int encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index) -{ - int result = 0; - lua_getfield(lsb->lua, index, name); - if (!lua_istable(lsb->lua, -1)) { - return result; - } - - size_t len_pos, len; - lua_checkstack(lsb->lua, 2); - lua_pushnil(lsb->lua); - while (result == 0 && lua_next(lsb->lua, -2) != 0) { - if (pb_write_tag(d, id, 2)) return 1; - len_pos = d->pos; - if (pb_write_varint(d, 0)) return 1; // length tbd later - if (lua_isstring(lsb->lua, -2)) { - const char* s = lua_tolstring(lsb->lua, -2, &len); - if (pb_write_string(d, 1, s, len)) return 1; - } else { - snprintf(lsb->error_message, LSB_ERROR_SIZE, - "field name must be a string"); - return 1; - } - if (encode_field_value(lsb, d, 1, NULL)) return 1; - if (update_field_length(d, len_pos)) return 1; - lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for - // the next interation. - } - lua_pop(lsb->lua, 1); // remove the fields table - return result; -} diff --git a/src/lua_serialize_protobuf.h b/src/lua_serialize_protobuf.h deleted file mode 100644 index ffc6c2d..0000000 --- a/src/lua_serialize_protobuf.h +++ /dev/null @@ -1,183 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/// Lua sandbox Heka protobuf serialization implementation @file -#ifndef lua_serialize_protobuf_h_ -#define lua_serialize_protobuf_h_ - -#include "lua_sandbox_private.h" - -/** - * Serialize a specific Lua table structure as Protobuf - * - * @param lsb Pointer to the sandbox. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero on failure. - */ -int serialize_table_as_pb(lua_sandbox* lsb, int index); - -/** - * Writes a varint encoded number to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Number to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_varint(output_data* d, long long i); - -/** - * Writes a double to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Double to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_double(output_data* d, double i); - -/** - * Writes a bool to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param i Number to be encoded. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_bool(output_data* d, int i); - -/** - * Writes a field tag (tag id/wire type) to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param wire_type Field wire type. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_tag(output_data* d, char id, char wire_type); - -/** - * Writes a string to the output buffer. - * - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param s String to output. - * @param len Length of s. - * - * @return int Zero on success, non-zero if out of memory. - */ -int pb_write_string(output_data* d, char id, const char* s, size_t len); - -/** - * Retrieve the string value for a Lua table entry (the table should be on top - * of the stack). If the entry is not found or not a string nothing is encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_string(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index); - -/** - * Retrieve the numeric value for a Lua table entry (the table should be on top - * of the stack). If the entry is not found or not a number nothing is encoded, - * otherwise the number is varint encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_int(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index); - -/** - * Encodes the entry on top of the Lua stack as a double. If the entry is not a - * number nothing is encoded. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_double(lua_sandbox* lsb, output_data* d, char id); - -/** - * Encodes a field that has an array of values. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param ltype Lua type of the array values. - * @param representation String representation of the field i.e., "ms" - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_field_array(lua_sandbox* lsb, output_data* d, int ltype, - const char* representation); - -/** - * Encodes a field that contains metadata in addition to its value. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_field_object(lua_sandbox* lsb, output_data* d); - -/** - * Encodes the field value. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param first Boolean indicator used to add addition protobuf data in the - * correct order. - * @param representation String representation of the field i.e., "ms" - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_field_value(lua_sandbox* lsb, output_data* d, int first, - const char* representation); - -/** - * Updates the field length in the output buffer once the size is known, this - * allows for single pass encoding. - * - * @param d Pointer to the output data buffer. - * @param len_pos Position in the output buffer where the length should be - * written. - * - * @return int Zero on success, non-zero if out of memory. - */ -int update_field_length(output_data* d, size_t len_pos); - -/** - * Iterates over the specified Lua table encoding the contents as user defined - * message fields. - * - * @param lsb Pointer to the sandbox. - * @param d Pointer to the output data buffer. - * @param id Field identifier. - * @param name Key used for the Lua table entry lookup. - * @param index Lua stack index of the table. - * - * @return int Zero on success, non-zero if out of memory. - */ -int encode_fields(lua_sandbox* lsb, output_data* d, char id, const char* name, - int index); - -#endif diff --git a/src/luasandbox.c b/src/luasandbox.c new file mode 100644 index 0000000..d6ff41e --- /dev/null +++ b/src/luasandbox.c @@ -0,0 +1,808 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandboxed implementation @file */ + +#define LUA_LIB +#include "luasandbox.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox/lualib.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox_defines.h" +#include "luasandbox_impl.h" +#include "luasandbox_serialize.h" + +lsb_err_id LSB_ERR_INIT = "already initialized"; +lsb_err_id LSB_ERR_LUA = "lua error"; // use lsb_get_error for details +lsb_err_id LSB_ERR_TERMINATED = "sandbox already terminated"; + +static jmp_buf g_jbuf; + +static const luaL_Reg preload_module_list[] = { + { LUA_BASELIBNAME, luaopen_base }, + { LUA_COLIBNAME, luaopen_coroutine }, + { LUA_TABLIBNAME, luaopen_table }, + { LUA_IOLIBNAME, luaopen_io }, + { LUA_OSLIBNAME, luaopen_os }, + { LUA_STRLIBNAME, luaopen_string }, + { LUA_MATHLIBNAME, luaopen_math }, + { NULL, NULL } +}; + +static int libsize(const luaL_Reg *l) +{ + int size = 0; + for (; l->name; l++) size++; + return size; +} + +static void preload_modules(lua_State *lua) +{ + const luaL_Reg *lib = preload_module_list; + luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", + libsize(preload_module_list)); + for (; lib->func; lib++) { + lua_pushstring(lua, lib->name); + lua_pushcfunction(lua, lib->func); + lua_rawset(lua, -3); + } + lua_pop(lua, 1); // remove the preloaded table +} + + +/** +* Implementation of the memory allocator for the Lua state. +* +* See: http://www.lua.org/manual/5.1/manual.html#lua_Alloc +* +* @param ud Pointer to the lsb_lua_sandbox +* @param ptr Pointer to the memory block being allocated/reallocated/freed. +* @param osize The original size of the memory block. +* @param nsize The new size of the memory block. +* +* @return void* A pointer to the memory block. +*/ +static void* memory_manager(void *ud, void *ptr, size_t osize, size_t nsize) +{ + lsb_lua_sandbox *lsb = (lsb_lua_sandbox *)ud; + + void *nptr = NULL; + if (nsize == 0) { + free(ptr); + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] -= osize; + } else { + size_t new_state_memory = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + nsize - osize; + if (0 == lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] + || new_state_memory + <= lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]) { + nptr = realloc(ptr, nsize); + if (nptr != NULL) { + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] = + new_state_memory; + if (lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT] + > lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; + } + } + } + } + return nptr; +} + + +static size_t instruction_usage(lsb_lua_sandbox *lsb) +{ + return lua_gethookcount(lsb->lua) - lua_gethookcountremaining(lsb->lua); +} + + +static void instruction_manager(lua_State *lua, lua_Debug *ar) +{ + if (LUA_HOOKCOUNT == ar->event) { + luaL_error(lua, "instruction_limit exceeded"); + } +} + + +static int output(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, __func__); + + int n = lua_gettop(lua); + if (n == 0) { + return luaL_argerror(lsb->lua, 0, "must have at least one argument"); + } + lsb_output_coroutine(lsb, lua, 1, n, 1); + return 0; +} + + +static int output_print(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "print() invalid " LSB_THIS_PTR); + + lsb->output.buf[0] = 0; + lsb->output.pos = 0; // clear the buffer + + int n = lua_gettop(lua); + if (!lsb->logger.cb || n == 0) { + return 0; + } + + lua_getglobal(lua, "tostring"); + for (int i = 1; i <= n; ++i) { + lua_pushvalue(lua, -1); // tostring + lua_pushvalue(lua, i); // value + lua_call(lua, 1, 1); + const char *s = lua_tostring(lua, -1); + if (s == NULL) { + return luaL_error(lua, LUA_QL("tostring") " must return a string to " + LUA_QL("print")); + } + if (i > 1) { + lsb_outputc(&lsb->output, '\t'); + } + + while (*s) { + if (isprint(*s)) { + lsb_outputc(&lsb->output, *s); + } else { + lsb_outputc(&lsb->output, ' '); + } + ++s; + } + lua_pop(lua, 1); + } + + const char *component = NULL; + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + // this makes an assumptions by looking for a Heka sandbox specific cfg + // variable but will fall back to the lua filename in the generic case + lua_getfield(lua, -1, "Logger"); + component = lua_tostring(lua, -1); + if (!component) { + component = lsb->lua_file; + } + } + + lsb->logger.cb(lsb->logger.context, component, 7, "%s", lsb->output.buf); + lsb->output.pos = 0; + return 0; +} + + +static int read_config(lua_State *lua) +{ + luaL_checkstring(lua, 1); + luaL_argcheck(lua, lua_gettop(lua) == 1, 0, "too many arguments"); + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + if (lua_type(lua, -1) == LUA_TTABLE) { + lua_getfield(lua, -1, lua_tostring(lua, 1)); + } else { + lua_pushnil(lua); + } + return 1; +} + + +static int unprotected_panic(lua_State *lua) +{ + (void)lua; + longjmp(g_jbuf, 1); + return 0; +} + + +static size_t get_size(lua_State *lua, int idx, const char *item) +{ + lua_getfield(lua, idx, item); + size_t size = (size_t)lua_tonumber(lua, -1); + lua_pop(lua, 1); + return size; +} + + +static int check_string(lua_State *L, int idx, const char *name, + const char *dflt) +{ + lua_getfield(L, idx, name); + int t = lua_type(L, -1); + switch (t) { + case LUA_TSTRING: + break; + case LUA_TNIL: + if (dflt) { + lua_pushstring(L, dflt); // add the default to the config + lua_setglobal(L, name); + } + break; + default: + lua_pushfstring(L, "%s must be set to a string", name); + return 1; + } + return 0; +} + + +static int check_unsigned(lua_State *L, int idx, const char *name, unsigned val) +{ + lua_getfield(L, idx, name); + double d; + switch (lua_type(L, -1)) { + case LUA_TNUMBER: + d = lua_tonumber(L, -1); + if (d < 0 || d > UINT_MAX) { + lua_pushfstring(L, "%s must be an unsigned int", name); + return 1; + } + break; + case LUA_TNIL: // add the default to the config + lua_pushnumber(L, (lua_Number)val); + lua_setglobal(L, name); + break; // use the default + default: + lua_pushfstring(L, "%s must be set to a number", name); + return 1; + } + lua_pop(L, 1); + return 0; +} + + +static int check_size(lua_State *L, int idx, const char *name, size_t val) +{ + lua_getfield(L, idx, name); + double d; + switch (lua_type(L, -1)) { + case LUA_TNUMBER: + d = lua_tonumber(L, -1); + if (d < 0 || d > SIZE_MAX) { + lua_pushfstring(L, "%s must be a size_t", name); + return 1; + } + break; + case LUA_TNIL: // add the default to the config + lua_pushnumber(L, (lua_Number)val); + lua_setglobal(L, name); + break; // use the default + default: + lua_pushfstring(L, "%s must be set to a number", name); + return 1; + } + lua_pop(L, 1); + return 0; +} + + +static lua_State* load_sandbox_config(const char *cfg, lsb_logger *logger) +{ + lua_State *L = luaL_newstate(); + if (!L) { + if (logger->cb) logger->cb(logger->context, __func__, 3, + "lua_State creation failed"); + return NULL; + } + + if (!cfg) cfg = ""; // use the default settings + + int ret = luaL_dostring(L, cfg); + if (ret) goto cleanup; + + ret = check_size(L, LUA_GLOBALSINDEX, LSB_INPUT_LIMIT, 64 * 1024); + if (ret) goto cleanup; + + ret = check_size(L, LUA_GLOBALSINDEX, LSB_OUTPUT_LIMIT, 64 * 1024); + if (ret) goto cleanup; + + ret = check_size(L, LUA_GLOBALSINDEX, LSB_MEMORY_LIMIT, 8 * 1024 * 1024); + if (ret) goto cleanup; + + ret = check_size(L, LUA_GLOBALSINDEX, LSB_INSTRUCTION_LIMIT, 1000000); + if (ret) goto cleanup; + + ret = check_unsigned(L, LUA_GLOBALSINDEX, LSB_LOG_LEVEL, 3); + if (ret) goto cleanup; + + ret = check_string(L, LUA_GLOBALSINDEX, LSB_LUA_PATH, NULL); + if (ret) goto cleanup; + + ret = check_string(L, LUA_GLOBALSINDEX, LSB_LUA_CPATH, NULL); + if (ret) goto cleanup; + +cleanup: + if (ret) { + if (logger->cb) { + logger->cb(logger->context, __func__, 3, "config error: %s", + lua_tostring(L, -1)); + } + lua_close(L); + return NULL; + } + return L; +} + + +static void copy_table(lua_State *sb, lua_State *cfg, lsb_logger *logger) +{ + lua_newtable(sb); + lua_pushnil(cfg); + while (lua_next(cfg, -2) != 0) { + int kt = lua_type(cfg, -2); + int vt = lua_type(cfg, -1); + switch (kt) { + case LUA_TNUMBER: + case LUA_TSTRING: + switch (vt) { + case LUA_TSTRING: + { + size_t len; + const char *tmp = lua_tolstring(cfg, -1, &len); + if (tmp) { + lua_pushlstring(sb, tmp, len); + if (kt == LUA_TSTRING) { + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + } else { + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); + } + } + } + break; + case LUA_TNUMBER: + lua_pushnumber(sb, lua_tonumber(cfg, -1)); + if (kt == LUA_TSTRING) { + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + } else { + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); + } + break; + case LUA_TBOOLEAN: + lua_pushboolean(sb, lua_toboolean(cfg, -1)); + if (kt == LUA_TSTRING) { + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + } else { + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); + } + break; + case LUA_TTABLE: + copy_table(sb, cfg, logger); + break; + default: + if (logger->cb) { + logger->cb(logger->context, __func__, 4, + "skipping config value type: %s", lua_typename(cfg, vt)); + } + break; + } + break; + default: + if (logger->cb) { + logger->cb(logger->context, __func__, 4, "skipping config key type: %s", + lua_typename(cfg, kt)); + } + break; + } + lua_pop(cfg, 1); + } + + switch (lua_type(cfg, -2)) { + case LUA_TSTRING: + lua_setfield(sb, -2, lua_tostring(cfg, -2)); + break; + case LUA_TNUMBER: + lua_rawseti(sb, -2, (int)lua_tointeger(cfg, -2)); + break; + } +} + + +static void set_random_seed() +{ + bool seeded = false; +#ifdef _WIN32 + // todo use CryptGenRandom to seed srand +#else + FILE *fh = fopen("/dev/urandom", "r" CLOSE_ON_EXEC); + if (fh) { + unsigned seed; + unsigned char advance; + if (fread(&seed, sizeof(unsigned), 1, fh) == 1 && + fread(&advance, sizeof(char), 1, fh) == 1) { + srand(seed); + // advance the sequence a random amount + for (unsigned i = 0; i < advance; ++i) { + rand(); + } + seeded = true; + } + fclose(fh); + } +#endif + if (!seeded) { + srand((unsigned)time(NULL)); + } +} + + +lsb_lua_sandbox* lsb_create(void *parent, + const char *lua_file, + const char *cfg, + lsb_logger *logger) +{ + if (!lua_file) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "lua_file must be specified"); + } + return NULL; + } + + if (!lsb_set_tz(NULL)) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "fail to set the TZ to UTC"); + } + return NULL; + } + + set_random_seed(); + + lsb_lua_sandbox *lsb = calloc(1, sizeof(*lsb)); + if (!lsb) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 3, "memory allocation failed"); + } + return NULL; + } + + lsb->lua = lua_newstate(memory_manager, lsb); + if (logger) { + lsb->logger = *logger; + } + + if (!lsb->lua) { + if (lsb->logger.cb) { + lsb->logger.cb(lsb->logger.context, __func__, 3, "lua state creation " + "failed"); + } + free(lsb); + return NULL; + } + + + // add the config to the lsb_config registry table + lua_State *lua_cfg = load_sandbox_config(cfg, &lsb->logger); + if (!lua_cfg) { + lua_close(lsb->lua); + free(lsb); + return NULL; + } + lua_pushnil(lua_cfg); + lua_pushvalue(lua_cfg, LUA_GLOBALSINDEX); + copy_table(lsb->lua, lua_cfg, &lsb->logger); + lua_pop(lua_cfg, 2); + lua_close(lua_cfg); + size_t ml = get_size(lsb->lua, -1, LSB_MEMORY_LIMIT); + size_t il = get_size(lsb->lua, -1, LSB_INSTRUCTION_LIMIT); + size_t ol = get_size(lsb->lua, -1, LSB_OUTPUT_LIMIT); + size_t log_level = get_size(lsb->lua, -1, LSB_LOG_LEVEL); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_CONFIG_TABLE); + lua_pushlightuserdata(lsb->lua, lsb); + lua_setfield(lsb->lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lua_pushcfunction(lsb->lua, &read_config); + lua_setglobal(lsb->lua, "read_config"); + + lua_pushcfunction(lsb->lua, &output); + lua_setglobal(lsb->lua, "output"); + preload_modules(lsb->lua); + + lsb->parent = parent; + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = ml; + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] = il; + lsb->usage[LSB_UT_OUTPUT][LSB_US_LIMIT] = ol; + lsb->state = LSB_UNKNOWN; + lsb->error_message[0] = 0; + lsb->lua_file = malloc(strlen(lua_file) + 1); + lsb->state_file = NULL; + if (log_level != 7) { + lsb->logger.cb = NULL; // only give the sandbox access to the logger (print) + // when debugging + } + + if (!lsb->lua_file || lsb_init_output_buffer(&lsb->output, ol)) { + if (lsb->logger.cb) { + lsb->logger.cb(lsb->logger.context, __func__, 3, "memory allocation " + "failed"); + } + lsb_free_output_buffer(&lsb->output); + free(lsb->lua_file); + lua_close(lsb->lua); + lsb->lua = NULL; + free(lsb); + return NULL; + } + strcpy(lsb->lua_file, lua_file); + return lsb; +} + + +lsb_err_value lsb_init(lsb_lua_sandbox *lsb, const char *state_file) +{ + if (!lsb) { + return LSB_ERR_UTIL_NULL; + } + + if (lsb->state != LSB_UNKNOWN) { + lsb_terminate(lsb, LSB_ERR_INIT); + return LSB_ERR_INIT; + } + + if (state_file && strlen(state_file) > 0) { + lsb->state_file = malloc(strlen(state_file) + 1); + if (!lsb->state_file) { + lsb_terminate(lsb, LSB_ERR_UTIL_OOM); + return LSB_ERR_UTIL_OOM; + } + strcpy(lsb->state_file, state_file); + } + + size_t mem_limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; + + // load package module + lua_pushcfunction(lsb->lua, luaopen_package); + lua_pushstring(lsb->lua, LUA_LOADLIBNAME); + lua_call(lsb->lua, 1, 1); + lua_newtable(lsb->lua); + lua_setmetatable(lsb->lua, -2); + lua_pop(lsb->lua, 1); + + // load base module + lua_getglobal(lsb->lua, "require"); + if (!lua_iscfunction(lsb->lua, -1)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_init() 'require' not found"); + lsb_terminate(lsb, NULL); + return LSB_ERR_LUA; + } + lua_pushstring(lsb->lua, LUA_BASELIBNAME); + if (lua_pcall(lsb->lua, 1, 0, 0)) { + const char *em = lua_tostring(lsb->lua, -1); + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_init %s", em ? em : LSB_NIL_ERROR); + lsb_terminate(lsb, NULL); + return LSB_ERR_LUA; + } + lsb_add_function(lsb, output_print, "print"); + + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { + lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, + (int)lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); + } else { + lua_sethook(lsb->lua, NULL, 0, 0); + } + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = mem_limit; + lua_CFunction pf = lua_atpanic(lsb->lua, unprotected_panic); + int jump = setjmp(g_jbuf); + if (jump || luaL_dofile(lsb->lua, lsb->lua_file) != 0) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "%s", + lua_tostring(lsb->lua, -1)); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, NULL); + return LSB_ERR_LUA; + } else { + lua_gc(lsb->lua, LUA_GCCOLLECT, 0); + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = instruction_usage(lsb); + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] + > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; + } + lsb->state = LSB_RUNNING; + if (lsb->state_file) { + lsb_err_value ret = restore_global_data(lsb); + if (ret) return ret; + } + } + lua_atpanic(lsb->lua, pf); + return NULL; +} + + +static void stop_hook(lua_State *lua, lua_Debug *ar) +{ + (void)ar; /* unused arg. */ + lua_sethook(lua, NULL, 0, 0); + luaL_error(lua, LSB_SHUTTING_DOWN); +} + + +void lsb_stop_sandbox_clean(lsb_lua_sandbox *lsb) +{ + if (!lsb) { + return; + } + lsb->state = LSB_STOP; +} + + +void lsb_stop_sandbox(lsb_lua_sandbox *lsb) +{ + if (!lsb) { + return; + } + + lua_State *lua = lsb_get_lua(lsb); + if (lua) { + lua_sethook(lua, stop_hook, LUA_MASKCALL | LUA_MASKRET | LUA_MASKCOUNT, 1); + } +} + + +char* lsb_destroy(lsb_lua_sandbox *lsb) +{ + char *err = NULL; + if (!lsb) { + return err; + } + + if (preserve_global_data(lsb)) { + size_t len = strlen(lsb->error_message); + err = malloc(len + 1); + if (err != NULL) { + strcpy(err, lsb->error_message); + } + } + + if (lsb->lua) { + lua_close(lsb->lua); + lsb->lua = NULL; + } + + lsb_free_output_buffer(&lsb->output); + free(lsb->state_file); + free(lsb->lua_file); + free(lsb); + return err; +} + + +size_t lsb_usage(lsb_lua_sandbox *lsb, lsb_usage_type utype, + lsb_usage_stat ustat) +{ + if (!lsb || utype >= LSB_UT_MAX || ustat >= LSB_US_MAX) { + return 0; + } + return lsb->usage[utype][ustat]; +} + + +const char* lsb_get_error(lsb_lua_sandbox *lsb) +{ + return lsb ? lsb->error_message : ""; +} + + +void lsb_set_error(lsb_lua_sandbox *lsb, const char *err) +{ + if (lsb) { + if (err) { + strncpy(lsb->error_message, err, LSB_ERROR_SIZE); + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } else { + lsb->error_message[0] = 0; + } + } +} + + +lua_State* lsb_get_lua(lsb_lua_sandbox *lsb) +{ + return lsb ? lsb->lua : NULL; +} + + +const char* lsb_get_lua_file(lsb_lua_sandbox *lsb) +{ + return lsb ? lsb->lua_file : NULL; +} + + +void* lsb_get_parent(lsb_lua_sandbox *lsb) +{ + return lsb ? lsb->parent : NULL; +} + + +const lsb_logger* lsb_get_logger(lsb_lua_sandbox *lsb) +{ + return lsb ? &lsb->logger : NULL; +} + + +lsb_state lsb_get_state(lsb_lua_sandbox *lsb) +{ + return lsb ? lsb->state : LSB_UNKNOWN; +} + + +void lsb_add_function(lsb_lua_sandbox *lsb, lua_CFunction func, + const char *func_name) +{ + if (!lsb || !func || !func_name) return; + if (lsb->state == LSB_TERMINATED) return; + + lua_pushcfunction(lsb->lua, func); + lua_setglobal(lsb->lua, func_name); +} + + +lsb_err_value lsb_pcall_setup(lsb_lua_sandbox *lsb, const char *func_name) +{ + if (!lsb || !func_name) return LSB_ERR_UTIL_NULL; + if (lsb->state == LSB_TERMINATED) return LSB_ERR_TERMINATED; + + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT] != 0) { + lua_sethook(lsb->lua, instruction_manager, LUA_MASKCOUNT, + (int)lsb->usage[LSB_UT_INSTRUCTION][LSB_US_LIMIT]); + } else { + lua_sethook(lsb->lua, NULL, 0, 0); + } + lua_getglobal(lsb->lua, func_name); + if (!lua_isfunction(lsb->lua, -1)) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, "%s() not found", + func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + return LSB_ERR_LUA; + } + return NULL; +} + + +void lsb_pcall_teardown(lsb_lua_sandbox *lsb) +{ + if (!lsb) return; + + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] = + (unsigned)instruction_usage(lsb); + if (lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT] + > lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_INSTRUCTION][LSB_US_CURRENT]; + } +} + + +void lsb_terminate(lsb_lua_sandbox *lsb, const char *err) +{ + if (!lsb) return; + + if (err) { + strncpy(lsb->error_message, err, LSB_ERROR_SIZE); + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + lsb->state = LSB_TERMINATED; +} diff --git a/src/luasandbox_defines.h b/src/luasandbox_defines.h new file mode 100644 index 0000000..aab10fa --- /dev/null +++ b/src/luasandbox_defines.h @@ -0,0 +1,30 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Shared compiler defines @file */ + +#ifndef luasandbox_defines_h_ +#define luasandbox_defines_h_ + +#ifdef _WIN32 +#if _MSC_VER < 1900 +#define snprintf _snprintf +#endif +#endif + +#if __linux +#define CLOSE_ON_EXEC "e" +#else +#define CLOSE_ON_EXEC "" +#endif + +#ifdef _MSC_VER +#ifndef __func__ +#define __func__ __FUNCTION__ +#endif +#endif + +#endif diff --git a/src/luasandbox_impl.h b/src/luasandbox_impl.h new file mode 100644 index 0000000..0a5f8c5 --- /dev/null +++ b/src/luasandbox_impl.h @@ -0,0 +1,46 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Lua sandbox private functions @file */ + +#ifndef luasandbox_impl_h_ +#define luasandbox_impl_h_ + +#include "luasandbox.h" +#include "luasandbox/lua.h" +#include "luasandbox/util/output_buffer.h" + +struct lsb_lua_sandbox { + lua_State *lua; + void *parent; + char *lua_file; + char *state_file; + lsb_logger logger; + lsb_state state; + lsb_output_buffer output; + size_t usage[LSB_UT_MAX][LSB_US_MAX]; + char error_message[LSB_ERROR_SIZE]; +}; + +/** + * Serialize all user global data to disk. + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_err_value NULL on success error message on failure + */ +lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb); + +/** + * Restores previously serialized data from disk. + * + * @param lsb Pointer to the sandbox. + * + * @return lsb_err_value NULL on success error message on failure + */ +lsb_err_value restore_global_data(lsb_lua_sandbox *lsb); + +#endif diff --git a/src/luasandbox_output.c b/src/luasandbox_output.c new file mode 100644 index 0000000..a1bad4c --- /dev/null +++ b/src/luasandbox_output.c @@ -0,0 +1,145 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox output buffer implementation @file */ + +#define LUA_LIB +#include "luasandbox_output.h" + +#include +#include +#include +#include + +#include "luasandbox/lauxlib.h" +#include "luasandbox_impl.h" +#include "luasandbox_serialize.h" + +static const char *output_function = "lsb_output"; +static const char *zero_copy_function = "lsb_zero_copy"; + +void lsb_add_output_function(lua_State *lua, lua_CFunction fp) +{ + lua_pushstring(lua, output_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +lua_CFunction lsb_get_output_function(lua_State *lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, output_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + + +void lsb_add_zero_copy_function(lua_State *lua, lua_CFunction fp) +{ + lua_pushstring(lua, zero_copy_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +lua_CFunction lsb_get_zero_copy_function(lua_State *lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, zero_copy_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + + +void lsb_output(lsb_lua_sandbox *lsb, int start, int end, int append) +{ + lsb_output_coroutine(lsb, lsb->lua, start, end, append); +} + + +void lsb_output_coroutine(lsb_lua_sandbox *lsb, lua_State *lua, int start, + int end, int append) +{ + if (!append) { + lsb->output.pos = 0; + } + + int result = 0; + for (int i = start; result == 0 && i <= end; ++i) { + switch (lua_type(lua, i)) { + case LUA_TNUMBER: + if (lsb_outputd(&lsb->output, lua_tonumber(lua, i))) { + result = 1; + } + break; + case LUA_TSTRING: + { + size_t len; + const char *s = lua_tolstring(lua, i, &len); + if (lsb_outputs(&lsb->output, s, len)) { + result = 1; + } + } + break; + case LUA_TNIL: + if (lsb_outputs(&lsb->output, "nil", 3)) { + result = 1; + } + break; + case LUA_TBOOLEAN: + if (lsb_outputf(&lsb->output, "%s", lua_toboolean(lua, i) + ? "true" : "false")) { + result = 1; + } + break; + case LUA_TUSERDATA: + { + lua_CFunction fp = lsb_get_output_function(lua, i); + if (!fp) { + luaL_argerror(lua, i, "unknown userdata type"); + return; // never reaches here but the compiler doesn't know it + } + lua_pushvalue(lua, i); + lua_pushlightuserdata(lua, &lsb->output); + result = fp(lua); + lua_pop(lua, 2); // remove the copy of the value and the output + } + break; + default: + luaL_argerror(lua, i, "unsupported type"); + break; + } + } + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] = lsb->output.pos; + if (lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT] + > lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM]) { + lsb->usage[LSB_UT_OUTPUT][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_OUTPUT][LSB_US_CURRENT]; + } + + if (result != 0) { + if (lsb->error_message[0] == 0) { + luaL_error(lua, "output_limit exceeded"); + } + luaL_error(lua, lsb->error_message); + } +} + + +const char* lsb_get_output(lsb_lua_sandbox *lsb, size_t *len) +{ + if (len) *len = lsb->output.pos; + if (lsb->output.pos == 0) return ""; + lsb->output.pos = 0; // internally reset the buffer for the next write + return lsb->output.buf; +} diff --git a/src/luasandbox_serialize.c b/src/luasandbox_serialize.c new file mode 100644 index 0000000..5297f86 --- /dev/null +++ b/src/luasandbox_serialize.c @@ -0,0 +1,576 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Sandbox serialization implementation @file */ + +#define LUA_LIB +#include "luasandbox_serialize.h" + +#include +#include +#include + +#include "luasandbox/lauxlib.h" +#include "luasandbox/lualib.h" +#include "luasandbox_defines.h" +#include "luasandbox_impl.h" + +#ifdef _MSC_VER +#pragma warning( disable : 4056 ) +#endif + +typedef struct +{ + const void *ptr; + size_t name_pos; +} table_ref; + +typedef struct +{ + size_t size; + size_t pos; + table_ref *array; +} table_ref_array; + +typedef struct +{ + FILE *fh; + lsb_output_buffer keys; + table_ref_array tables; + const void *globals; +} serialization_data; + +static const char *preservation_version = "_PRESERVATION_VERSION"; +static const char *serialize_function = "lsb_serialize"; + +/** + * Serializes a Lua table structure. + * + * @param lsb Pointer to the sandbox. + * @param data Pointer to the serialization state data. + * @param parent Index pointing to the parent's name in the table array. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); + +/** + * Serializes a Lua data value. + * + * @param lsb Pointer to the sandbox. + * @param index Lua stack index where the data resides. + * @param ob Pointer the output buffer. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value serialize_data(lsb_lua_sandbox *lsb, + int index, + lsb_output_buffer *ob); + +/** + * Serializes a table key value pair. + * + * @param lsb Pointer to the sandbox. + * @param data Pointer to the serialization state data. + * @param parent Index pointing to the parent's name in the table array. + * + * @return lsb_err_value NULL on success error message on failure + */ +static lsb_err_value +serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent); + +/** + * Returns the serialization function if the userdata implemented one. + * + * @param lua Lua State. + * @param index Position on the stack where the userdata pointer resides. + * + * @return lua_CFunction NULL if not found + */ +static lua_CFunction get_serialize_function(lua_State *lua, int index) +{ + lua_CFunction fp = NULL; + lua_getfenv(lua, index); + lua_pushstring(lua, serialize_function); + lua_rawget(lua, -2); + fp = lua_tocfunction(lua, -1); + lua_pop(lua, 2); // environment and field + return fp; +} + +/** + * Helper function to determine what data should not be serialized. + * + * @param lsb Pointer to the sandbox. + * @param data Pointer to the serialization state data. + * @param index Lua stack index where the data resides. + * @param fp Function pointer to be set for userdata serialization + * + * @return int + */ +static int +ignore_value_type(lsb_lua_sandbox *lsb, serialization_data *data, int index, + lua_CFunction *fp) +{ + switch (lua_type(lsb->lua, index)) { + case LUA_TSTRING: + case LUA_TNUMBER: + case LUA_TBOOLEAN: + return 0; + case LUA_TTABLE: + if (lua_getmetatable(lsb->lua, index) != 0) { + lua_pop(lsb->lua, 1); // Remove the metatable. + return 1; + } + if (lua_topointer(lsb->lua, index) == data->globals) { + return 1; + } + return 0; + case LUA_TUSERDATA: + *fp = get_serialize_function(lsb->lua, index); + return !*fp ? 1 : 0; + default: + break; + } + return 1; +} + + +/** + * Looks for a table to see if it has already been processed. + * + * @param tra Pointer to the table references. + * @param ptr Pointer value of the table. + * + * @return table_ref* NULL if not found. + */ +static table_ref* find_table_ref(table_ref_array *tra, const void *ptr) +{ + for (size_t i = 0; i < tra->pos; ++i) { + if (ptr == tra->array[i].ptr) { + return &tra->array[i]; + } + } + return NULL; +} + + +/** + * Adds a table to the processed array. + * + * @param tra Pointer to the table references. + * @param ptr Pointer value of the table. + * @param name_pos Index pointing to name in the table array. + * + * @return table_ref* Pointer to the table reference or NULL if out of memory. + */ +static table_ref* +add_table_ref(table_ref_array *tra, const void *ptr, size_t name_pos) +{ + if (tra->pos == tra->size) { + size_t newsize = tra->size * 2; + void *p = realloc(tra->array, newsize * sizeof(table_ref)); + if (p != NULL) { + tra->array = p; + tra->size = newsize; + } else { + return NULL; + } + } + tra->array[tra->pos].ptr = ptr; + tra->array[tra->pos].name_pos = name_pos; + return &tra->array[tra->pos++]; +} + + +/** + * Extracts the current preservation from the sandbox. + * + * @param lua Lua state + * + * @return int Version numebr + */ +static int get_preservation_version(lua_State *lua) +{ + int ver = 0; + lua_getglobal(lua, preservation_version); + int t = lua_type(lua, -1); + if (t == LUA_TNUMBER) { + ver = (int)lua_tointeger(lua, -1); + } + lua_pop(lua, 1); // remove the version from the stack + + if (t != LUA_TNIL) { // remove the version from the data preservation + lua_pushnil(lua); + lua_setglobal(lua, preservation_version); + } + return ver; +} + + +static lsb_err_value +serialize_table(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) +{ + lsb_err_value ret = NULL; + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (!ret && lua_next(lsb->lua, -2) != 0) { + ret = serialize_kvp(lsb, data, parent); + lua_pop(lsb->lua, 1); // Remove the value leaving the key on top for + // the next interation. + } + return ret; +} + + +static lsb_err_value +serialize_data(lsb_lua_sandbox *lsb, int index, lsb_output_buffer *ob) +{ + lsb_err_value ret = NULL; + ob->pos = 0; // clear the buffer + switch (lua_type(lsb->lua, index)) { + case LUA_TNUMBER: + ret = lsb_serialize_double(ob, lua_tonumber(lsb->lua, index)); + break; + case LUA_TSTRING: + // The stack is cleaned up on failure by preserve_global_data + // but for clarity it is incrementally cleaned up anyway. + lua_checkstack(lsb->lua, 4); + lua_getglobal(lsb->lua, LUA_STRLIBNAME); + lua_getfield(lsb->lua, -1, "format"); + if (!lua_isfunction(lsb->lua, -1)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "serialize_data cannot access the string format function"); + lua_pop(lsb->lua, 2); // Remove the bogus format function and + // string table. + return LSB_ERR_LUA; + } + lua_pushstring(lsb->lua, "%q"); + lua_pushvalue(lsb->lua, index - 3); + if (lua_pcall(lsb->lua, 2, 1, 0) == 0) { + const char* em = lua_tostring(lsb->lua, -1); + ret = lsb_outputf(ob, "%s", em ? em : LSB_NIL_ERROR); + if (ret) { + lua_pop(lsb->lua, 1); // Remove the string table. + return ret; + } + } else { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, + "serialize_data '%s'", + lua_tostring(lsb->lua, -1)); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + lua_pop(lsb->lua, 2); // Remove the error message and the string + // table. + return LSB_ERR_LUA; + } + lua_pop(lsb->lua, 2); // Remove the pcall result and the string table. + break; + case LUA_TBOOLEAN: + ret = lsb_outputf(ob, "%s", lua_toboolean(lsb->lua, index) ? "true" : + "false"); + break; + default: + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "serialize_data cannot preserve type '%s'", + lua_typename(lsb->lua, lua_type(lsb->lua, index))); + ret = LSB_ERR_LUA; + } + return ret; +} + + +static lsb_err_value +serialize_kvp(lsb_lua_sandbox *lsb, serialization_data *data, size_t parent) +{ + lsb_err_value ret = NULL; + lua_CFunction fp = NULL; + int kindex = -2, vindex = -1; + if (ignore_value_type(lsb, data, vindex, &fp)) { + return ret; + } + ret = serialize_data(lsb, kindex, &lsb->output); + if (ret) { + return ret; + } + + size_t pos = data->keys.pos; + ret = lsb_outputf(&data->keys, "%s[%s]", data->keys.buf + parent, + lsb->output.buf); + if (ret) return ret; + + if (lua_type(lsb->lua, vindex) == LUA_TTABLE) { + const void *ptr = lua_topointer(lsb->lua, vindex); + table_ref *seen = find_table_ref(&data->tables, ptr); + if (seen == NULL) { + seen = add_table_ref(&data->tables, ptr, pos); + if (seen != NULL) { + data->keys.pos += 1; + if (lua_tabletype(lsb->lua, vindex) == LUA_TTARRAY + && lua_objlen(lsb->lua, vindex) == 0) { + fprintf(data->fh, "%s = {nil}\n", data->keys.buf + pos); + } else { + fprintf(data->fh, "%s = {}\n", data->keys.buf + pos); + ret = serialize_table(lsb, data, pos); + } + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_serialize preserve table out of memory"); + return LSB_ERR_UTIL_OOM; + } + } else { + fprintf(data->fh, "%s = ", data->keys.buf + pos); + data->keys.pos = pos; + fprintf(data->fh, "%s\n", data->keys.buf + seen->name_pos); + } + } else if (lua_type(lsb->lua, vindex) == LUA_TUSERDATA) { + void *ud = lua_touserdata(lsb->lua, vindex); + table_ref *seen = find_table_ref(&data->tables, ud); + if (seen == NULL) { + seen = add_table_ref(&data->tables, ud, pos); + if (seen != NULL) { + data->keys.pos += 1; + lua_pushlightuserdata(lsb->lua, data->keys.buf + pos); + lua_pushlightuserdata(lsb->lua, &lsb->output); + lsb->output.pos = 0; + int result = fp(lsb->lua); + lua_pop(lsb->lua, 2); // remove the key and the output + if (!result) { + size_t n = fwrite(lsb->output.buf, 1, lsb->output.pos, data->fh); + if (n != lsb->output.pos) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_serialize failed %s", data->keys.buf + pos); + return LSB_ERR_LUA; + } + } + } else { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "lsb_serialize out of memory %s", data->keys.buf + pos); + return LSB_ERR_UTIL_OOM; + } + } else { + fprintf(data->fh, "%s = ", data->keys.buf + pos); + data->keys.pos = pos; + fprintf(data->fh, "%s\n", data->keys.buf + + seen->name_pos); + } + } else { + fprintf(data->fh, "%s = ", data->keys.buf + pos); + data->keys.pos = pos; + ret = serialize_data(lsb, vindex, &lsb->output); + if (!ret) { + fprintf(data->fh, "%s\n", lsb->output.buf); + } + } + return ret; +} + + +lsb_err_value preserve_global_data(lsb_lua_sandbox *lsb) +{ + + if (!lsb->lua || !lsb->state_file || lsb->state == LSB_TERMINATED) { + return NULL; + } + lua_sethook(lsb->lua, NULL, 0, 0); + + // make sure the string library is loaded before we start + lua_getglobal(lsb->lua, LUA_STRLIBNAME); + if (!lua_istable(lsb->lua, -1)) { + lua_getglobal(lsb->lua, "require"); + if (!lua_iscfunction(lsb->lua, -1)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "preserve_global_data 'require' function not found"); + return LSB_ERR_LUA; + } + lua_pushstring(lsb->lua, LUA_STRLIBNAME); + if (lua_pcall(lsb->lua, 1, 1, 0)) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, + "preserve_global_data failed loading 'string'"); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + return LSB_ERR_LUA; + } + } + lua_pop(lsb->lua, 1); + + lua_pushvalue(lsb->lua, LUA_GLOBALSINDEX); + + FILE *fh = fopen(lsb->state_file, "wb" CLOSE_ON_EXEC); + if (fh == NULL) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, + "preserve_global_data could not open: %s", + lsb->state_file); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + return LSB_ERR_LUA;; + } + + lsb_err_value ret = NULL; + serialization_data data; + data.fh = fh; + +// Clear the sandbox limits during preservation. +// size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; +// size_t cur_output_size = lsb->output.size; +// size_t max_output_size = lsb->output.maxsize; + lsb->output.maxsize = 0; +// end clear + + data.tables.size = 64; + data.tables.pos = 0; + data.tables.array = malloc(data.tables.size * sizeof(table_ref)); + if (data.tables.array == NULL || lsb_init_output_buffer(&data.keys, 0)) { + snprintf(lsb->error_message, LSB_ERROR_SIZE, + "preserve_global_data out of memory"); + ret = LSB_ERR_UTIL_OOM; + } else { + fprintf(data.fh, "if %s and %s ~= %d then return end\n", + preservation_version, + preservation_version, + get_preservation_version(lsb->lua)); + ret = lsb_outputs(&data.keys, "_G", 2); + if (!ret) { + data.keys.pos += 1; // preserve the NUL in this use case + data.globals = lua_topointer(lsb->lua, -1); + lua_checkstack(lsb->lua, 2); + lua_pushnil(lsb->lua); + while (!ret && lua_next(lsb->lua, -2) != 0) { + ret = serialize_kvp(lsb, &data, 0); + lua_pop(lsb->lua, 1); + } + } + lua_pop(lsb->lua, lua_gettop(lsb->lua)); + // Wipe the entire Lua stack. Since incremental cleanup on failure + // was added the stack should only contain table _G. + } + free(data.tables.array); + lsb_free_output_buffer(&data.keys); + fclose(fh); + if (ret) remove(lsb->state_file); + +// Uncomment if we start preserving state when not destroying the sandbox +// Note: serialization uses the output buffer, inprogress output can be +// destroyed if the user was collecting output between calls. +/* +// Restore the sandbox limits after preservation + lua_gc(lsb->lua, LUA_GCCOLLECT, 0); + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; + lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; + lsb->output.maxsize = max_output_size; + lsb_clear_output_buffer(lsb->output); + if (lsb->output.size > cur_output_size) { + void* ptr = realloc(lsb->output.data, cur_output_size); + if (!ptr) return 1; + lsb->output.data = ptr; + lsb->output.size = cur_output_size; + } +// end restore +*/ + return ret; +} + + +static int file_exists(const char *fn) +{ + FILE *fh = fopen(fn, "r" CLOSE_ON_EXEC); + if (fh) { + fclose(fh); + return 1; + } + return 0; +} + + +lsb_err_value restore_global_data(lsb_lua_sandbox *lsb) +{ + if (!lsb) { + return LSB_ERR_UTIL_NULL; + } + if (!lsb->state_file || !file_exists(lsb->state_file)) { + return NULL; + } + + // Clear the sandbox limits during restoration. + size_t limit = lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT]; + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = 0; + lua_sethook(lsb->lua, NULL, 0, 0); + + int err = luaL_dofile(lsb->lua, lsb->state_file); + if (err) { + if (LUA_ERRFILE != err) { + int len = snprintf(lsb->error_message, LSB_ERROR_SIZE, + "restore_global_data %s", + lua_tostring(lsb->lua, -1)); + if (len >= LSB_ERROR_SIZE || len < 0) { + lsb->error_message[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, NULL); + return LSB_ERR_LUA; + } + } + lua_gc(lsb->lua, LUA_GCCOLLECT, 0); + lsb->usage[LSB_UT_MEMORY][LSB_US_LIMIT] = limit; + lsb->usage[LSB_UT_MEMORY][LSB_US_MAXIMUM] = + lsb->usage[LSB_UT_MEMORY][LSB_US_CURRENT]; + return NULL; +} + + +void lsb_add_serialize_function(lua_State *lua, lua_CFunction fp) +{ + lua_pushstring(lua, serialize_function); + lua_pushcfunction(lua, fp); + lua_rawset(lua, -3); +} + + +lsb_err_value lsb_serialize_binary(lsb_output_buffer *ob, const void *src, size_t len) +{ + lsb_err_value ret = NULL; + const char *uc = (const char *)src; + for (unsigned i = 0; !ret && i < len; ++i) { + switch (uc[i]) { + case '\n': + ret = lsb_outputs(ob, "\\n", 2); + break; + case '\r': + ret = lsb_outputs(ob, "\\r", 2); + break; + case '"': + ret = lsb_outputs(ob, "\\\"", 2); + break; + case '\\': + ret = lsb_outputs(ob, "\\\\", 2); + break; + default: + ret = lsb_outputc(ob, uc[i]); + break; + } + } + return ret; +} + + +lsb_err_value lsb_serialize_double(lsb_output_buffer *ob, double d) +{ + if (isnan(d)) { + return lsb_outputs(ob, "0/0", 3); + } + if (d == INFINITY) { + return lsb_outputs(ob, "1/0", 3); + } + if (d == -INFINITY) { + return lsb_outputs(ob, "-1/0", 4); + } + return lsb_outputfd(ob, d); +} diff --git a/src/redis_hyperloglog.c b/src/redis_hyperloglog.c deleted file mode 100644 index 8e5c450..0000000 --- a/src/redis_hyperloglog.c +++ /dev/null @@ -1,503 +0,0 @@ -/* hyperloglog.c - Redis HyperLogLog probabilistic cardinality approximation. - * This file implements the algorithm and the exported Redis commands. - * - * Copyright (c) 2014, Salvatore Sanfilippo - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of Redis nor the names of its contributors may be used - * to endorse or promote products derived from this software without - * specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* This file has been modified for use in the Mozilla lua_sandbox. Dependencies - on the redis.h header file and the unneed sparse and raw implementations have - been removed. The dense data representation remains unchanged. */ - -#include -#include -#include - -#include "lua_hyperloglog.h" - - -/* The Redis HyperLogLog implementation is based on the following ideas: - * - * * The use of a 64 bit hash function as proposed in [1], in order to don't - * limited to cardinalities up to 10^9, at the cost of just 1 additional - * bit per register. - * * The use of 16384 6-bit registers for a great level of accuracy, using - * a total of 12k per key. - * * The use of the Redis string data type. No new type is introduced. - * * No attempt is made to compress the data structure as in [1]. Also the - * algorithm used is the original HyperLogLog Algorithm as in [2], with - * the only difference that a 64 bit hash function is used, so no correction - * is performed for values near 2^32 as in [1]. - * - * [1] Heule, Nunkesser, Hall: HyperLogLog in Practice: Algorithmic - * Engineering of a State of The Art Cardinality Estimation Algorithm. - * - * [2] P. Flajolet, �?ric Fusy, O. Gandouet, and F. Meunier. Hyperloglog: The - * analysis of a near-optimal cardinality estimation algorithm. - * - * The "dense" representation where every entry is represented by a - * 6-bit integer. - * - * HLL header - * === - * - * Both the dense and sparse representation have a 16 byte header as follows: - * - * +------+---+-----+----------+ - * | HYLL | E | N/U | Cardin. | - * +------+---+-----+----------+ - * - * The first 4 bytes are a magic string set to the bytes "HYLL". - * "E" is one byte encoding, currently set to HLL_DENSE or - * HLL_SPARSE. N/U are three not used bytes. - * - * The "Cardin." field is a 64 bit integer stored in little endian format - * with the latest cardinality computed that can be reused if the data - * structure was not modified since the last computation (this is useful - * because there are high probabilities that HLLADD operations don't - * modify the actual data structure and hence the approximated cardinality). - * - * When the most significant bit in the most significant byte of the cached - * cardinality is set, it means that the data structure was modified and - * we can't reuse the cached value that must be recomputed. - * - * Dense representation - * === - * - * The dense representation used by Redis is the following: - * - * +--------+--------+--------+------// //--+ - * |11000000|22221111|33333322|55444444 .... | - * +--------+--------+--------+------// //--+ - * - * The 6 bits counters are encoded one after the other starting from the - * LSB to the MSB, and using the next bytes as needed. - * - */ - -#define HLL_P_MASK (HLL_REGISTERS-1) /* Mask to index register. */ -#define HLL_REGISTER_MAX ((1< 6 - * - * Right shift b0 of 'fb' bits. - * - * +--------+ - * |11000000| <- Initial value of b0 - * |00000011| <- After right shift of 6 pos. - * +--------+ - * - * Left shift b1 of bits 8-fb bits (2 bits) - * - * +--------+ - * |22221111| <- Initial value of b1 - * |22111100| <- After left shift of 2 bits. - * +--------+ - * - * OR the two bits, and finally AND with 111111 (63 in decimal) to - * clean the higher order bits we are not interested in: - * - * +--------+ - * |00000011| <- b0 right shifted - * |22111100| <- b1 left shifted - * |22111111| <- b0 OR b1 - * | 111111| <- (b0 OR b1) AND 63, our value. - * +--------+ - * - * We can try with a different example, like pos = 0. In this case - * the 6-bit counter is actually contained in a single byte. - * - * b0 = 6 * pos / 8 = 0 - * - * +--------+ - * |11000000| <- Our byte at b0 - * +--------+ - * - * fb = 6 * pos % 8 = 0 - * - * So we right shift of 0 bits (no shift in practice) and - * left shift the next byte of 8 bits, even if we don't use it, - * but this has the effect of clearing the bits so the result - * will not be affacted after the OR. - * - * ------------------------------------------------------------------------- - * - * Setting the register is a bit more complex, let's assume that 'val' - * is the value we want to set, already in the right range. - * - * We need two steps, in one we need to clear the bits, and in the other - * we need to bitwise-OR the new bits. - * - * Let's try with 'pos' = 1, so our first byte at 'b' is 0, - * - * "fb" is 6 in this case. - * - * +--------+ - * |11000000| <- Our byte at b0 - * +--------+ - * - * To create a AND-mask to clear the bits about this position, we just - * initialize the mask with the value 63, left shift it of "fs" bits, - * and finally invert the result. - * - * +--------+ - * |00111111| <- "mask" starts at 63 - * |11000000| <- "mask" after left shift of "ls" bits. - * |00111111| <- "mask" after invert. - * +--------+ - * - * Now we can bitwise-AND the byte at "b" with the mask, and bitwise-OR - * it with "val" left-shifted of "ls" bits to set the new bits. - * - * Now let's focus on the next byte b1: - * - * +--------+ - * |22221111| <- Initial value of b1 - * +--------+ - * - * To build the AND mask we start again with the 63 value, right shift - * it by 8-fb bits, and invert it. - * - * +--------+ - * |00111111| <- "mask" set at 2&6-1 - * |00001111| <- "mask" after the right shift by 8-fb = 2 bits - * |11110000| <- "mask" after bitwise not. - * +--------+ - * - * Now we can mask it with b+1 to clear the old bits, and bitwise-OR - * with "val" left-shifted by "rs" bits to set the new value. - */ - -/* Note: if we access the last counter, we will also access the b+1 byte - * that is out of the array, but sds strings always have an implicit null - * term, so the byte exists, and we can skip the conditional (or the need - * to allocate 1 byte more explicitly). */ - -/* Store the value of the register at position 'regnum' into variable 'target'. - * 'p' is an array of unsigned bytes. */ -#define HLL_DENSE_GET_REGISTER(target,p,regnum) do { \ - uint8_t *_p = (uint8_t*) p; \ - unsigned long _byte = regnum*HLL_BITS/8; \ - unsigned long _fb = regnum*HLL_BITS&7; \ - unsigned long _fb8 = 8 - _fb; \ - unsigned long b0 = _p[_byte]; \ - unsigned long b1 = _p[_byte+1]; \ - target = ((b0 >> _fb) | (b1 << _fb8)) & HLL_REGISTER_MAX; \ -} while(0) - -/* Set the value of the register at position 'regnum' to 'val'. - * 'p' is an array of unsigned bytes. */ -#define HLL_DENSE_SET_REGISTER(p,regnum,val) do { \ - uint8_t *_p = (uint8_t*) p; \ - unsigned long _byte = regnum*HLL_BITS/8; \ - unsigned long _fb = regnum*HLL_BITS&7; \ - unsigned long _fb8 = 8 - _fb; \ - unsigned long _v = val; \ - _p[_byte] &= ~(HLL_REGISTER_MAX << _fb); \ - _p[_byte] |= _v << _fb; \ - _p[_byte+1] &= ~(HLL_REGISTER_MAX >> _fb8); \ - _p[_byte+1] |= _v >> _fb8; \ -} while(0) - - -/* ========================= HyperLogLog algorithm ========================= */ - -/* Our hash function is MurmurHash2, 64 bit version. - * It was modified for Redis in order to provide the same result in - * big and little endian archs (endian neutral). */ -uint64_t MurmurHash64A(const void* key, int len, unsigned int seed) -{ - const uint64_t m = 0xc6a4a7935bd1e995; - const int r = 47; - uint64_t h = seed ^ (len * m); - const uint8_t* data = (const uint8_t*)key; - const uint8_t* end = data + (len - (len & 7)); - - while (data != end) { - uint64_t k; - -#if (BYTE_ORDER == LITTLE_ENDIAN) - k = *((uint64_t*)data); -#else - k = (uint64_t)data[0]; - k |= (uint64_t)data[1] << 8; - k |= (uint64_t)data[2] << 16; - k |= (uint64_t)data[3] << 24; - k |= (uint64_t)data[4] << 32; - k |= (uint64_t)data[5] << 40; - k |= (uint64_t)data[6] << 48; - k |= (uint64_t)data[7] << 56; -#endif - - k *= m; - k ^= k >> r; - k *= m; - h ^= k; - h *= m; - data += 8; - } - - switch (len & 7) { - case 7: - h ^= (uint64_t)data[6] << 48; - case 6: - h ^= (uint64_t)data[5] << 40; - case 5: - h ^= (uint64_t)data[4] << 32; - case 4: - h ^= (uint64_t)data[3] << 24; - case 3: - h ^= (uint64_t)data[2] << 16; - case 2: - h ^= (uint64_t)data[1] << 8; - case 1: - h ^= (uint64_t)data[0]; - h *= m; - }; - - h ^= h >> r; - h *= m; - h ^= h >> r; - return h; -} - -/* Given a string element to add to the HyperLogLog, returns the length - * of the pattern 000..1 of the element hash. As a side effect 'regp' is - * set to the register index this element hashes to. */ -int hllPatLen(unsigned char* ele, size_t elesize, long* regp) -{ - uint64_t hash, bit, index; - int count; - - /* Count the number of zeroes starting from bit HLL_REGISTERS - * (that is a power of two corresponding to the first bit we don't use - * as index). The max run can be 64-P+1 bits. - * - * Note that the final "1" ending the sequence of zeroes must be - * included in the count, so if we find "001" the count is 3, and - * the smallest count possible is no zeroes at all, just a 1 bit - * at the first position, that is a count of 1. - * - * This may sound like inefficient, but actually in the average case - * there are high probabilities to find a 1 after a few iterations. */ - hash = MurmurHash64A(ele, (int)elesize, 0xadc83b19ULL); - index = hash & HLL_P_MASK; /* Register index. */ - hash |= ((uint64_t)1 << 63); /* Make sure the loop terminates. */ - bit = HLL_REGISTERS; /* First bit not used to address the register. */ - count = 1; /* Initialized to 1 since we count the "00000...1" pattern. */ - while ((hash & bit) == 0) { - count++; - bit <<= 1; - } - *regp = (int)index; - return count; -} - -/* ================== Dense representation implementation ================== */ - -/* "Add" the element in the dense hyperloglog data structure. - * Actually nothing is added, but the max 0 pattern counter of the subset - * the element belongs to is incremented if needed. - * - * 'registers' is expected to have room for HLL_REGISTERS plus an - * additional byte on the right. This requirement is met by sds strings - * automatically since they are implicitly null terminated. - * - * The function always succeed, however if as a result of the operation - * the approximated cardinality changed, 1 is returned. Otherwise 0 - * is returned. */ -int hllDenseAdd(uint8_t* registers, unsigned char* ele, size_t elesize) -{ - uint8_t oldcount, count; - long index; - - /* Update the register if this element produced a longer run of zeroes. */ - count = hllPatLen(ele, elesize, &index); - HLL_DENSE_GET_REGISTER(oldcount, registers, index); - if (count > oldcount) { - HLL_DENSE_SET_REGISTER(registers, index, count); - return 1; - } else { - return 0; - } -} - -/* Compute SUM(2^-reg) in the dense representation. - * PE is an array with a pre-computer table of values 2^-reg indexed by reg. - * As a side effect the integer pointed by 'ezp' is set to the number - * of zero registers. */ -double hllDenseSum(uint8_t* registers, double* PE, int* ezp) -{ - double E = 0; - int j, ez = 0; - - /* Redis default is to use 16384 registers 6 bits each. The code works - * with other values by modifying the defines, but for our target value - * we take a faster path with unrolled loops. */ - if (HLL_REGISTERS == 16384 && HLL_BITS == 6) { - uint8_t* r = registers; - unsigned long r0, r1, r2, r3, r4, r5, r6, r7, r8, r9, - r10, r11, r12, r13, r14, r15; - for (j = 0; j < 1024; j++) { - /* Handle 16 registers per iteration. */ - r0 = r[0] & 63; if (r0 == 0) ez++; - r1 = (r[0] >> 6 | r[1] << 2) & 63; if (r1 == 0) ez++; - r2 = (r[1] >> 4 | r[2] << 4) & 63; if (r2 == 0) ez++; - r3 = (r[2] >> 2) & 63; if (r3 == 0) ez++; - r4 = r[3] & 63; if (r4 == 0) ez++; - r5 = (r[3] >> 6 | r[4] << 2) & 63; if (r5 == 0) ez++; - r6 = (r[4] >> 4 | r[5] << 4) & 63; if (r6 == 0) ez++; - r7 = (r[5] >> 2) & 63; if (r7 == 0) ez++; - r8 = r[6] & 63; if (r8 == 0) ez++; - r9 = (r[6] >> 6 | r[7] << 2) & 63; if (r9 == 0) ez++; - r10 = (r[7] >> 4 | r[8] << 4) & 63; if (r10 == 0) ez++; - r11 = (r[8] >> 2) & 63; if (r11 == 0) ez++; - r12 = r[9] & 63; if (r12 == 0) ez++; - r13 = (r[9] >> 6 | r[10] << 2) & 63; if (r13 == 0) ez++; - r14 = (r[10] >> 4 | r[11] << 4) & 63; if (r14 == 0) ez++; - r15 = (r[11] >> 2) & 63; if (r15 == 0) ez++; - - /* Additional parens will allow the compiler to optimize the - * code more with a loss of precision that is not very relevant - * here (floating point math is not commutative!). */ - E += (PE[r0] + PE[r1]) + (PE[r2] + PE[r3]) + (PE[r4] + PE[r5]) + - (PE[r6] + PE[r7]) + (PE[r8] + PE[r9]) + (PE[r10] + PE[r11]) + - (PE[r12] + PE[r13]) + (PE[r14] + PE[r15]); - r += 12; - } - } else { - for (j = 0; j < HLL_REGISTERS; j++) { - unsigned long reg; - - HLL_DENSE_GET_REGISTER(reg, registers, j); - if (reg == 0) { - ez++; - /* Increment E at the end of the loop. */ - } else { - E += PE[reg]; /* Precomputed 2^(-reg[j]). */ - } - } - E += ez; /* Add 2^0 'ez' times. */ - } - *ezp = ez; - return E; -} - -/* ========================= HyperLogLog Count ============================== - * This is the core of the algorithm where the approximated count is computed. - * The function uses the lower level hllDenseSum() function as helpers to - * compute the SUM(2^-reg) part of the computation, which is - * representation-specific, while all the rest is common. */ - -/* Return the approximated cardinality of the set based on the harmonic - * mean of the registers values. 'hdr' points to the start of the SDS - * representing the String object holding the HLL representation. - * - * If the sparse representation of the HLL object is not valid, the integer - * pointed by 'invalid' is set to non-zero, otherwise it is left untouched. */ -uint64_t hllCount(hyperloglog* hdr) -{ - double m = HLL_REGISTERS; - double E, alpha = 0.7213 / (1 + 1.079 / m); - int j, ez; /* Number of registers equal to 0. */ - -/* We precompute 2^(-reg[j]) in a small table in order to - * speedup the computation of SUM(2^-register[0..i]). */ - static int initialized = 0; - static double PE[64]; - if (!initialized) { - PE[0] = 1; /* 2^(-reg[j]) is 1 when m is 0. */ - for (j = 1; j < 64; j++) { - /* 2^(-reg[j]) is the same as 1/2^reg[j]. */ - PE[j] = 1.0 / (1ULL << j); - } - initialized = 1; - } - -/* Compute SUM(2^-register[0..i]). */ - E = hllDenseSum(hdr->registers, PE, &ez); - -/* Muliply the inverse of E for alpha_m * m^2 to have the raw estimate. */ - E = (1 / E) * alpha * m * m; - -/* Use the LINEARCOUNTING algorithm for small cardinalities. - * For larger values but up to 72000 HyperLogLog raw approximation is - * used since linear counting error starts to increase. However HyperLogLog - * shows a strong bias in the range 2.5*16384 - 72000, so we try to - * compensate for it. */ - if (E < m * 2.5 && ez != 0) { - E = m * log(m / ez); /* LINEARCOUNTING() */ - } else if (m == 16384 && E < 72000) { - /* We did polynomial regression of the bias for this range, this - * way we can compute the bias for a given cardinality and correct - * according to it. Only apply the correction for P=14 that's what - * we use and the value the correction was verified with. */ - double bias = 5.9119 * 1.0e-18 * (E * E * E * E) - - 1.4253 * 1.0e-12 * (E * E * E) + - 1.2940 * 1.0e-7 * (E * E) - - 5.2921 * 1.0e-3 * E + - 83.3216; - E -= E * (bias / 100); - } -/* We don't apply the correction for E > 1/30 of 2^32 since we use - * a 64 bit function and 6 bit counters. To apply the correction for - * 1/30 of 2^64 is not needed since it would require a huge set - * to approach such a value. */ - return (uint64_t)E; -} diff --git a/src/struct.c b/src/struct.c deleted file mode 100644 index 339df2f..0000000 --- a/src/struct.c +++ /dev/null @@ -1,420 +0,0 @@ -/* -** {====================================================== -** Library for packing/unpacking structures. -** $Id: struct.c,v 1.4 2012/07/04 18:54:29 roberto Exp $ -** See Copyright Notice at the end of this file -** ======================================================= -*/ -/* -** Valid formats: -** > - big endian -** < - little endian -** ![num] - alignment -** x - pading -** b/B - signed/unsigned byte -** h/H - signed/unsigned short -** l/L - signed/unsigned long -** T - size_t -** i/In - signed/unsigned integer with size `n' (default is size of int) -** cn - sequence of `n' chars (from/to a string); when packing, n==0 means - the whole string; when unpacking, n==0 means use the previous - read number as the string length -** s - zero-terminated string -** f - float -** d - double -** ' ' - ignored -*/ - - -#include -#include -#include -#include -#include - - -#include "lua.h" -#include "lauxlib.h" - - -#if (LUA_VERSION_NUM >= 502) - -#define luaL_register(L,n,f) luaL_newlib(L,f) - -#endif - - -/* basic integer type */ -#if !defined(STRUCT_INT) -#define STRUCT_INT long long -#endif - -typedef STRUCT_INT Inttype; - -/* corresponding unsigned version */ -typedef unsigned STRUCT_INT Uinttype; - - -/* maximum size (in bytes) for integral types */ -#define MAXINTSIZE 32 - -/* is 'x' a power of 2? */ -#define isp2(x) ((x) > 0 && ((x) & ((x) - 1)) == 0) - -/* dummy structure to get alignment requirements */ -struct cD { - char c; - double d; -}; - - -#define PADDING (sizeof(struct cD) - sizeof(double)) -#define MAXALIGN (PADDING > sizeof(int) ? PADDING : sizeof(int)) - - -/* endian options */ -#define BIG 0 -#define LITTLE 1 - - -static union { - int dummy; - char endian; -} const native = { 1 }; - - -typedef struct Header { - int endian; - int align; -} Header; - - -static int getnum(const char** fmt, int df) -{ - if (!isdigit(**fmt)) /* no number? */ - return df; /* return default value */ - else { - int a = 0; - do { - a = a * 10 + *((*fmt)++) - '0'; - } - while (isdigit(**fmt)); - return a; - } -} - - -#define defaultoptions(h) ((h)->endian = native.endian, (h)->align = 1) - - - -static size_t optsize(lua_State* L, char opt, const char** fmt) -{ - switch (opt) { - case 'B': case 'b': return sizeof(char); - case 'H': case 'h': return sizeof(short); - case 'L': case 'l': return sizeof(long); - case 'T': return sizeof(size_t); - case 'f': return sizeof(float); - case 'd': return sizeof(double); - case 'x': return 1; - case 'c': return getnum(fmt, 1); - case 'i': case 'I': { - int sz = getnum(fmt, sizeof(int)); - if (sz > MAXINTSIZE) luaL_error(L, "integral size %d is larger than limit of %d", - sz, MAXINTSIZE); - return sz; - } - default: return 0; /* other cases do not need alignment */ - } -} - - -/* -** return number of bytes needed to align an element of size 'size' -** at current position 'len' -*/ -static int gettoalign(size_t len, Header* h, int opt, size_t size) -{ - if (size == 0 || opt == 'c') return 0; - if (size > (size_t)h->align) size = h->align; /* respect max. alignment */ - return (int)(size - (len & (size - 1))) & (size - 1); -} - - -/* -** options to control endianess and alignment -*/ -static void controloptions(lua_State* L, int opt, const char** fmt, - Header* h) -{ - switch (opt) { - case ' ': return; /* ignore white spaces */ - case '>': h->endian = BIG; return; - case '<': h->endian = LITTLE; return; - case '!': { - int a = getnum(fmt, MAXALIGN); - if (!isp2(a)) luaL_error(L, "alignment %d is not a power of 2", a); - h->align = a; - return; - } - default: { - const char* msg = lua_pushfstring(L, "invalid format option '%c'", opt); - luaL_argerror(L, 1, msg); - } - } -} - - -static void putinteger(lua_State* L, luaL_Buffer* b, int arg, int endian, - int size) -{ - lua_Number n = luaL_checknumber(L, arg); - Uinttype value; - char buff[MAXINTSIZE]; - if (n < 0) value = (Uinttype)(Inttype)n; - else value = (Uinttype)n; - if (endian == LITTLE) { - int i; - for (i = 0; i < size; i++) { - buff[i] = (value & 0xff); - value >>= 8; - } - } else { - int i; - for (i = size - 1; i >= 0; i--) { - buff[i] = (value & 0xff); - value >>= 8; - } - } - luaL_addlstring(b, buff, size); -} - - -static void correctbytes(char* b, int size, int endian) -{ - if (endian != native.endian) { - int i = 0; - while (i < --size) { - char temp = b[i]; - b[i++] = b[size]; - b[size] = temp; - } - } -} - - -static int b_pack(lua_State* L) -{ - luaL_Buffer b; - const char* fmt = luaL_checkstring(L, 1); - Header h; - int arg = 2; - size_t totalsize = 0; - defaultoptions(&h); - lua_pushnil(L); /* mark to separate arguments from string buffer */ - luaL_buffinit(L, &b); - while (*fmt != '\0') { - int opt = *fmt++; - size_t size = optsize(L, opt, &fmt); - int toalign = gettoalign(totalsize, &h, opt, size); - totalsize += toalign; - while (toalign-- > 0) luaL_addchar(&b, '\0'); - switch (opt) { - case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ - putinteger(L, &b, arg++, h.endian, (int)size); - break; - } - case 'x': { - luaL_addchar(&b, '\0'); - break; - } - case 'f': { - float f = (float)luaL_checknumber(L, arg++); - correctbytes((char*)&f, (int)size, h.endian); - luaL_addlstring(&b, (char*)&f, size); - break; - } - case 'd': { - double d = luaL_checknumber(L, arg++); - correctbytes((char*)&d, (int)size, h.endian); - luaL_addlstring(&b, (char*)&d, size); - break; - } - case 'c': case 's': { - size_t l; - const char* s = luaL_checklstring(L, arg++, &l); - if (size == 0) size = l; - luaL_argcheck(L, l >= (size_t)size, arg, "string too short"); - luaL_addlstring(&b, s, size); - if (opt == 's') { - luaL_addchar(&b, '\0'); /* add zero at the end */ - size++; - } - break; - } - default: controloptions(L, opt, &fmt, &h); - } - totalsize += size; - } - luaL_pushresult(&b); - return 1; -} - - -static lua_Number getinteger(const char* buff, int endian, - int issigned, int size) -{ - Uinttype l = 0; - int i; - if (endian == BIG) { - for (i = 0; i < size; i++) { - l <<= 8; - l |= (Uinttype)(unsigned char)buff[i]; - } - } else { - for (i = size - 1; i >= 0; i--) { - l <<= 8; - l |= (Uinttype)(unsigned char)buff[i]; - } - } - if (!issigned) return (lua_Number)l; - else { /* signed format */ - Uinttype mask = (Uinttype)(~((Uinttype)0)) << (size * 8 - 1); - if (l & mask) /* negative value? */ - l |= mask; /* signal extension */ - return (lua_Number)(Inttype)l; - } -} - - -static int b_unpack(lua_State* L) -{ - Header h; - const char* fmt = luaL_checkstring(L, 1); - size_t ld; - const char* data = luaL_checklstring(L, 2, &ld); - size_t pos = luaL_optinteger(L, 3, 1) - 1; - defaultoptions(&h); - lua_settop(L, 2); - while (*fmt) { - int opt = *fmt++; - size_t size = optsize(L, opt, &fmt); - pos += gettoalign(pos, &h, opt, size); - luaL_argcheck(L, pos + size <= ld, 2, "data string too short"); - luaL_checkstack(L, 1, "too many results"); - switch (opt) { - case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ - int issigned = islower(opt); - lua_Number res = getinteger(data + pos, h.endian, issigned, (int)size); - lua_pushnumber(L, res); - break; - } - case 'x': { - break; - } - case 'f': { - float f; - memcpy(&f, data + pos, size); - correctbytes((char*)&f, sizeof(f), h.endian); - lua_pushnumber(L, f); - break; - } - case 'd': { - double d; - memcpy(&d, data + pos, size); - correctbytes((char*)&d, sizeof(d), h.endian); - lua_pushnumber(L, d); - break; - } - case 'c': { - if (size == 0) { - if (!lua_isnumber(L, -1)) luaL_error(L, "format `c0' needs a previous size"); - size = (unsigned)lua_tonumber(L, -1); - lua_pop(L, 1); - luaL_argcheck(L, pos + size <= ld, 2, "data string too short"); - } - lua_pushlstring(L, data + pos, size); - break; - } - case 's': { - const char* e = (const char*)memchr(data + pos, '\0', ld - pos); - if (e == NULL) luaL_error(L, "unfinished string in data"); - size = (e - (data + pos)) + 1; - lua_pushlstring(L, data + pos, size - 1); - break; - } - default: controloptions(L, opt, &fmt, &h); - } - pos += size; - } - lua_pushinteger(L, pos + 1); - return lua_gettop(L) - 2; -} - - -static int b_size(lua_State* L) -{ - Header h; - const char* fmt = luaL_checkstring(L, 1); - size_t pos = 0; - defaultoptions(&h); - while (*fmt) { - int opt = *fmt++; - size_t size = optsize(L, opt, &fmt); - pos += gettoalign(pos, &h, opt, size); - if (opt == 's') luaL_argerror(L, 1, "option 's' has no fixed size"); - else if (opt == 'c' && size == 0) luaL_argerror(L, 1, "option 'c0' has no fixed size"); - if (!isalnum(opt)) controloptions(L, opt, &fmt, &h); - pos += size; - } - lua_pushinteger(L, pos); - return 1; -} - -/* }====================================================== */ - - - -static const struct luaL_Reg thislib[] = { - { "pack", b_pack }, - { "unpack", b_unpack }, - { "size", b_size }, - { NULL, NULL } -}; - - -LUALIB_API int luaopen_struct(lua_State* L); - -LUALIB_API int luaopen_struct(lua_State* L) -{ - luaL_register(L, "struct", thislib); - return 1; -} - - -/****************************************************************************** -* Copyright (C) 2010-2012 Lua.org, PUC-Rio. All rights reserved. -* -* Permission is hereby granted, free of charge, to any person obtaining -* a copy of this software and associated documentation files (the -* "Software"), to deal in the Software without restriction, including -* without limitation the rights to use, copy, modify, merge, publish, -* distribute, sublicense, and/or sell copies of the Software, and to -* permit persons to whom the Software is furnished to do so, subject to -* the following conditions: -* -* The above copyright notice and this permission notice shall be -* included in all copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, -* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -******************************************************************************/ - diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 77ccf5e..439ed9e 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -2,9 +2,21 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -add_executable(test_lua_sandbox test_lua_sandbox.c) -target_link_libraries(test_lua_sandbox luasandbox) +add_library(luasandboxtest SHARED sandbox.c) +set_target_properties(luasandboxtest PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_compile_definitions(luasandboxtest PRIVATE -Dluasandboxtest_EXPORTS) +target_link_libraries(luasandboxtest luasandbox) +install(TARGETS luasandboxtest DESTINATION ${CMAKE_INSTALL_LIBDIR}) -add_test(NAME test_sandbox WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} COMMAND test_lua_sandbox) +add_executable(test_generic_sandbox test_generic_sandbox.c) +target_link_libraries(test_generic_sandbox luasandboxtest) +set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src;${CMAKE_BINARY_DIR}/src/util;${CMAKE_BINARY_DIR}/src/test") + +add_test(NAME test_move_luasandbox_tests COMMAND cmake -E copy_directory ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +add_test(NAME test_generic_sandbox COMMAND test_generic_sandbox) + +if(WIN32) + STRING(REPLACE ";" "\\\\;" LIBRARY_PATHS "${LIBRARY_PATHS}") + set_tests_properties(test_generic_sandbox PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) +endif() -install(TARGETS test_lua_sandbox DESTINATION lib) diff --git a/src/test/lua/bloom_filter.lua b/src/test/lua/bloom_filter.lua deleted file mode 100644 index 46648a1..0000000 --- a/src/test/lua/bloom_filter.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "bloom_filter" - -bf = bloom_filter.new(20, 0.01) -count = 0 - -function process(ts) - if not bf:query(ts) then - if not bf:add(ts) then - error("key existed") - end - - count = count + 1 - end - - return 0 -end - -function report(tc) - if tc == 99 then - bf:clear() - count = 0 - else - write_output(count) - end -end - diff --git a/src/test/lua/bloom_filter_benchmark.lua b/src/test/lua/bloom_filter_benchmark.lua deleted file mode 100644 index ea05a07..0000000 --- a/src/test/lua/bloom_filter_benchmark.lua +++ /dev/null @@ -1,21 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "bloom_filter" - -bf = bloom_filter.new(6e6, 0.01) -count = 0 - -function process(ts) - if bf:add(ts) then - count = count + 1 - end - - return 0 -end - -function report(tc) - write_output(count) -end - diff --git a/src/test/lua/bloom_filter_errors.lua b/src/test/lua/bloom_filter_errors.lua deleted file mode 100644 index 0ff7936..0000000 --- a/src/test/lua/bloom_filter_errors.lua +++ /dev/null @@ -1,43 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "bloom_filter" - -function process(tc) - if tc == 0 then - local bf = bloom_filter.new(2) -- new() incorrect # args - elseif tc == 1 then - local bf = bloom_filter.new(nil, 0.01) -- new() non numeric item - elseif tc == 2 then - local bf = bloom_filter.new(0, 0.01) -- invalid items - elseif tc == 3 then - local bf = bloom_filter.new(2, nil) -- nil probability - elseif tc == 4 then - local bf = bloom_filter.new(2, 0) -- invalid probability - elseif tc == 5 then - local bf = bloom_filter.new(2, 1) -- invalid probability - elseif tc == 6 then - local bf = bloom_filter.new(20, 0.01) - bf:add() --incorrect # args - elseif tc == 7 then - local bf = bloom_filter.new(20, 0.01) - bf:add({}) --incorrect argument type - elseif tc == 8 then - local bf = bloom_filter.new(20, 0.01) - bf:query() --incorrect # args - elseif tc == 9 then - local bf = bloom_filter.new(20, 0.01) - bf:query({}) --incorrect argument type - elseif tc == 10 then - local bf = bloom_filter.new(20, 0.01) - bf:clear(1) --incorrect # args - elseif tc == 11 then - local bf = bloom_filter.new(20, 0.01) - bf:fromstring({}) --incorrect argument type - elseif tc == 12 then - local bf = bloom_filter.new(20, 0.01) - bf:fromstring(" ") --incorrect argument length - end -return 0 -end diff --git a/src/test/lua/circular_buffer.lua b/src/test/lua/circular_buffer.lua deleted file mode 100644 index 238759e..0000000 --- a/src/test/lua/circular_buffer.lua +++ /dev/null @@ -1,253 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "string" -require "math" -require "circular_buffer" - -data = circular_buffer.new(3, 3, 1) -local ADD_COL = data:set_header(1, "Add column") -local SET_COL = data:set_header(2, "Set column", "count") -local GET_COL = data:set_header(3, "Get column", "count", "sum") - - -function process(ts) - if data:add(ts, ADD_COL, 1) then - data:set(ts, GET_COL, data:get(ts, ADD_COL)) - end - data:set(ts, SET_COL, 1) - return 0 -end - -function report(tc) - if tc == 0 then - write_output(data) - elseif tc == 1 then - cbufs = {} - for i=1,3,1 do - cbufs[i] = circular_buffer.new(2,1,1) - cbufs[i]:set_header(1, "Header_1", "count") - end - elseif tc == 2 then - write_output(cbufs[1]) - elseif tc == 3 then - local stats = circular_buffer.new(5, 1, 1) - stats:set(1e9, 1, 1) - stats:set(2e9, 1, 2) - stats:set(3e9, 1, 3) - stats:set(4e9, 1, 4) - local t, c = stats:compute("sum", 1) - if 10 ~= t then - error(string.format("no range sum = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("avg", 1) - if 2.5 ~= t then - error(string.format("no range avg = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("variance", 1) - if 1.25 ~= t then - error(string.format("no range variance = %G", t)) - end - t, c = stats:compute("sd", 1) - if math.sqrt(1.25) ~= t then - error(string.format("no range sd = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("min", 1) - if 1 ~= t then - error(string.format("no range min = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - t, c = stats:compute("max", 1) - if 4 ~= t then - error(string.format("no range max = %G", t)) - end - if 4 ~= c then - error(sting.format("active_rows = %d", c)) - end - - t = stats:compute("sum", 1, 3e9, 4e9) - if 7 ~= t then - error(string.format("range 3-4 sum = %G", t)) - end - t = stats:compute("avg", 1, 3e9, 4e9) - if 3.5 ~= t then - error(string.format("range 3-4 avg = %G", t)) - end - t = stats:compute("sd", 1, 3e9, 4e9) - if math.sqrt(0.25) ~= t then - error(string.format("range 3-4 sd = %G", t)) - end - - t = stats:compute("sum", 1, 3e9) - if 7 ~= t then - error(string.format("range 3- = %G", t)) - end - t = stats:compute("sum", 1, 3e9, nil) - if 7 ~= t then - error(string.format("range 3-nil = %G", t)) - end - t = stats:compute("sum", 1, nil, 2e9) - if 3 ~= t then - error(string.format("range nil-2 sum = %G", t)) - end - t = stats:compute("sum", 1, 11e9, 14e9) - if nil ~= t then - error(string.format("out of range = %G", t)) - end - elseif tc == 4 then - local stats = circular_buffer.new(4, 1, 1) - stats:set(1e9, 1, 0/0) - stats:set(2e9, 1, 8) - stats:set(3e9, 1, 8) - local t = stats:compute("avg", 1) - if 8 ~= t then - error(string.format("no range avg = %G", t)) - end - elseif tc == 5 then - local stats = circular_buffer.new(2, 1, 1) - local nan = stats:get(0, 1) - if nan == nan then - error(string.format("initial value is a number %G", nan)) - end - local v = stats:set(0, 1, 1) - if v ~= 1 then - error(string.format("set failed = %G", v)) - end - v = stats:add(0, 1, 0/0) - if v == v then - error(string.format("adding nan returned a number %G", v)) - end - elseif tc == 6 then - local stats = circular_buffer.new(2, 1, 1) - local cbuf_time = stats:current_time() - if cbuf_time ~= 1e9 then - error(string.format("current_time = %G", cbuf_time)) - end - local v = stats:set(0, 1, 1) - if stats:get(0, 1) ~= 1 then - error(string.format("set failed = %G", v)) - end - stats:fromstring("1 1 nan 99") - local nan = stats:get(0, 1) - if nan == nan then - error(string.format("restored value is a number %G", nan)) - end - v = stats:get(1e9, 1) - if v ~= 99 then - error(string.format("restored value is %G", v)) - end - elseif tc == 7 then - local empty = circular_buffer.new(4,1,1) - local nan = empty:compute("avg", 1) - if nan == nan then - error(string.format("avg is a number %G", nan)) - end - nan = empty:compute("sd", 1) - if nan == nan then - error(string.format("std is a number %G", nan)) - end - nan = empty:compute("max", 1) - if nan == nan then - error(string.format("max is a number %G", m)) - end - nan = empty:compute("min", 1) - if nan == nan then - error(string.format("min is a number %G", m)) - end - elseif tc == 8 then - local cb = circular_buffer.new(20,1,1) - local u, p = cb:mannwhitneyu(1, 0e9, 9e9, 10e9, 19e9) - if u or p then - error("all the same values should return nil results") - end - elseif tc == 9 then -- default - local cb = circular_buffer.new(40,1,1) - local data = {15309,14092,13661,13412,14205,15042,14142,13820,14917,13953,14320,14472,15133,13790,14539,14129,14363,14202,13841,13610,13759,14428,14851,13838,13819,14468,14989,15557,14380,13500,14818,14632,13631,14663,14532,14188,14537,14109,13925,15022} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 20e9, 21e9, 40e9) - if u ~= 171 or math.floor(p * 100000) ~= 22037 then - error(string.format("u is %g p is %g", u, p)) - end - elseif tc == 10 then -- no continuity correction - local cb = circular_buffer.new(40,1,1) - local data = {15309,14092,13661,13412,14205,15042,14142,13820,14917,13953,14320,14472,15133,13790,14539,14129,14363,14202,13841,13610,13759,14428,14851,13838,13819,14468,14989,15557,14380,13500,14818,14632,13631,14663,14532,14188,14537,14109,13925,15022} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 20e9, 21e9, 40e9, false) - if u ~= 171 or math.floor(p * 100000) ~= 21638 then - error(string.format("u is %g p is %g", u, p)) - end - elseif tc == 11 then -- tie correction - local cb = circular_buffer.new(40,1,1) - local data = {15309,14092,13661,13412,14205,15042,14142,13820,14917,13953,14320,14472,15133,13790,14539,14129,14363,14202,13841,13610,13759,14428,14851,13838,13819,14468,14989,15557,14380,13500,14818,14632,13631,14663,14532,14188,14537,14109,13925,15309} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 20e9, 21e9, 40e9) - if u ~= 168.5 or math.floor(p * 100000) ~= 20084 then - error(string.format("u is %g p is %g", u, p)) - end - elseif tc == 12 then - local cb = circular_buffer.new(40,1,1) - local u, p = cb:mannwhitneyu(1, 41e9, 60e9, 61e9, 80e9) - if u or p then - error("times outside of buffer should return nil results") - end - elseif tc == 13 then - local cb = circular_buffer.new(10,1,1) - local data = {1,1,1,1,1,1,1,1,1,1} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u, p = cb:mannwhitneyu(1, 1e9, 5e9, 6e9, 10e9) - if u or p then - error("all the same values should return nil results") - end - elseif tc == 14 then - local cb = circular_buffer.new(10,1,1) - local rows, cols, spr = cb:get_configuration() - assert(rows == 10, "invalid rows") - assert(cols == 1 , "invalid columns") - assert(spr == 1 , "invalid seconds_per_row") - elseif tc == 15 then - local cb = circular_buffer.new(10,1,1) - local args = {"widget", "count", "max"} - local col = cb:set_header(1, args[1], args[2], args[3]) - assert(col == 1, "invalid column") - local n, u, m = cb:get_header(col) - assert(n == args[1], "invalid name") - assert(u == args[2], "invalid unit") - assert(m == args[3], "invalid aggregation_method") - elseif tc == 16 then - local cb = circular_buffer.new(10,1,1) - assert(not cb:get(10*1e9, 1), "value found beyond the end of the buffer") - cb:set(20*1e9, 1, 1) - assert(not cb:get(10*1e9, 1), "value found beyond the start of the buffer") - elseif tc == 17 then -- default - local cb = circular_buffer.new(120,1,1) - local data = {1,1,1,2,1,3,3,6,4,0/0,0/0,0/0,1,0/0,2,0/0,0/0,0/0,0/0,0/0,1,5,1,0/0,1,1,0/0,0/0,3,4,1,1,1,0/0,7,1,0/0,6,0/0,0/0,1,3,4,3,0/0,1,5,0/0,1,0/0,0/0,1,6,4,0/0,4,2,6,4,3,2,6,2,11,2,0/0,2,0/0,2,0/0,0/0,0/0,4,0/0,3,2,0/0,0/0,1,2,2,2,1,1,0/0,3,0/0,4,0/0,0/0,2,3,5,6,3,1,0/0,0/0,3,2,0/0,4,1,2,1,1,0/0,0/0,0/0,0/0,0/0,0/0,0/0,7,1,1,2,1,0/0,0/0} - for i,v in ipairs(data) do - cb:set(i*1e9, 1, v) - end - local u1 = cb:mannwhitneyu(1, 61e9, 120e9, 1e9, 60e9) - local u2 = cb:mannwhitneyu(1, 1e9, 60e9, 61e9, 120e9) - if u1 + u2 ~= 3600 then - error(string.format("u1 is %g u2 is %g %g", u1, u2, maxu)) - end - end -end diff --git a/src/test/lua/circular_buffer_add.lua b/src/test/lua/circular_buffer_add.lua deleted file mode 100644 index 14def70..0000000 --- a/src/test/lua/circular_buffer_add.lua +++ /dev/null @@ -1,12 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "circular_buffer" - -data = circular_buffer.new(1440, 10, 1) - -function process(ts) - data:add(ts, 4, 1) - return 0 -end diff --git a/src/test/lua/circular_buffer_delta.lua b/src/test/lua/circular_buffer_delta.lua deleted file mode 100644 index fb6a2ad..0000000 --- a/src/test/lua/circular_buffer_delta.lua +++ /dev/null @@ -1,78 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "circular_buffer" - -data = circular_buffer.new(3, 3, 1, true) -local ADD_COL = data:set_header(1, "Add column") -local SET_COL = data:set_header(2, "Set column", "count") -local GET_COL = data:set_header(3, "Get column", "count", "sum") - -local cb = circular_buffer.new(2, 2, 1, true) -local SUM_COL = cb:set_header(1, "Sum column") -local MIN_COL = cb:set_header(2, "Min", "count", "min") - -function process(ts) - if data:add(ts, ADD_COL, 1) then - data:set(ts, GET_COL, data:get(ts, ADD_COL)) - end - data:set(ts, SET_COL, 1) - return 0 -end - -function report(tc) - if tc == 0 then - write_output(data:format("cbuf")) - elseif tc == 1 then - write_output(data:format("cbufd")) - elseif tc == 2 then - local ts = 2e9 - if data:add(ts, ADD_COL, 0/0) then - data:set(ts, GET_COL, data:get(ts, ADD_COL)) - end - write_output(data:format("cbufd")) - elseif tc == 3 then - -- the sum delta should reflect the difference - -- the min delta should reflect the current value - cb:set(0, SUM_COL, 3) - cb:set(0, MIN_COL, 6) - write_output(cb:format("cbufd")) - cb:set(0, SUM_COL, 5) - cb:set(0, MIN_COL, 5) - write_output(cb:format("cbufd")) - elseif tc == 4 then - cb:add(0, SUM_COL, 3) - cb:add(0, MIN_COL, 3) - write_output(cb:format("cbufd")) - elseif tc == 5 then - cb:add(0, SUM_COL, 1) - cb:set(0, SUM_COL, 0/0) - cb:add(0, MIN_COL, 1) - write_output(cb:format("cbufd")) - elseif tc == 6 then - cb:add(0, SUM_COL, 1) - cb:add(0, SUM_COL, 0/0) - cb:add(0, MIN_COL, 1) - write_output(cb:format("cbufd")) - elseif tc == 7 then - cb:add(0, SUM_COL, 1) - cb:set(0, MIN_COL, 0/0) - write_output(cb:format("cbufd")) - elseif tc == 8 then - cb:add(0, SUM_COL, 1) - cb:set(0, MIN_COL, 9) - cb:add(0, MIN_COL, 0/0) - write_output(cb:format("cbufd")) - elseif tc == 9 then - cb:set(0, SUM_COL, 0) - cb:set(0, MIN_COL, 0) - write_output(cb:format("cbufd")) - elseif tc == 10 then - cb:add(0, SUM_COL, 0) - cb:add(0, MIN_COL, 0) - write_output(cb:format("cbufd")) - elseif tc == 11 then - cb:fromstring(0, 0, 1, 2, 3, 4, 0, 3, 4) - end -end diff --git a/src/test/lua/circular_buffer_errors.lua b/src/test/lua/circular_buffer_errors.lua deleted file mode 100644 index a6eddd0..0000000 --- a/src/test/lua/circular_buffer_errors.lua +++ /dev/null @@ -1,116 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "circular_buffer" - -function process(tc) - if tc == 0 then - local cb = circular_buffer.new(2) -- new() incorrect # args - elseif tc == 1 then - local cb = circular_buffer.new(nil, 1, 1) -- new() non numeric row - elseif tc == 2 then - local cb = circular_buffer.new(1, 1, 1) -- new() 1 row - elseif tc == 3 then - local cb = circular_buffer.new(2, nil, 1) -- new() non numeric column - elseif tc == 4 then - local cb = circular_buffer.new(2, 0, 1) -- new() zero column - elseif tc == 5 then - local cb = circular_buffer.new(2, 1, nil) -- new() non numeric seconds_per_row - elseif tc == 6 then - local cb = circular_buffer.new(2, 1, 0) -- new() zero seconds_per_row - elseif tc == 7 then - error("removed test") - elseif tc == 8 then - local cb = circular_buffer.new(1000, 10, 1) -- new() too much memory - elseif tc == 9 then - local cb = circular_buffer.new(2, 1, 1) -- set() out of range column - cb:set(0, 2, 1.0) - elseif tc == 10 then - local cb = circular_buffer.new(2, 1, 1) -- set() zero column - cb:set(0, 0, 1.0) - elseif tc == 11 then - local cb = circular_buffer.new(2, 1, 1) -- set() non numeric column - cb:set(0, nil, 1.0) - elseif tc == 12 then - local cb = circular_buffer.new(2, 1, 1) -- set() non numeric time - cb:set(nil, 1, 1.0) - elseif tc == 13 then - local cb = circular_buffer.new(2, 1, 1) -- get() invalid object - local invalid = 1 - cb.get(invalid, 1, 1) - elseif tc == 14 then - local cb = circular_buffer.new(2, 1, 1) -- set() non numeric value - cb:set(0, 1, nil) - elseif tc == 15 then - local cb = circular_buffer.new(2, 1, 1) -- set() incorrect # args - cb:set(0) - elseif tc == 16 then - local cb = circular_buffer.new(2, 1, 1) -- add() incorrect # args - cb:add(0) - elseif tc == 17 then - local cb = circular_buffer.new(2, 1, 1) -- get() incorrect # args - cb:get(0) - elseif tc == 18 then - local cb = circular_buffer.new(2, 1, 1) -- compute() incorrect # args - cb:compute(0) - elseif tc == 19 then - local cb = circular_buffer.new(2, 1, 1) -- compute() incorrect function - cb:compute("func", 1) - elseif tc == 20 then - local cb = circular_buffer.new(2, 1, 1) -- compute() incorrect column - cb:compute("sum", 0) - elseif tc == 21 then - local cb = circular_buffer.new(2, 1, 1) -- compute() start > end - cb:compute("sum", 1, 2e9, 1e9) - elseif tc == 22 then - local cb = circular_buffer.new(2, 1, 1) -- format() invalid - cb:format("invalid") - elseif tc == 23 then - local cb = circular_buffer.new(2, 1, 1) -- format() extra - cb:format("cbuf", true) - elseif tc == 24 then - local cb = circular_buffer.new(2, 1, 1) -- format() missing - cb:format() - elseif tc == 25 then - local cb = circular_buffer.new(2, 1, 1) -- too few - cb:fromstring("") - elseif tc == 26 then - local cb = circular_buffer.new(2, 1, 1) -- too few invalid - cb:fromstring("0 0 na 1") - elseif tc == 27 then - local cb = circular_buffer.new(2, 1, 1) -- too many - cb:fromstring("0 0 1 2 3") - elseif tc == 28 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu() -- incorrect # args - elseif tc == 29 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(nil, 0, 0, 0, 0) -- non numeric column - elseif tc == 30 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(0, 0, 0, 0, 0) -- invalid column - elseif tc == 31 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 0, 5, 2, 7) -- overlapping x,y - elseif tc == 32 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 5, 0, 2, 7) -- inverted x - elseif tc == 33 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 0, 5, 10, 6) -- inverted y - elseif tc == 34 then - local cb = circular_buffer.new(2, 1, 1) - cb:mannwhitneyu(1, 0, 1, 2, 3, true, 5) -- incorrect # args - elseif tc == 35 then - local cb = circular_buffer.new(10, 1, 1) - cb:mannwhitneyu(1, 0, 5, 6, 10, "a") -- invalid use_continuity flag - elseif tc == 36 then - local cb = circular_buffer.new(10, 1, 1) - cb:get_header() -- incorrect # args - elseif tc == 37 then - local cb = circular_buffer.new(10, 1, 1) - cb:get_header(99) -- out of range column - end -return 0 -end diff --git a/src/test/lua/cjson.lua b/src/test/lua/cjson.lua deleted file mode 100644 index a5232e7..0000000 --- a/src/test/lua/cjson.lua +++ /dev/null @@ -1,31 +0,0 @@ -require "string" -require "table" -local cj = require "cjson" -local js = '["this is a test","this is a test","this is a test","this is a test","this is a test"]' - -function process() - assert(cj == cjson, "cjson not creating a global table") - - local value = cjson.decode("[ true, { \"foo\": \"bar\" } ]") - assert("bar" == value[2].foo, string.format("bar: %s", tostring(value[2].foo))) - - local null_json = '{"test" : 1, "null" : null}' - local value = cjson.decode(null_json) - assert(value.null == nil, "null not discarded") - - cjson.decode_null(true) - value = cjson.decode(null_json) - assert(type(value.null) == "userdata", "null discarded") - - local t = cjson.decode(js) - assert(#t == 5, "could not decode a JSON string bigger than the output buffer") - - local ok, json = pcall(cjson.encode, t) - assert(not ok, "could encode an array bigger than the the output buffer") - - assert(not cjson.new) - - assert(not cjson.encode_keep_buffer) - - return 0 -end diff --git a/src/test/lua/cjson_unlimited.lua b/src/test/lua/cjson_unlimited.lua deleted file mode 100644 index 08750cf..0000000 --- a/src/test/lua/cjson_unlimited.lua +++ /dev/null @@ -1,13 +0,0 @@ -require "cjson" - -function process() - local ls = "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" - local t = {} - - for i=1, 1000 do - t[#t+1] = ls - end - write_output(cjson.encode(t)) - - return 0 -end diff --git a/src/test/lua/decoder.lua b/src/test/lua/decoder.lua deleted file mode 100644 index 53be314..0000000 --- a/src/test/lua/decoder.lua +++ /dev/null @@ -1,42 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require("lpeg") -local l = lpeg -l.locale(l) - -local space = l.space^1 - -local timestamp = l.Cg((l.R"09"^1 / "%0000000000"), "Timestamp") - -local severity = l.Cg( - l.Cs(l.P"debug" /"7") - + l.Cs(l.P"info" /"6") - + l.Cs(l.P"notice" /"5") - + l.Cs((l.P"warning" + "warn")/"4") - + l.Cs((l.P"error" + "err") /"3") - + l.Cs(l.P"crit" /"2") - + l.Cs(l.P"alert" /"1") - + l.Cs((l.P"emerg" + "panic") /"0") - , "Severity") - -local key = l.C(l.alpha^1) - -local value = l.C(l.R"!~"^1) - -local pair = space * l.Cg(key * "=" * value) - -local fields = l.Cg(l.Cf(l.Ct("") * pair^0, rawset), "Fields") - -local grammar = l.Ct(timestamp * space * severity * fields) - -function process () - local t = grammar:match("1376389920 debug id=2321 url=example.com item=1") - if t then - write_message(t) - else - return -1 - end - return 0 -end diff --git a/src/test/lua/errors.lua b/src/test/lua/errors.lua index ab1643e..badaa0b 100644 --- a/src/test/lua/errors.lua +++ b/src/test/lua/errors.lua @@ -33,11 +33,7 @@ function process(tc) elseif tc == 10 then local v = require "pathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflow" elseif tc == 11 then - package = nil - require "os" - elseif tc == 12 then - package.loaded = nil - require "os" + local v = require "foo.bar" end return 0 end diff --git a/src/test/lua/hyperloglog.lua b/src/test/lua/hyperloglog.lua deleted file mode 100644 index 9aae7cd..0000000 --- a/src/test/lua/hyperloglog.lua +++ /dev/null @@ -1,21 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "hyperloglog" - -hll = hyperloglog.new() - -function process(ts) - hll:add(ts) - return 0 -end - -function report(tc) - if tc == 99 then - hll:clear() - else - write_output(hll:count()) - end -end - diff --git a/src/test/lua/hyperloglog_errors.lua b/src/test/lua/hyperloglog_errors.lua deleted file mode 100644 index d7f9db0..0000000 --- a/src/test/lua/hyperloglog_errors.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "hyperloglog" - -function process(tc) - if tc == 0 then - local hll = hyperloglog.new(2) -- new() incorrect # args - elseif tc == 1 then - local hll = hyperloglog.new() - hll:add({}) --incorrect argument type - elseif tc == 2 then - local hll = hyperloglog.new() - hll:add() --incorrect # args - elseif tc == 3 then - local hll = hyperloglog.new() - hll:count(1) --incorrect # args - elseif tc == 4 then - local hll = hyperloglog.new() - hll:clear(1) --incorrect # args - elseif tc == 5 then - local hll = hyperloglog.new() - hll:fromstring({}) --incorrect argument type - elseif tc == 6 then - local hll = hyperloglog.new() - hll:fromstring(" ") --incorrect argument length - end -return 0 -end diff --git a/src/test/lua/lpeg.lua b/src/test/lua/lpeg.lua deleted file mode 100644 index 6120e4d..0000000 --- a/src/test/lua/lpeg.lua +++ /dev/null @@ -1,18 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "lpeg" - --- csv grammar -local field = '"' * lpeg.Cs(((lpeg.P(1) - '"') + lpeg.P'""' / '"')^0) * '"' + - lpeg.C((1 - lpeg.S',\n"')^0) -local record = lpeg.Ct(field * (',' * field)^0) * (lpeg.P'\n' + -1) - -function process () - local t = lpeg.match(record, '1,string with spaces,"quoted string, with comma and ""quoted"" text"') - assert(t[1], "1", t[1]) - assert(t[2], "string with spaces", t[2]) - assert(t[3], 'quoted string, with comma and "quoted" text', t[3]) - return 0 -end diff --git a/src/test/lua/lpeg_cbufd.lua b/src/test/lua/lpeg_cbufd.lua deleted file mode 100644 index ea9972c..0000000 --- a/src/test/lua/lpeg_cbufd.lua +++ /dev/null @@ -1,30 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "lpeg" - -local cbufd = require("cbufd") - -function process(tc) - if tc == 0 then - local t = lpeg.match(cbufd.grammar, "header\n1\t2\t3\n2\tnan\t-4\n3\t-4.56\t5.67\n") - if not t then - return error("no match") - end - if t.header ~= "header" then error("header:" .. t.header) end - if t[1].time ~= 1e9 then return error("col 1 timestamp:" .. t[1].time) end - if t[1][1] ~= 2 then return error("col 1 val 1:" .. t[1][1]) end - if t[1][2] ~= 3 then return error("col 1 val 2:" .. t[1][2]) end - - if t[2].time ~= 2e9 then return error("col 2 timestamp:" .. t[2].time) end - if t[2][1] == t[2][1] then error("col 2 val 1:" .. t[2][1]) end - if t[2][2] ~= -4 then return error("col 2 val 2:" .. t[2][2]) end - - if t[3].time ~= 3e9 then error("col 3 timestamp:" .. t[3].time) end - if t[3][1] ~= -4.56 then error("col 3 val 1:" .. t[3][1]) end - if t[3][2] ~= 5.67 then error("col 3 val 2:" .. t[3][2]) end - end - - return 0 -end diff --git a/src/test/lua/lpeg_clf.lua b/src/test/lua/lpeg_clf.lua deleted file mode 100644 index 6488216..0000000 --- a/src/test/lua/lpeg_clf.lua +++ /dev/null @@ -1,496 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local clf = require "common_log_format" -require "string" - -local combined_log = '127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0"' - -local combined_result = { - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - remote_user = "-", - time = 1392050801000000000, - request = "GET / HTTP/1.1", - status = 304, - body_bytes_sent = {value = 0, representation = "B"}, - http_referer = "-", - http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" -} - -local function verify_result(fields, expected) - if not fields then - error("no result", 2) - end - - for k, v in pairs(expected) do - local f = fields[k] - if not f then - error(string.format("key: %s is not found in the result", k), 2) - end - - if type(v) == "table" then - if f.value ~= v.value then - error(string.format("key: %s value: %s received:%s", - k, - tostring(v.value), - tostring(f.value)), - 2) - end - if f.representation ~= v.representation then - error(string.format("key: %s representation: %s received: %s", - k, - tostring(v.representation), - tostring(f.representation)), - 2) - end - else - if f ~= v then - error(string.format("key: %s expected: %s received: %s", - k, - tostring(v), - tostring(f)), - 2) - end - end - end -end - -local function nginx_formats() - local tests = { -- format, input, result - {'"$arg_generic"' ,'"test \\"item\\""' ,'test \\"item\\"'} - ,{"$binary_remote_addr" ,"aaaa" ,"aaaa"} - ,{"$body_bytes_sent" ,"23" ,{23,"B"}} - ,{"$bytes_sent" ,"24" ,{24,"B"}} - ,{"$connection" ,"3" ,3} - ,{"$connection_requests" ,"0" ,0} - ,{"$content_length" ,"123" ,{123,"B"}} - ,{"$host" ,"example.com" ,{"example.com","hostname"}} - ,{"$hostname" ,"::1" ,{"::1","ipv6"}} - ,{"$https" ,"on" ,"on"} - ,{"$https" ,"" ,""} - ,{"$is_args" ,"?" ,"?"} - ,{"$is_args" ,"" ,""} - ,{"$msec" ,"1391794831.755" ,1391794831755000000} - ,{"$nginx_version" ,"1.1.1" ,"1.1.1"} - ,{"$pid" ,"99" ,99} - ,{"$pipe" ,"p" ,"p"} - ,{"$pipe" ,"." ,"."} - ,{"$proxy_protocol_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"$remote_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"$remote_port" ,"5435" ,5435} - ,{"$request_completion" ,"OK" ,"OK"} - ,{"$request_completion" ,"" ,""} - ,{"$request_length" ,"23" ,{23,"B"}} - ,{"$request_method" ,"GET" ,"GET"} - ,{"$request_time" ,"1.123" ,{1.123,"s"}} - ,{"$scheme" ,"http" ,"http"} - ,{"$scheme" ,"https" ,"https"} - ,{"$server_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"$server_name" ,"example.com" ,{"example.com","hostname"}} - ,{"$server_port" ,"5435" ,5435} - ,{"$server_protocol" ,"HTTP/1.0" ,"HTTP/1.0"} - ,{"$status" ,"200" ,200} - ,{"$tcpinfo_rtt" ,"200" ,200} - ,{"$tcpinfo_rttvar" ,"200" ,200} - ,{"$tcpinfo_snd_cwnd" ,"200" ,200} - ,{"$tcpinfo_rcv_space" ,"200" ,200} - ,{"$time_iso8601" ,"2014-02-10T08:46:41-08:00" ,1392050801000000000} - ,{"$time_local" ,"10/Feb/2014:08:46:41 -0800" ,1392050801000000000} - } - - for i, v in ipairs(tests) do - local grammar = clf.build_nginx_grammar(v[1]) - local fields = grammar:match(v[2]) - if not fields then - error(string.format("test: %s failed to match: %s", v[1], v[2])) - end - - local key - if i == 1 then - key = "arg_generic" - else - key = string.sub(v[1], 2) - end - if key == "msec" or key == "time_iso8601" or key == "time_local" then - key = "time" - end - - - if type(v[3]) == "table" then - if fields[key].value ~= v[3][1] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[3][1], fields[key].value)) - end - if fields[key].representation ~= v[3][2] then - error(string.format("test: %s expected representation: %s received: %s", v[1], v[3][2], fields[key].representation)) - end - else - if fields[key] ~= v[3] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[3], fields[key])) - end - end - end -end - -local function nginx_combined() - local grammar = clf.build_nginx_grammar('$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"') - local fields = grammar:match(combined_log) - verify_result(fields, combined_result) -end - -local function nginx_msec() - local grammar = clf.build_nginx_grammar('$msec $remote_addr "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"') - local log = '1391794831.755 127.0.0.1 "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" "-" 0.000' - local fields = grammar:match(log) - local result = { - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - time = 1391794831755000000, - request = "GET / HTTP/1.1", - status = 304, - body_bytes_sent = {value = 0, representation = "B"}, - http_referer = "-", - http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:26.0) Gecko/20100101 Firefox/26.0" - } - verify_result(fields, result) -end - -local function user_agent_normalization() - local user_agents = { - "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36" - ,"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)" - ,"Mozilla/5.0 (Windows NT 6.1; rv:29.0) Gecko/20100101 Firefox/29.0" - ,"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:29.0) Gecko/20100101 Firefox/29.0" - ,"Mozilla/5.0 (Mobile; rv:29.0) Gecko/20100101 Firefox/29.0" - ,"curl/7.29.0" - ,"Apache-HttpClient/UNAVAILABLE (java 1.5)" - ,"Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405" - ,"Googlebot/2.1 (+http://www.google.com/bot.html)" - ,"" - ,"-" - ,"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0; MSIE; )" -- https://bugzilla.mozilla.org/show_bug.cgi?id=1009280 - ,"Firefox/29.0 FxSync/1.31.0.20140327113732.desktop" -- https://github.com/mozilla-services/puppet-config/issues/312 - ,"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25" - ,"Opera/9.80 (J2ME/MIDP; Opera Mini/9.80 (S60; SymbOS; Opera Mobi/23.348; U; en) Presto/2.5.25 Version/10.54" - ,"Opera/12.02 (Android 4.1; Linux; Opera Mobi/ADR-1111101157; U; en-US) Presto/2.9.201 Version/12.02" - ,"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14" - } - local results = { - {"Chrome" , 32 , "Windows 8"} - ,{"MSIE" , 10 , "Windows 7"} - ,{"Firefox" , 29 , "Windows 7"} - ,{"Firefox" , 29 , "Macintosh"} - ,{"Firefox" , 29 , "FirefoxOS"} - ,{nil , nil , nil} - ,{nil , nil , nil} - ,{nil , nil , "iPad"} - ,{nil , nil , nil} - ,{nil , nil , nil} - ,{nil , nil , nil} - ,{"MSIE" , nil , "Windows 7"} - ,{"Firefox" , 29 , nil} - ,{"Safari" , 8536 , "iPad"} - ,{"Opera Mini" , 10 , nil} - ,{"Opera Mobi" , 12 , "Android"} - ,{"Opera" , 12 , "Windows Vista"} - } - - for i, v in ipairs(user_agents) do - local browser, version, os = clf.normalize_user_agent(v) - assert(results[i][1] == browser and results[i][2] == version and results[i][3] == os, - string.format("user_agent: %d browser: %s version: %s os: %s expected browser: %s version: %s os: %s", - i, tostring(browser), tostring(version), tostring(os), - tostring(results[i][1]), tostring(results[i][2]), tostring(results[i][3]))) - end -end - -local function apache_formats() - local tests = { -- format, key, input, result - {"%a", "remote_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"%A", "server_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"%B", "body_bytes_sent" ,"23" ,{23,"B"}} - ,{"%b", "body_bytes_sent" ,"23" ,{23,"B"}} - ,{"%b", "body_bytes_sent" ,"-" ,{0,"B"}} - ,{"%D", "request_time" ,"123" ,{123,"us"}} - ,{"%f", "request_filename" ,"test.txt" ,"test.txt"} - ,{"%h", "remote_addr" ,"example.com" ,{"example.com","hostname"}} - ,{"%H", "server_protocol" ,"HTTP/1.0" ,"HTTP/1.0"} - ,{"%k", "connection_requests" ,"0" ,0} - ,{"%l", "remote_user" ,"waldo" ,"waldo"} - ,{"%L", "request_log_id" ,"???" ,"???"} - ,{"%m", "request_method" ,"GET" ,"GET"} - ,{"%p", "server_port" ,"5435" ,5435} - ,{"%P", "pid" ,"99" ,99} - ,{"%q", "query_string" ,"query" ,"query"} - ,{"%r", "request" ,"request" ,"request"} - ,{"%R", "request_handler" ,"handler" ,"handler"} - ,{"%s", "status" ,"200" ,200} - ,{"%t", "time" ,"[10/Feb/2014:08:46:41 -0800]" ,1392050801000000000} - ,{"%T", "request_time" ,"1" ,{1,"s"}} - ,{"%u", "remote_user" ,"remote" ,"remote"} - ,{"%U", "uri" ,"uri" ,"uri"} - ,{"%v", "server_name" ,"example.com" ,{"example.com","hostname"}} - ,{"%V", "server_name" ,"example.com" ,{"example.com","hostname"}} - ,{'\\"%V\\"', "server_name" ,'"example.com"' ,{"example.com","hostname"}} - ,{"%X", "connection_status" ,"X" ,"X"} - ,{"%X", "connection_status" ,"+" ,"+"} - ,{"%X", "connection_status" ,"-" ,"-"} - ,{"%I", "request_length" ,"23" ,{23,"B"}} - ,{"%O", "response_length" ,"23" ,{23,"B"}} - ,{"%S", "total_length" ,"23" ,{23,"B"}} - - ,{"%>s" ,"status" ,"200" ,200} - ,{"%{c}a" ,"remote_addr" ,"127.0.0.1" ,{"127.0.0.1","ipv4"}} - ,{"%{Test-item}C" ,"cookie_Test-item" ,"item" ,"item"} - ,{"%{Test-item}e" ,"env_Test-item" ,"item" ,"item"} - ,{'"%{User-agent}i"' ,"http_user_agent" ,'"Mozilla/5.0 (comment)"' ,"Mozilla/5.0 (comment)"} - ,{'"%400,501{User-agent}i"' ,"http_user_agent" ,'"Mozilla/5.0 (comment)"' ,"Mozilla/5.0 (comment)"} - ,{'"%!400,501{User-agent}i"',"http_user_agent" ,'"Mozilla/5.0 (comment)"' ,"Mozilla/5.0 (comment)"} - ,{"%{Test-item}n" ,"module_Test-item" ,"item" ,"item"} - ,{"%{Test-item}o" ,"sent_http_test_item" ,"item" ,"item"} - ,{"%{canonical}p" ,"server_port" ,"99" ,99} - ,{"%{local}p" ,"server_port" ,"99" ,99} - ,{"%{remote}p" ,"remote_port" ,"99" ,99} - ,{"%{pid}P" ,"pid" ,"99" ,99} - ,{"%{tid}P" ,"tid" ,"99" ,99} - ,{"%{hextid}P" ,"tid" ,"63" ,99} - ,{"%{sec}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{"%{begin:sec}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{"%{end:sec}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{"%{msec}t" ,"time" ,"1392050801000" ,1392050801000000000} - ,{"%{usec}t" ,"time" ,"1392050801000000" ,1392050801000000000} - ,{"%{msec_frac}t" ,"sec_frac" ,"010" ,0.010} - ,{"%{usec_frac}t" ,"sec_frac" ,"010000" ,0.010} - ,{"%{%d/%b/%Y:%H:%M:%S %z}t","time" ,"10/Feb/2014:08:46:41 -0800" ,1392050801000000000} - ,{"%{%s}t" ,"time" ,"1392050801" ,1392050801000000000} - ,{'"%r"' , "request" ,'"test \\"item\\""' ,'test \\"item\\"'} - ,{'"%{Test-item}o"' ,"sent_http_test_item" ,'"test \\"item\\""' ,'test \\"item\\"'} - } - - if os.date("%c"):find("^%d") then -- windows - tests[55][3] = "10/Feb/2014:08:46:41 PST" - end - - for i, v in ipairs(tests) do - local grammar = clf.build_apache_grammar(v[1]) - local fields = grammar:match(v[3]) - if not fields then - error(string.format("test: %s failed to match: %s", v[1], v[3])) - end - local key = v[2] - - if type(v[4]) == "table" then - if fields[key].value ~= v[4][1] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[4][1], fields[key].value)) - end - if fields[key].representation ~= v[4][2] then - error(string.format("test: %s expected representation: %s received: %s", v[1], v[4][2], fields[key].representation)) - end - else - if fields[key] ~= v[4] then - error(string.format("test: %s expected value: %s received: %s", v[1], v[4], fields[key])) - end - end - end -end - -local function apache_custom() - -- test each of the unique grammars - local grammar = clf.build_apache_grammar('%% %a %B %b %D %f %k %p %P %s %t %T %v %X %I %O %S %A') - local log = '% 127.0.0.1 235 235 204 test.txt 2 80 1234 404 [20/Mar/2014:08:56:26 -0700] 0 example.com + 311 498 809 ::1' - local fields = grammar:match(log) - local result = { - server_addr = {value = "::1", representation = "ipv6"}, - pid = 1234, - total_length = {value = 809, representation = "B"}, - server_port = 80, - request_length = {value = 311, representation = "B"}, - connection_requests = 2, - connection_status = "+", - body_bytes_sent = {value = 235, representation = "B"}, - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - time = 1395330986000000000, - response_length = {value = 498, representation = "B"}, - request_time = {value = 0, representation = "s"}, - request_filename = "test.txt", - status = 404, - server_name = {value = "example.com", representation = "hostname"} - } - verify_result(fields, result) -end - -local function apache_clf() - local grammar = clf.build_apache_grammar('%h %l %u %t \"%r\" %>s %b') - local log = '127.0.0.1 - - [10/Feb/2014:08:46:41 -0800] "GET / HTTP/1.1" 304 0' - local fields = grammar:match(log) - local result = { - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - remote_user = "-", - body_bytes_sent = {value = 0, representation = "B"}, - time = 1392050801000000000, - request = "GET / HTTP/1.1", - status = 304, - } - verify_result(fields, result) -end - -local function apache_vhost_combined() - local grammar = clf.build_apache_grammar('%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"') - local log = '127.0.1.1:80 127.0.0.1 - - [20/Mar/2014:12:38:34 -0700] "GET / HTTP/1.1" 404 492 "-" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0"' - local fields = grammar:match(log) - local result = { - remote_user = "-", - server_port = 80, - http_referer = "-", - remote_addr = {value = "127.0.0.1", representation = "ipv4"}, - time = 1395344314000000000, - response_length = {value = 492, representation = "B"}, - request = "GET / HTTP/1.1", - http_user_agent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:27.0) Gecko/20100101 Firefox/27.0", - status = 404, - server_name = {value = "127.0.1.1", representation = "ipv4"} - } - verify_result(fields, result) -end - -local function apache_combined() - local grammar = clf.build_apache_grammar('%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"') - local fields = grammar:match(combined_log) - verify_result(fields, combined_result) -end - -local function apache_referer() - local grammar = clf.build_apache_grammar('%{Referer}i -> %U') - local log = '- -> /' - local fields = grammar:match(log) - local result = { - uri = "/", - http_referer = "-" - } - verify_result(fields, result) -end - -local function nginx_error() - local log = '2014/03/01 11:29:39 [notice] 16842#0: using inherited sockets from "6;"' - local fields = clf.nginx_error_grammar:match(log) - local result = { - Pid = 16842, - Payload = 'using inherited sockets from "6;"', - Severity = 5, - time = 1393673379000000000 - } - verify_result(fields, result) - assert(fields.Fields.tid == 0, "expected a thread id of 0") -end - -local function nginx_error_connection() - -- optional connection - local log = '2014/03/01 11:29:39 [notice] 16842#0: 8878 using inherited sockets from "6;"' - local fields = clf.nginx_error_grammar:match(log) - local result = { - Pid = 16842, - Payload = 'using inherited sockets from "6;"', - Severity = 5, - time = 1393673379000000000 - } - verify_result(fields, result) - assert(fields.Fields.tid == 0, "invalid tid") - assert(fields.Fields.connection == 8878, "invalid connection") -end - -local function nginx_upstream() - local addrs = {"192.168.1.1:80", "192.168.1.2:80", "unix:/tmp/sock", "192.168.10.1:80", "192.168.10.2:80"} - local lengths = {1, 2, 3, 4, 5} - local times = {1.1, 1.2, 1.3, 1.4, 1.5} - local statuses = {200, 201, 202, 203, 204} - local uscs = "HIT" - local header = "header field" - - local grammar = clf.build_nginx_grammar('$upstream_addr $upstream_cache_status $upstream_response_length $upstream_response_time $upstream_status "$upstream_http_test" $upstream_cache_last_modified') - local log = string.format('%s, %s, %s : %s, %s %s %d, %d, %d : %d, %d %g, %g, %g : %g, %g %d, %d, %d : %d, %d "%s" Mon, 28 Sep 1970 06:00:00 GMT', - addrs[1], addrs[2], addrs[3], addrs[4], addrs[5], uscs, - lengths[1], lengths[2], lengths[3], lengths[4], lengths[5], - times[1], times[2], times[3], times[4], times[5], - statuses[1], statuses[2], statuses[3], statuses[4], statuses[5], header) - local fields = grammar:match(log) - if not fields then error(string.format("failed match: %s", log)) end - - if #fields.upstream_addr ~= #addrs then - error(string.format("#upstream_addr = %d", #fields.upstream_addr)) - end - for i, v in ipairs(addrs) do - if fields.upstream_addr[i] ~= v then - error(string.format("expected value: '%s' received: '%s'", v, fields.upstream_addr[i])) - end - end - - if #fields.upstream_response_length.value ~= #lengths then - error(string.format("#upstream_response_length = %d", #fields.upstream_response_length.value)) - end - if fields.upstream_response_length.representation ~= "B" then - error(string.format("upstream_response_length representation = '%s'", fields.upstream_response_length.representation)) - end - for i, v in ipairs(lengths) do - if fields.upstream_response_length.value[i] ~= v then - error(string.format("expected value: %d received: %d", v, fields.upstream_response_length.value[i])) - end - end - - if #fields.upstream_response_time.value ~= #times then - error(string.format("#upstream_response_time = %d", #fields.upstream_response_time.value)) - end - if fields.upstream_response_time.representation ~= "s" then - error(string.format("upstream_response_time representation = '%s'", fields.upstream_response_time.representation)) - end - for i, v in ipairs(times) do - if fields.upstream_response_time.value[i] ~= v then - error(string.format("expected value: %g received: %g", v, fields.upstream_response_time.value[i])) - end - end - - if #fields.upstream_status ~= #statuses then - error(string.format("#upstream_status = %d", #fields.upstream_status)) - end - for i, v in ipairs(statuses) do - if fields.upstream_status[i] ~= v then - error(string.format("expected value: %d received: %d", v, fields.upstream_status[i])) - end - end - - if fields.upstream_cache_status ~= uscs then - error(string.format("expected value: '%s' received: '%s'", uscs, fields.upstream_cache_status)) - end - if fields.upstream_http_test ~= header then - error(string.format("expected value: '%s' received: '%s'", header, fields.upstream_http_test)) - end - local usclm = 2.33496e16 - if fields.upstream_cache_last_modified ~= usclm then - error(string.format("expected value: '%s' received: '%s'", usclm, fields.upstream_cache_last_modified)) - end -end - -local function nginx_upstream_defaults() - local grammar = clf.build_nginx_grammar('$upstream_addr $upstream_cache_status $upstream_response_length $upstream_response_time $upstream_status "$upstream_http_test" $upstream_cache_last_modified') - local log = '- - - - - "-" -' - local fields = grammar:match(log) - if not fields then error(string.format("failed match: %s", log)) end -end - - -function process() - nginx_formats() - nginx_combined() - nginx_msec() - user_agent_normalization() - apache_formats() - apache_custom() - apache_clf() - apache_vhost_combined() - apache_combined() - apache_referer() - nginx_error() - nginx_error_connection() - nginx_upstream() - nginx_upstream_defaults() - - return 0 -end diff --git a/src/test/lua/lpeg_date_time.lua b/src/test/lua/lpeg_date_time.lua deleted file mode 100644 index 4da4b2e..0000000 --- a/src/test/lua/lpeg_date_time.lua +++ /dev/null @@ -1,140 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "string" - -local dt = require("date_time") - -local function test_valid(tc, grammar, tests, results) - for i,v in ipairs(tests) do - t = grammar:match(v) - if not t then - error(v) - else - ns = dt.time_to_ns(t) - if ns ~= results[i] then - error(string.format("tc: %d %s expected: %g received: %g", tc, v, results[i], ns)) - end - end - end -end - -local function rfc3339() - local tests = {"1999-05-05T23:23:59.217-07:00", - "1985-04-12T23:20:50.52Z", - "1996-12-19T16:39:57-08:00", - "1990-12-31T23:59:60Z", - "1990-12-31T15:59:60-08:00", - "1937-01-01T12:00:27.87+00:20"} - local results = {925971839217000064, - 482196050520000000, - 851042397000000000, - 662688000000000000, - 662688000000000000, - -1041337172130000000} - if os.date("%c"):find("^%d") then - results[6] = 0 -- windows will fail to convert this time - end - test_valid(tc, dt.rfc3339, tests, results) -end - -local function rfc3339_invalid() - local tests = {"1985-04-12t23:20:50.52Z", - "1985-04-12T23:20:50.52z", - "1985-04-12", - "1999-05-05T23:23:59.217-0700"} - for i,v in ipairs(tests) do - if dt.rfc3339:match(v) then - error(v) - end - end -end - -local function clf() - local tests = {"10/Feb/2014:08:46:36 -0800"} - local results = {1392050796000000000} - test_valid(tc, dt.clf_timestamp, tests, results) -end - -local function rfc3164() - local tests = {"Feb 10 16:46:36"} - local results = {1392050796000000000} - test_valid(tc, dt.rfc3164_timestamp, tests, results) -end - -local function mysql() - local tests = {"20140210164636"} - local results = {1392050796000000000} - test_valid(tc, dt.mysql_timestamp, tests, results) -end - -local function pgsql() - local tests = {"2014-02-10 16:46:36"} - local results = {1392050796000000000} - test_valid(tc, dt.pgsql_timestamp, tests, results) -end - -local function strftime_all() - local formats - if os.date("%c"):find("^%d") then -- windows C89 support only - formats = {"%a", "%A", "%b", "%B","%c", "%d", - "%H", "%I", "%j", "%m", "%M", "%p", - "%S", "%U", "%w", "%W", "%x", - "%X", "%y", "%Y", "%z", "%Z", "%%", "test string"} - else - formats = {"%a", "%A", "%b", "%B","%c","%C", "%d", "%D", "%e", - "%F", "%g", "%G", "%h", "%H", "%I", "%j", "%k", "%l", "%m", - "%M", "%n", "%p", "%r", "%R", "%s", "%S", "%t", "%T", "%u", - "%U", "%V", "%w", "%W", "%x","%X", "%y", "%Y", "%z", "%Z", - "%%", "test string", "%~"} - end - - for i,v in ipairs(formats) do - local test = os.date(v) - local g = dt.build_strftime_grammar(v) - if not g:match(test) then - error(string.format("failed parsing: %s %s", v, test)) - end - end -end - -local function strftime_composite() - local formats = {"%c","%D %T","%D %r","%d/%b/%Y:%H:%M:%S %z"} - local inputs = {"Mon Feb 10 16:46:36 2014", "02/10/14 16:46:36", "02/10/14 04:46:36 PM", "10/Feb/2014:16:46:36 +0000"} - if os.date("%c"):find("^%d") then -- windows %c is non-standard - inputs[1] = inputs[2] - end - local result = 1392050796000000000 - for i,v in ipairs(formats) do - local g = dt.build_strftime_grammar(v) - local t = g:match(inputs[i]) - if not t then - error(string.format("failed parsing: %s %s", v, inputs[i])) - end - if result ~= dt.time_to_ns(t) then - error(string.format("time conversion failed %s", inputs[i])) - end - end -end - -local function strftime_invalid() - local r, g = pcall(dt.build_strftime_grammar, "%E") - if r then - error("allowed to build a grammar with an invalid specifier") - end -end - -function process(tc) - rfc3339() - rfc3339_invalid() - clf() - rfc3164() - mysql() - pgsql() - strftime_all() - strftime_composite() - strftime_invalid() - - return 0 -end diff --git a/src/test/lua/lpeg_ip_address.lua b/src/test/lua/lpeg_ip_address.lua deleted file mode 100644 index 020b435..0000000 --- a/src/test/lua/lpeg_ip_address.lua +++ /dev/null @@ -1,100 +0,0 @@ - --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "lpeg" - -local ip = require("ip_address") - -local function ipv6() - local tests = {"2001:0db8:85a3:0000:0000:8a2e:0370:7334", - "2001:db8:85a3:0:0:8a2e:370:7334", - "2001:db8:85a3::8a2e:370:7334", - "2001:0db8:0000:0000:0000:0000:1428:57ab", - "2001:0db8:0000:0000:0000::1428:57ab", - "2001:0db8:0:0:0:0:1428:57ab", - "2001:0db8:0:0::1428:57ab", - "2001:0db8::1428:57ab", - "2001:db8::1428:57ab", - "0000:0000:0000:0000:0000:0000:0000:0001", - "::1", - "::ffff:12.34.56.78", - "::ffff:0c22:384e", - "2001:0db8:1234:0000:0000:0000:0000:0000", - "2001:0db8:1234:ffff:ffff:ffff:ffff:ffff", - "2001:db8:a::123", - "fe80::", - "::ffff:192.0.2.128", - "::ffff:c000:280", - "::"} - for i,v in ipairs(tests) do - if not ip.v6:match(v) then - error(v) - end - end -end - -local function ipv6_invalid() - local tests = { - "123", - "ldkfj", - "2001::FFD3::57ab", - "2001:db8:85a3::8a2e:37023:7334", - "2001:db8:85a3::8a2e:370k:7334", - "1:2:3:4:5:6:7:8:9", - "1::2::3", - "1:::3:4:5", - "1:2:3::4:5:6:7:8:9", - "::ffff:2.3.4", - "::ffff:257.1.2.3", - "1.2.3.4"} - local grammar = ip.v6 * lpeg.P(-1) -- make it a full match - for i,v in ipairs(tests) do - if grammar:match(v) then - error(v) - end - end -end - -local function ipv4() - local tests = { - "0.0.0.0", - "192.168.1.10", - "127.0.0.1", - "249.255.255.255", - "255.255.255.255", - "1.2.3.4"} - for i,v in ipairs(tests) do - if not ip.v4:match(v) then - error(v) - end - end -end - -local function ipv4_invalid() - local grammar = ip.v4 * lpeg.P(-1) -- make it a full match - local tests = { - "00.0.0.0", - "256.255.255.255", - "255.2550.255.255", - "1", - "1.2", - "1.2.3.", - "1.2.3.a", - "1.2.3.4.5", - "."} - for i,v in ipairs(tests) do - if grammar:match(v) then - error(v) - end - end -end - -function process(tc) - ipv6() - ipv6_invalid() - ipv4() - ipv4_invalid() - return 0 -end diff --git a/src/test/lua/lpeg_mysql.lua b/src/test/lua/lpeg_mysql.lua deleted file mode 100644 index a3f092f..0000000 --- a/src/test/lua/lpeg_mysql.lua +++ /dev/null @@ -1,173 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -require "string" -local mysql = require("mysql") - -local slow_query_log = { -[[ -# Time: 140507 18:14:18 -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -use widget; -SET last_insert_id=999,insert_id=1000,timestamp=1399500744; -# administrator command: do something -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Query_time: 2.964652 Lock_time: 0.000050 Rows_sent: 251 Rows_examined: 9773 -SET last_insert_id=999,timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]] -} - -local mariadb_slow_query_log = { -[[ -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Thread_id: 110804 Schema: weave0 QC_hit: No -# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Thread_id: 110804 Schema: weave0 QC_hit: No -# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]], -[[ -# Time: 140623 16:43:22 -# User@Host: syncrw[syncrw] @ [127.0.0.1] -# Thread_id: 110804 Schema: weave0 QC_hit: No -# Query_time: 1.178108 Lock_time: 0.000053 Rows_sent: 198 Rows_examined: 198 -# Full_scan: Yes Full_join: Yes Tmp_table: Yes Tmp_table_on_disk: No -# Filesort: Yes Filesort_on_disk: No Merge_passes: 3 -SET timestamp=1399500744; -/* [queryName=FIND_ITEMS] */ SELECT * -FROM widget -WHERE id = 10; -]] -} - -local fields = { - {"Query_time", 2.964652, "s"}, - {"Rows_examined", 9773}, - {"Lock_time", 0.000050, "s"}, - {"Rows_sent", 251}, -} - -local mariadb_fields = { - {"Query_time", 1.178108, "s"}, - {"Rows_examined", 198}, - {"Lock_time", 0.000053, "s"}, - {"Rows_sent", 198}, - {"QC_hit", "No"}, - {"Thread_id", 110804}, - {"Schema", "weave0"} -} - -local mariadb_verbose_fields = { - {"Query_time", 1.178108, "s"}, - {"Rows_examined", 198}, - {"Lock_time", 0.000053, "s"}, - {"Rows_sent", 198}, - {"QC_hit", "No"}, - {"Thread_id", 110804}, - {"Schema", "weave0"}, - {"Full_scan", "Yes"}, - {"Full_join", "Yes"}, - {"Filesort", "Yes"}, - {"Filesort_on_disk", "No"}, - {"Tmp_table_on_disk", "No"}, - {"Tmp_table", "Yes"}, - {"Merge_passes", 3} -} - -local function validate(fields, t) - if t.Timestamp ~= "1399500744000000000" then return error("Timestamp:" .. t.Timestamp) end - if t.Payload ~= "/* [queryName=FIND_ITEMS] */ SELECT *\nFROM widget\nWHERE id = 10;\n" then return error("Payload:" .. t.Payload) end - for i, v in ipairs(fields) do - if #v == 3 then - if t.Fields[v[1]].value ~= v[2] or t.Fields[v[1]].representation ~= v[3] then - if type(v[2]) == "string" then - return error(string.format("field:%s: %s (%s)", v[1], t.Fields[v[1]].value, t.Fields[v[1]].representation), 2) - else - return error(string.format("field:%s: %G (%s)", v[1], t.Fields[v[1]].value, t.Fields[v[1]].representation), 2) - end - end - else - if t.Fields[v[1]] ~= v[2] then return error(string.format("field:%s:", v[1]) .. t.Fields[v[1]], 2) end - end - end -end - -local function header() - local t = mysql.slow_query_grammar:match(slow_query_log[1]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end - validate(fields, t) -end - -local function standard() - local t = mysql.slow_query_grammar:match(slow_query_log[2]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end - validate(fields, t) -end - -local function short() - local t = mysql.short_slow_query_grammar:match(slow_query_log[3]) - if not t then return error("no match") end - if t.Hostname then return error("Hostname:" .. t.Hostname) end - validate(fields, t) -end - -local function mariadb_standard() - local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[1]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end - validate(mariadb_fields, t) -end - -local function mariadb_short() - local t = mysql.mariadb_short_slow_query_grammar:match(mariadb_slow_query_log[2]) - if not t then return error("no match") end - if t.Hostname then return error("Hostname:" .. t.Hostname) end - validate(mariadb_fields, t) -end - -local function mariadb_verbose() - local t = mysql.mariadb_slow_query_grammar:match(mariadb_slow_query_log[3]) - if not t then return error("no match") end - if t.Hostname ~= "127.0.0.1" then return error("Hostname:" .. t.Hostname) end - validate(mariadb_verbose_fields, t) -end - - -function process(tc) - header() - standard() - short() - mariadb_standard() - mariadb_short() - mariadb_verbose() - - return 0 -end diff --git a/src/test/lua/lpeg_syslog.lua b/src/test/lua/lpeg_syslog.lua deleted file mode 100644 index 017aa33..0000000 --- a/src/test/lua/lpeg_syslog.lua +++ /dev/null @@ -1,111 +0,0 @@ --- This Source Code Form is subject to the terms of the Mozilla Public --- License, v. 2.0. If a copy of the MPL was not distributed with this --- file, You can obtain one at http://mozilla.org/MPL/2.0/. - -local syslog = require("syslog") - -local function traditional_file_format() - local grammar = syslog.build_rsyslog_grammar('%TIMESTAMP% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n') - local log = 'Feb 9 14:17:01 trink-x230 CRON[20758]: (root) CMD ( cd / && run-parts --report /etc/cron.hourly)\n' - local fields = grammar:match(log) - assert(fields.msg == "(root) CMD ( cd / && run-parts --report /etc/cron.hourly)", fields.msg) - assert(fields.timestamp == 1391955421000000000, fields.timestamp) - assert(fields.syslogtag.pid == 20758, fields.syslogtag.pid) - assert(fields.syslogtag.programname == "CRON", fields.syslogtag.programname) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function file_format() - local grammar = syslog.build_rsyslog_grammar('%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n') - local log = '2014-02-10T08:21:59.111537-08:00 trink-x230 kernel: Kernel logging (proc) stopped.\n' - local fields = grammar:match(log) - assert(fields.msg == "Kernel logging (proc) stopped.", fields.msg) - assert(fields.timestamp == 1392049319111537000, fields.timestamp) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function forward_format() - local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%') - local log = '<6>2014-02-10T08:33:15.407935-08:00 trink-x230 kernel: imklog 5.8.6, log source = /proc/kmsg started.' - local fields = grammar:match(log) - assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) - assert(fields.timestamp == 1392049995407935000, fields.timestamp) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 0, fields.pri.facility) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function traditional_forward_format() - local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%') - local log = '<6>Feb 10 08:38:47 trink-x230 kernel: imklog 5.8.6, log source = /proc/kmsg started.' - local fields = grammar:match(log) - assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) - assert(fields.timestamp == 1392021527000000000, fields.timestamp) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 0, fields.pri.facility) - assert(fields.hostname == "trink-x230", fields.hostname) -end - -local function kitchen_sink() - local grammar = syslog.build_rsyslog_grammar("%HOSTNAME% %SOURCE% %FROMHOST% %FROMHOST-IP% %SYSLOGTAG% %PROGRAMNAME% %PRI% %PRI-TEXT% %IUT% %SYSLOGFACILITY% %SYSLOGFACILITY-TEXT% %SYSLOGPRIORITY% %SYSLOGPRIORITY-TEXT% %TIMEGENERATED% %TIMEREPORTED:::date-mysql% %TIMESTAMP:::date-rfc3339% %PROTOCOL-VERSION% %STRUCTURED-DATA% %APP-NAME% %PROCID% %MSGID% %$NOW% %$YEAR% %$MONTH% %$DAY% %$HOUR% %$HHOUR% %$QHOUR% %$MINUTE% %MSG%\n") - local log = 'trink-x230 trink-x230 trink-x230 127.0.0.1 kernel: kernel 6 kern.info<6> 1 0 kern 6 info Feb 10 09:20:53 20140210092053 2014-02-10T09:20:53.559934-08:00 0 - kernel - 2014-02-10 2014 02 10 09 00 01 20 imklog 5.8.6, log source = /proc/kmsg started.\n' - local fields = grammar:match(log) - assert(fields.hostname == "trink-x230", fields.hostname) - assert(fields.source == "trink-x230", fields.source) - assert(fields.fromhost == "trink-x230", fields.fromhost) - assert(fields["fromhost-ip"] == "127.0.0.1", fields["fromhost-ip"]) - assert(fields.syslogtag.programname == "kernel", fields.syslogtag.programname) - assert(fields.programname == "kernel", fields.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 0, fields.pri.facility) - assert(fields["pri-text"] == 0, fields["pri-text"]) - assert(fields.iut == "1", fields.iut) - assert(fields.syslogfacility == 0, fields.syslogfacility) - assert(fields["syslogfacility-text"] == 0, fields["syslogfacility-text"]) - assert(fields.syslogpriority == 6, fields.syslogpriority) - assert(fields["syslogpriority-text"] == 6, fields["syslogpriority-text"]) - assert(fields.timegenerated == 1392024053000000000, fields.timegenerated) - -- time reported is mapped to timestamp - assert(fields.timestamp == 1392052853559934000, fields.timestamp) - assert(fields["protocol-version"] == "0", fields["protocol-version"]) - assert(fields["structured-data"] == "-", fields["structured-data"]) - assert(fields["app-name"] == "kernel", fields["app-name"]) - assert(fields.procid == "", fields.procid) - assert(fields.msgid == "-", fields.msgid) - assert(fields["$now"] == "2014-02-10", fields["$now"]) - assert(fields["$year"] == "2014") - assert(fields["$month"] == "02") - assert(fields["$day"] == "10") - assert(fields["$hour"] == "09") - assert(fields["$hhour"] == "00") - assert(fields["$qhour"] == "01") - assert(fields["$minute"] == "20") - assert(fields.msg == "imklog 5.8.6, log source = /proc/kmsg started.", fields.msg) -end - -local function no_colon_in_tag() - local grammar = syslog.build_rsyslog_grammar('<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag:1:32%%msg:::sp-if-no-1st-sp%%msg%') - local log = '<166>2014-06-26T23:13:23-07:00 example.com at java.net.SocketInputStream.socketRead0(Native Method)' - local fields = grammar:match(log) - assert(fields.msg == "at java.net.SocketInputStream.socketRead0(Native Method)", fields.msg) - assert(fields.timestamp == 1403849603000000000, fields.timestamp) - assert(fields.syslogtag.programname == "", fields.syslogtag.programname) - assert(fields.pri.severity == 6, fields.pri.severity) - assert(fields.pri.facility == 20, fields.pri.facility) - assert(fields.hostname == "example.com", fields.hostname) -end - -function process() - traditional_file_format() - file_format() - forward_format() - traditional_forward_format() - kitchen_sink() - no_colon_in_tag() - - return 0 -end - diff --git a/src/test/lua/no_external_modules.lua b/src/test/lua/no_external_modules.lua new file mode 100644 index 0000000..c446b69 --- /dev/null +++ b/src/test/lua/no_external_modules.lua @@ -0,0 +1,8 @@ +require "foo" + +function process(tc) + return 0 +end + +function report() +end diff --git a/src/test/lua/output.lua b/src/test/lua/output.lua index 5dd6ad6..43de543 100644 --- a/src/test/lua/output.lua +++ b/src/test/lua/output.lua @@ -2,38 +2,14 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "circular_buffer" -require "cjson" - -local cbuf = circular_buffer.new(1440, 3, 60) -local benchmark = {Timestamp = 1e9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} function process(tc) if tc == 0 then -- lua types write_output(1.2, " string ", nil, " ", true, " ", false) - elseif tc == 1 then -- cbuf - write_output("annotation\n", cbuf) - elseif tc == 2 then -- heka message - local hm = {Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9} - write_message(hm) - elseif tc == 3 then -- heka message field - local hm = {Timestamp = 1e9, Fields = {count=1}} - write_message(hm) - elseif tc == 4 then -- heka message field array - local hm = {Timestamp = 1e9, Fields = {counts={2,3,4}}} - write_message(hm) - elseif tc == 5 then -- hekamessage field metadata - local hm = {Timestamp = 1e9, Fields = {count={value=5,representation="count"}}} - write_message(hm) - elseif tc == 6 then -- heka message field metadata array - local hm = {Timestamp = 1e9, Fields = {counts={value={6,7,8},representation="count"}}} - write_message(hm) - elseif tc == 7 then -- heka message field all types - local hm = benchmark - write_message(hm) - elseif tc == 8 then -- heka message force memmove - local hm = {Timestamp = 1e9, Fields = {string="0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789"}} - write_message(hm) + elseif tc == 1 then -- user data + require "ud" + local udv = ud.new("foo") + write_output(udv) end return 0 end diff --git a/src/test/lua/output_errors.lua b/src/test/lua/output_errors.lua index 1bc3944..401470e 100644 --- a/src/test/lua/output_errors.lua +++ b/src/test/lua/output_errors.lua @@ -2,9 +2,9 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "cjson" -require "circular_buffer" -require "lpeg" +require "io" + + function process(tc) if tc == 0 then -- error internal reference output({}) @@ -14,26 +14,8 @@ function process(tc) escape = escape .. escape end output(escape) - elseif tc == 2 then -- heka error mis-match field array - local hm = {Timestamp = 1e9, Fields = {counts={2,"ten",4}}} - write_message(hm) - elseif tc == 3 then -- heka error nil field - local hm = {Timestamp = 1e9, Fields = {counts={}}} - write_message(hm) - elseif tc == 4 then -- unsupported userdata - write_output(lpeg.P"") - elseif tc == 5 then -- overflow - local cb = circular_buffer.new(1000, 1, 60); - write_output(cb) - elseif tc == 6 then -- invalid array type - local hm = {Timestamp = 1e9, Fields = {counts={{1},{2}}}} - write_message(hm) - elseif tc == 7 then -- overflow cjson encode buffer - local t = {} - for i=1, 10 do - t[#t+1] = "this is a test" - end - cjson.encode(t) + elseif tc == 2 then -- unsupported userdata + write_output(io.stdin) end return 0 end diff --git a/src/test/lua/print.lua b/src/test/lua/print.lua new file mode 100644 index 0000000..dc74cfc --- /dev/null +++ b/src/test/lua/print.lua @@ -0,0 +1,19 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +function process(tc) + if tc == 0 then + print() + elseif tc == 1 then + print("foo\n", 10, true) + elseif tc == 2 then + print("f\r\0", 10, true) + elseif tc == 3 then + tostring = nil + local ok, err = pcall(print, "foo") + assert(not ok) + assert(err == "attempt to call a nil value", err) + end + return 0 +end diff --git a/src/test/lua/read_config.lua b/src/test/lua/read_config.lua new file mode 100644 index 0000000..30d1c8d --- /dev/null +++ b/src/test/lua/read_config.lua @@ -0,0 +1,18 @@ +assert(type(read_config) == "function") +assert(read_config("memory_limit") == 65765) +assert(read_config("instruction_limit") == 1000) +assert(read_config("output_limit") == 1024) +assert(read_config("input_limit") == 1024 * 64) + +local array = read_config("array") +assert(type(array) == "table") +assert(array[1] == "foo") +assert(array[2] == 99) + +local hash = read_config("hash") +assert(type(hash) == "table") +assert(hash.foo == "bar") +assert(hash.hash1.subfoo == "subbar") + +assert(type(read_config("path")) == "string") +assert(type(read_config("cpath")) == "string") diff --git a/src/test/lua/sandbox_config.lua b/src/test/lua/sandbox_config.lua new file mode 100644 index 0000000..c54d1cc --- /dev/null +++ b/src/test/lua/sandbox_config.lua @@ -0,0 +1,145 @@ +local ok, err = pcall(require, "io") +assert(not ok, "the io module is disabled") +assert(err == "module 'io' disabled", err) + +require "math" +require "os" +require "string" +require "table" + +local function all_there(test) + for k, v in pairs(test.values) do + assert(test.t[k], string.format("missing: %s.%s", test.name, k)) + end +end + +local function nothing_extra(test) + for k, v in pairs(test.t) do + assert(test.values[k], string.format("extra: %s.%s", test.name, k)) + end +end + +local tests = { + {name = "base", t = _G, values = { + _G=1, + _VERSION=1, + assert=1, + error=1, + gcinfo=1, + getfenv=1, + getmetatable=1, + ipairs=1, + read_config=1, + math=1, + module=1, + next=1, + os=1, + output=1, + package=1, + pairs=1, + pcall=1, + print=1, + rawequal=1, + rawget=1, + rawset=1, + require=1, + select=1, + setfenv=1, + setmetatable=1, + string=1, + table=1, + tonumber=1, + tostring=1, + type=1, + unpack=1, + xpcall=1, + } + }, + + {name = "math", t = math, values = { + abs=1, + acos=1, + asin=1, + atan2=1, + atan=1, + ceil=1, + cos=1, + cosh=1, + deg=1, + erf=1, + erfc=1, + exp=1, + floor=1, + fmod=1, + frexp=1, + huge=1, + ldexp=1, + log10=1, + log=1, + max=1, + min=1, + mod=1, -- compat + modf=1, + pi=1, + pow=1, + rad=1, + random=1, + randomseed=1, + sin=1, + sinh=1, + sqrt=1, + tan=1, + tanh=1, + } + }, + + {name = "os", t = os, values = { + clock=1, + date=1, + difftime=1, + time=1, + } + }, + + {name = "package", t = package, values = { + seeall=1, + } + }, + + {name = "string", t = string, values = { + byte=1, + char=1, + dump=1, + find=1, + format=1, + gfind=1, -- compat + gmatch=1, + gsub=1, + len=1, + lower=1, + match=1, + rep=1, + reverse=1, + sub=1, + upper=1, + } + }, + + {name = "table", t = table, values = { + concat=1, + foreach=1, + foreachi=1, + getn=1, + insert=1, + maxn=1, + remove=1, + setn=1, + sort=1, + } + }, +} + +for i, v in ipairs(tests) do + all_there(v) + nothing_extra(v) +end diff --git a/src/test/lua/serialize.lua b/src/test/lua/serialize.lua index 96703d2..cdb088c 100644 --- a/src/test/lua/serialize.lua +++ b/src/test/lua/serialize.lua @@ -2,14 +2,13 @@ -- License, v. 2.0. If a copy of the MPL was not distributed with this -- file, You can obtain one at http://mozilla.org/MPL/2.0/. -require "circular_buffer" -util = require "util" +require "ud" count = 0 rate = 0.12345678 rates = {99.1,98,97,92.002,91.10001,key="val"} kvp = {a="foo", b="bar", r=rates} -nested = {arg1=1, arg2=2, nested={n1="one",n2="two"}, empty = nil, cb = circular_buffer.new(2,6,1)} +nested = {arg1=1, arg2=2, nested={n1="one",n2="two"}, empty = nil, ud = ud.new("ud1")} _G["key with spaces"] = "kws" boolean = true empty = nil @@ -18,6 +17,9 @@ uuids = { {uuid="BD48B609-8922-4E59-A358-C242075CE088", type="test"}, {uuid="BD48B609-8922-4E59-A358-C242075CE089", type="test1"} } +nan = 0/0 +inf = 1/0 +ninf = -1/0 large_key = { aaaaaaaaaaaaaaaaaaa = {["BD48B609-8922-4E59-A358-C242075CE081"] = 1, @@ -36,13 +38,17 @@ cycleb = {type="cycle b"} cyclea["b"] = cycleb cycleb["a"] = cyclea -data = circular_buffer.new(3,3,1) - -delta = circular_buffer.new(2,1,1, true) -delta:add(0, 1, 2) +data = ud.new("ud2") dataRef = data +empty_array = {nil} +empty_array1 = {} +empty_array1[1] = nil +empty_object = {} +empty_object1 = {[1] = nil} +array = {1, "two", 3, "four", 5} + function process_message () return 0 end @@ -51,4 +57,3 @@ end function timer_event(ns) end - diff --git a/src/test/lua/struct.lua b/src/test/lua/struct.lua deleted file mode 100644 index 3f41da8..0000000 --- a/src/test/lua/struct.lua +++ /dev/null @@ -1,11 +0,0 @@ -require "struct" - -local fmt = " +#include +#include + +#include "luasandbox.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox_output.h" +#include "../luasandbox_defines.h" + +const char *lsb_test_output = NULL; +size_t lsb_test_output_len = 0; + +static void logger(void *context, + const char *component, + int level, + const char *fmt, + ...) +{ + (void)context; + va_list args; + fprintf(stderr, "%lld [%d] %s ", (long long)time(NULL), level, + component ? component : "unnamed"); + va_start(args, fmt); + vfprintf(stderr, fmt, args); + va_end(args); + fwrite("\n", 1, 1, stderr); +} +lsb_logger lsb_test_logger = { .context = NULL, .cb = logger }; + + +int lsb_test_write_output(lua_State *lua) +{ + lua_getfield(lua, LUA_REGISTRYINDEX, LSB_THIS_PTR); + lsb_lua_sandbox *lsb = lua_touserdata(lua, -1); + lua_pop(lua, 1); // remove this ptr + if (!lsb) return luaL_error(lua, "%s() invalid " LSB_THIS_PTR, __func__); + + int n = lua_gettop(lua); + lsb_output(lsb, 1, n, 1); + lsb_test_output = lsb_get_output(lsb, &lsb_test_output_len); + return 0; +} + + +int lsb_test_process(lsb_lua_sandbox *lsb, double tc) +{ + static const char *func_name = "process"; + lua_State *lua = lsb_get_lua(lsb); + if (!lua) return 1; + + if (lsb_pcall_setup(lsb, func_name)) return 1; + + lua_pushnumber(lua, tc); + if (lua_pcall(lua, 1, 2, 0) != 0) { + char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); + int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, + em ? em : LSB_NIL_ERROR); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + + if (!lua_isnumber(lua, 1)) { + char err[LSB_ERROR_SIZE]; + int len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a numeric error code", func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + + int status = (int)lua_tointeger(lua, 1); + switch (lua_type(lua, 2)) { + case LUA_TNIL: + lsb_set_error(lsb, NULL); + break; + case LUA_TSTRING: + lsb_set_error(lsb, lua_tostring(lua, 2)); + break; + default: + { + char err[LSB_ERROR_SIZE]; + int len = snprintf(err, LSB_ERROR_SIZE, + "%s() must return a nil or string error message", + func_name); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + break; + } + lua_pop(lua, 2); + + lsb_pcall_teardown(lsb); + + return status; +} + + +int lsb_test_report(lsb_lua_sandbox *lsb, double tc) +{ + static const char *func_name = "report"; + lua_State *lua = lsb_get_lua(lsb); + if (!lua) return 1; + + if (lsb_pcall_setup(lsb, func_name)) return 1; + + lua_pushnumber(lua, tc); + if (lua_pcall(lua, 1, 0, 0) != 0) { + char err[LSB_ERROR_SIZE]; + const char *em = lua_tostring(lua, -1); + int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, + em ? em : LSB_NIL_ERROR); + if (len >= LSB_ERROR_SIZE || len < 0) { + err[LSB_ERROR_SIZE - 1] = 0; + } + lsb_terminate(lsb, err); + return 1; + } + + lsb_pcall_teardown(lsb); + lua_gc(lua, LUA_GCCOLLECT, 0); + + return 0; +} diff --git a/src/test/test_generic_sandbox.c b/src/test/test_generic_sandbox.c new file mode 100644 index 0000000..b16d2e2 --- /dev/null +++ b/src/test/test_generic_sandbox.c @@ -0,0 +1,939 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Lua sandbox unit tests @file */ + +#include +#include +#include +#include +#include + +#include "../luasandbox_impl.h" +#include "luasandbox/lauxlib.h" +#include "luasandbox/lua.h" +#include "luasandbox/test/mu_test.h" +#include "luasandbox/test/sandbox.h" +#include "luasandbox/util/util.h" +#include "luasandbox_output.h" +#include "luasandbox_serialize.h" + +char *e = NULL; +static char print_out[2048] = { 0 }; + + +#ifdef _WIN32 +#define MODULE_PATH "path = 'modules\\\\?.lua';cpath = 'modules\\\\?.dll'\n" +#else +#define MODULE_PATH "path = 'modules/?.lua';cpath = 'modules/?.so'\n" +#endif + +static const char *mozsvc_test_ud = "mozsvc.test_ud"; + +typedef struct test_ud { + char name[10]; +} test_ud; + + +static int ud_new(lua_State *lua) +{ + size_t len; + const char *name = luaL_checklstring(lua, 1, &len); + test_ud *ud = lua_newuserdata(lua, sizeof(test_ud)); + if (len < 10) { + strcpy(ud->name, name); + } else { + memcpy(ud->name, name, 9); + ud->name[9] = 0; + } + luaL_getmetatable(lua, mozsvc_test_ud); + lua_setmetatable(lua, -2); + return 1; +} + + +static int ud_serialize(lua_State *lua) +{ + lsb_output_buffer *ob = lua_touserdata(lua, -1); + const char *key = lua_touserdata(lua, -2); + test_ud *ud = lua_touserdata(lua, -3); + if (!(ob && key && ud)) {return 1;} + return lsb_outputf(ob, "if %s == nil then %s = ud.new('%s') end\n", key, key, + ud->name) == NULL ? 0 : 1; +} + + +static int ud_output(lua_State *lua) +{ + lsb_output_buffer *ob = lua_touserdata(lua, -1); + test_ud *ud = lua_touserdata(lua, -2); + if (!(ob && ud)) {return 1; } + return lsb_outputf(ob, "%s", ud->name) == NULL ? 0 : 1; +} + + +static const struct luaL_reg testudlib_f[] = +{ + { "new", ud_new }, { NULL, NULL } +}; + + +static int luaopen_ud(lua_State *lua) +{ + lua_newtable(lua); + lsb_add_serialize_function(lua, ud_serialize); + lsb_add_output_function(lua, ud_output); + lua_replace(lua, LUA_ENVIRONINDEX); + luaL_newmetatable(lua, mozsvc_test_ud); + luaL_register(lua, "ud", testudlib_f); + return 1; +} + + +static void add_ud_module(lsb_lua_sandbox *sb) +{ + lua_State *lua = lsb_get_lua(sb); + luaL_findtable(lua, LUA_REGISTRYINDEX, "_PRELOADED", 1); + lua_pushstring(lua, "ud"); + lua_pushcfunction(lua, luaopen_ud); + lua_rawset(lua, -3); + lua_pop(lua, 1); // remove the preloaded table +} + + +static const char *test_cfg = + "userflag = true\n" + "memory_limit = 0\n" + "instruction_limit = 0\n" + "output_limit = 0\n" + "remove_entries = {\n" + "[''] = {'collectgarbage','dofile','load','loadfile','loadstring'," + "'newproxy','print'},\n" + "os = {'getenv','execute','exit','remove','rename','setlocale','tmpname'}\n" + "}\n" + "disable_modules = {io = 1, coroutine = 1}\n" + "log_level = 7\n" + MODULE_PATH +; + + +int file_exists(const char *fn) +{ + FILE *fh; + fh = fopen(fn, "r"); + if (fh) { + fclose(fh); + return 1; + } + return 0; +} + + +void print(void *context, const char *component, int level, const char *fmt, ...) +{ + (void)context; + va_list args; + int n = snprintf(print_out, sizeof print_out, "%d %s ", level, + component ? component : "unnamed"); + va_start(args, fmt); + n = vsnprintf(print_out + n, sizeof print_out - n, fmt, args); + va_end(args); +} +static lsb_logger printer = { .context = NULL, .cb = print }; + + +static char* test_api_assertion() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", "", NULL); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + lsb_stop_sandbox(NULL); + mu_assert(lsb_destroy(NULL) == NULL, "not null"); + mu_assert(lsb_usage(NULL, 0, 0) == 0, "not 0"); + mu_assert(lsb_usage(sb, LSB_UT_MAX, 0) == 0, "not 0"); + mu_assert(lsb_usage(sb, 0, LSB_US_MAX) == 0, "not 0"); + mu_assert(strcmp(lsb_get_error(NULL), "") == 0, "not empty"); + lsb_set_error(NULL, "foo"); + mu_assert(lsb_get_lua(NULL) == NULL, "not null"); + mu_assert(lsb_get_lua_file(NULL) == NULL, "not null"); + mu_assert(lsb_get_parent(NULL) == NULL, "not null"); + mu_assert(lsb_get_logger(NULL) == NULL, "not null"); + mu_assert(lsb_get_state(NULL) == LSB_UNKNOWN, "not unknown"); + lsb_add_function(NULL, lsb_test_write_output, "foo"); + lsb_add_function(sb, NULL, "foo"); + lsb_add_function(sb, lsb_test_write_output, NULL); + mu_assert(lsb_pcall_setup(NULL, "foo") == LSB_ERR_UTIL_NULL, "not null"); + mu_assert(lsb_pcall_setup(sb, NULL) == LSB_ERR_UTIL_NULL, "not null"); + lsb_add_function(NULL, NULL, NULL); + lsb_pcall_teardown(NULL); + lsb_terminate(NULL, NULL); + lsb_terminate(sb, NULL); + lsb_add_function(sb, lsb_test_write_output, "write_output"); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_create() +{ + static char *cfg = "function foo() return 0 end\nt = {[true] = 1}\n"; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", cfg, &lsb_test_logger); + mu_assert(sb, "lsb_create() failed"); + lsb_destroy(sb); + sb = lsb_create(NULL, "lua/counter.lua", cfg, NULL); + mu_assert(sb, "lsb_create() failed"); + lsb_destroy(sb); + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 3e9", NULL); + mu_assert(sb, "lsb_create() failed"); + lsb_destroy(sb); + return NULL; +} + + +static char* test_create_error() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, NULL, NULL, NULL); + mu_assert(!sb, "lsb_create() null lua_file"); + + sb = lsb_create(NULL, "lua/counter.lua", "input_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "output_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = -1", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "memory_limit = 1.85e19", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "instruction_limit = 'aaa'", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "path = 1", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "cpath = 1", NULL); + mu_assert(!sb, "lsb_create() invalid config"); + + sb = lsb_create(NULL, "lua/counter.lua", "test = {", &lsb_test_logger); + mu_assert(!sb, "lsb_create() invalid config"); + + return NULL; +} + + +static char* test_read_config() +{ + const char *cfg = "memory_limit = 65765\n" + "instruction_limit = 1000\n" + "output_limit = 1024\n" + "array = {'foo', 99}\n" + "hash = {foo = 'bar', hash1 = {subfoo = 'subbar'}}\n" + MODULE_PATH; + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/read_config.lua", cfg, NULL); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_init_error() +{ + // null sandbox + lsb_err_value ret = lsb_init(NULL, NULL); + mu_assert(ret == LSB_ERR_UTIL_NULL, "lsb_init() null sandbox ptr"); + + // load error + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple1.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + ret = lsb_init(sb, NULL); + mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); + lsb_state s = lsb_get_state(sb); + mu_assert(s == LSB_TERMINATED, "lsb_get_state() received: %d", s); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + // out of memory + sb = lsb_create(NULL, "lua/simple.lua", "memory_limit = 6000", NULL); + mu_assert(sb, "lsb_create() received: NULL"); + ret = lsb_init(sb, NULL); + mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s\n", e); + + sb = lsb_create(NULL, "lua/no_external_modules.lua", NULL, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + // disabled external modules + ret = lsb_init(sb, NULL); + mu_assert(ret == LSB_ERR_LUA, "lsb_init() received: %s", lsb_err_string(ret)); + const char *expected = "no 'path' configuration was specified for the " + "sandbox; external modules have been disabled"; + mu_assert(strcmp(lsb_get_error(sb), expected) == 0, + "lsb_get_error() received: %s", lsb_get_error(sb)); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_destroy_error() +{ + const char *expected = "preserve_global_data could not open: " + "invaliddir/simple.preserve"; + e = lsb_destroy(NULL); + mu_assert(!e, "lsb_destroy() received: %s", e); + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + lsb_err_value ret = lsb_init(sb, "invaliddir/simple.preserve"); + mu_assert(!ret, "lsb_init() received: %s", ret); + e = lsb_destroy(sb); + mu_assert(e, "lsb_destroy() received NULL"); + mu_assert(strcmp(e, expected) == 0, + "lsb_destroy() received: %s", e); + free(e); + e = NULL; + + return NULL; +} + + +static char* test_usage_error() +{ + size_t u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); + mu_assert(u == 0, "NULL sandbox memory usage received: %" PRIuSIZE, u); + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + u = lsb_usage(NULL, LSB_UT_MAX + 1, LSB_US_CURRENT); + mu_assert(u == 0, "Invalid usage type received: %" PRIuSIZE, u); + + u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_MAX + 1); + mu_assert(u == 0, "Invalid usage stat received: %" PRIuSIZE, u); + + mu_assert(sb, "lsb_create() received: NULL"); + lsb_terminate(sb, "forced termination"); + lsb_state s = lsb_get_state(sb); + mu_assert(s == LSB_TERMINATED, "lsb_get_state() received: %d", s); + u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); + mu_assert(u > 0, "Terminated memory usage received: 0"); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_stop() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_stop_sandbox(sb); + lua_getglobal(lsb_get_lua(sb), "process"); + lua_pushnumber(lsb_get_lua(sb), 0); + mu_assert_rv(2, lua_pcall(lsb_get_lua(sb), 1, 2, 0)); + const char *msg = lua_tostring(lsb_get_lua(sb), -1); + mu_assert(strcmp(LSB_SHUTTING_DOWN, msg) == 0, "received: %s", msg); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_simple() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", + "memory_limit = 65765;" + "instruction_limit = 1000;" + "output_limit = 1024;", NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, "simple.preserve"); + mu_assert(!ret, "lsb_init() received: %s", ret); + + size_t u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); + mu_assert(u > 0, "Current memory usage received: %" PRIuSIZE, u); + printf("cur_mem %" PRIuSIZE "\n", u); + + u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_MAXIMUM); + mu_assert(u > 0, "Maximum memory usage received: %" PRIuSIZE, u); + printf("max_mem %" PRIuSIZE "\n", u); + + u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_LIMIT); + mu_assert(u == 65765, "Memory limit received: %" PRIuSIZE, u); + + u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_CURRENT); + mu_assert(u == 7, "Current instructions received: %" PRIuSIZE, u); + + u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_MAXIMUM); + mu_assert(u == 7, "Maximum instructions received: %" PRIuSIZE, u); + printf("max_ins %" PRIuSIZE "\n", u); + + u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_LIMIT); + mu_assert(u == 1000, "Instruction limit received: %" PRIuSIZE, u); + + u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_CURRENT); + mu_assert(u == 0, "Current output received: %" PRIuSIZE, u); + + u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_MAXIMUM); + mu_assert(u == 0, "Maximum output received: %" PRIuSIZE, u); + printf("max_out %" PRIuSIZE "\n", u); + + u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); + mu_assert(u == 1024, "Output limit received: %" PRIuSIZE, u); + + u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); + mu_assert(u == 1024, "Output limit received: %" PRIuSIZE, u); + + lsb_state s = lsb_get_state(sb); + mu_assert(s == LSB_RUNNING, "lsb_get_state() received: %d", s); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + + +static char* test_simple_error() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/simple.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + int result = lsb_test_process(sb, 1); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + + mu_assert(strcmp("ok", lsb_get_error(sb)) == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + + result = lsb_test_process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + + mu_assert(strcmp("", lsb_get_error(sb)) == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + + result = lsb_test_process(sb, 2); + mu_assert(result == 1, "process() received: %d %s", result, + lsb_get_error(sb)); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_output() +{ + const char *outputs[] = { + "1.2 string nil true false", + "foo", + NULL + }; + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + + for (int x = 0; outputs[x]; ++x) { + int result = lsb_test_process(sb, x); + mu_assert(!result, "process() test: %d failed: %d %s", x, result, + lsb_get_error(sb)); + if (outputs[x][0]) { + mu_assert(strcmp(outputs[x], lsb_test_output) == 0, + "test: %d received: %s", x, lsb_test_output); + } + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_output_errors() +{ + const char *tests[] = + { + "process() lua/output_errors.lua:10: bad argument #1 to 'output' (unsupported type)" + , "process() lua/output_errors.lua:16: output_limit exceeded" + , "process() lua/output_errors.lua:18: bad argument #1 to 'write_output' (unknown userdata type)" + , NULL + }; + + for (int i = 0; tests[i]; ++i) { + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output_errors.lua", + MODULE_PATH "output_limit = 128", NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + + int result = lsb_test_process(sb, i); + mu_assert(result == 1, "test: %d received: %d", i, result); + + const char *le = lsb_get_error(sb); + mu_assert(le, "test: %d received NULL", i); + mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + } + + return NULL; +} + + +static char* test_errors() +{ + const char *tests[] = { +#ifdef _WIN32 + "process() lua/errors.lua:9: module 'unknown' not found:\n\tno file 'lua\\unknown.lua'\n\tno file 'lua\\unknown.dll'" +#else + "process() lua/errors.lua:9: module 'unknown' not found:\n\tno file 'lua/unknown.lua'\n\tno file 'lua/unknown.so'" +#endif + , "process() lua/errors.lua:11: bad argument #0 to 'output' (must have at least one argument)" + , "process() not enough memory" + , "process() instruction_limit exceeded" + , "process() lua/errors.lua:20: attempt to perform arithmetic on global 'x' (a nil value)" + , "process() must return a numeric error code" + , "process() must return a numeric error code" + , "process() lua/errors.lua:27: output_limit exceeded" +#ifdef _WIN32 + , "process() lua\\bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" +#else + , "process() lua/bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" +#endif + , "process() invalid module name '../invalid'" + , "process() lua/errors.lua:34: module 'pathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpathoverflowpa" +#ifdef _WIN32 + , "process() lua/errors.lua:36: module 'foo.bar' not found:\n\tno file 'lua\\foo\\bar.lua'\n\tno file 'lua\\foo\\bar.dll'\n\tno file 'lua\\foo.dll'" +#else + , "process() lua/errors.lua:36: module 'foo.bar' not found:\n\tno file 'lua/foo/bar.lua'\n\tno file 'lua/foo/bar.so'\n\tno file 'lua/foo.so'" +#endif + , NULL + }; + + for (int i = 0; tests[i]; ++i) { + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/errors.lua", + "memory_limit = 32767;" + "instruction_limit = 1000;" + "output_limit = 128;" +#ifdef _WIN32 + "path = 'lua\\\\?.lua'" + "cpath = 'lua\\\\?.dll'", +#else + "path = 'lua/?.lua'" + "cpath = 'lua/?.so'", +#endif + NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + int result = lsb_test_process(sb, i); + mu_assert(result == 1, "test: %d received: %d", i, result); + + const char *le = lsb_get_error(sb); + mu_assert(le, "test: %d received NULL", i); + mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + } + + return NULL; +} + + +static char* test_serialize() +{ + const char *output_file = "serialize.preserve"; + + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); + + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + char *expected = lsb_read_file("output/serialize.lua51.data"); + char *actual = lsb_read_file(output_file); + mu_assert(strcmp(expected, actual) == 0, "serialization mismatch"); + free(expected); + free(actual); + + return NULL; +} + + +static char* test_restore() +{ + const char *output_file = "restore.preserve"; + + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + int result = lsb_test_process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + mu_assert(strcmp("101", lsb_test_output) == 0, "test: initial load received: %s", + lsb_test_output); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + // re-load to test the preserved data + sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + result = lsb_test_process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + mu_assert(strcmp("102", lsb_test_output) == 0, "test: reload received: %s", + lsb_test_output); + result = lsb_test_report(sb, 2); // change the preservation version + mu_assert(result == 0, "report() received: %d", result); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + // re-load to test the preserved data with a version change + sb = lsb_create(NULL, "lua/restore.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + result = lsb_test_process(sb, 0); + mu_assert(result == 0, "process() received: %d %s", result, + lsb_get_error(sb)); + mu_assert(strcmp("101", lsb_test_output) == 0, + "test: reload with version change received: %s", lsb_test_output); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_serialize_failure() +{ + const char *output_file = "serialize_failure.preserve"; + const char *expected = "serialize_data cannot preserve type 'function'"; + + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, + "lua/serialize_failure.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s", ret); + e = lsb_destroy(sb); + mu_assert(e, "lsb_destroy() received: no error"); + mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); + free(e); + e = NULL; + mu_assert(file_exists(output_file) == 0, "output file was not cleaned up"); + + return NULL; +} + + +static char* test_sandbox_config() +{ + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/sandbox_config.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + + return NULL; +} + + +static char* test_print() +{ + const char *tests[] = + { + "" + , "7 lua/print.lua foo \t10\ttrue" + , "7 lua/print.lua f \t10\ttrue" + , "" + , NULL + }; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;", &printer); + mu_assert(sb, "lsb_create() received: NULL"); + mu_assert(lsb_get_logger(sb) != NULL, "no logger"); + mu_assert(lsb_get_logger(sb)->cb == printer.cb, "incorrect logger callback"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + for (int i = 0; tests[i]; ++i) { + print_out[0] = 0; + int result = lsb_test_process(sb, i); + mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); + mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_print_disabled() +{ + const char *tests[] = + { + "" + , "" + , NULL + }; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 6;", &printer); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + for (int i = 0; tests[i]; ++i) { + print_out[0] = 0; + int result = lsb_test_process(sb, i); + mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); + mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_print_lsb_test_logger() +{ + const char *tests[] = + { + "" + , "7 test.print foo \t10\ttrue" + , NULL + }; + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/print.lua", "log_level = 7;Logger = 'test.print';", &printer); + mu_assert(sb, "lsb_create() received: NULL"); + + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + + for (int i = 0; tests[i]; ++i) { + print_out[0] = 0; + int result = lsb_test_process(sb, i); + mu_assert(result == 0, "test: %d received: %d error: %s", i, result, lsb_get_error(sb)); + mu_assert(strcmp(tests[i], print_out) == 0, "test: %d expected: %s received: %s", i, tests[i], print_out); + } + + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + return NULL; +} + + +static char* test_serialize_binary() +{ + size_t size = 512; + lsb_output_buffer b; + lsb_err_value ret = lsb_init_output_buffer(&b, size); + mu_assert(ret == NULL, "received: %s", lsb_err_string(ret)); + lsb_serialize_binary(&b, "a\r\n\\\"", 5); + mu_assert(b.pos == 9, "received %d", (int)b.pos); + mu_assert(memcmp(b.buf, "a\\r\\n\\\\\\\"", 5) == 0, "received %.*s", 9, + b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* benchmark_counter() +{ + int iter = 10000000; + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/counter.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_test_process(sb, 0); + } + t = clock() - t; + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + printf("benchmark_counter() %g seconds\n", ((double)t) / CLOCKS_PER_SEC + / iter); + + return NULL; +} + + +static char* benchmark_serialize() +{ + int iter = 1000; + const char *output_file = "serialize.preserve"; + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + remove(output_file); + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); + + lsb_err_value ret = lsb_init(sb, output_file); + mu_assert(!ret, "lsb_init() received: %s %s", ret, lsb_get_error(sb)); + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + } + t = clock() - t; + printf("benchmark_serialize() %g seconds\n", ((double)t) / CLOCKS_PER_SEC + / iter); + + return NULL; +} + + +static char* benchmark_deserialize() +{ + int iter = 1000; + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/serialize.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + add_ud_module(sb); + + lsb_err_value ret = lsb_init(sb, "output/serialize.data"); + mu_assert(!ret, "lsb_init() received: %s", ret); + free(sb->state_file); + sb->state_file = NULL; // poke the internals to prevent serialization + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + } + t = clock() - t; + printf("benchmark_deserialize() %g seconds\n", ((double)t) / CLOCKS_PER_SEC + / iter); + + return NULL; +} + + +static char* benchmark_lua_types_output() +{ + int iter = 1000000; + + lsb_lua_sandbox *sb = lsb_create(NULL, "lua/output.lua", test_cfg, NULL); + mu_assert(sb, "lsb_create() received: NULL"); + lsb_err_value ret = lsb_init(sb, NULL); + mu_assert(!ret, "lsb_init() received: %s", ret); + lsb_add_function(sb, &lsb_test_write_output, "write_output"); + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(0 == lsb_test_process(sb, 0), "%s", lsb_get_error(sb)); + } + t = clock() - t; + e = lsb_destroy(sb); + mu_assert(!e, "lsb_destroy() received: %s", e); + printf("benchmark_lua_types_output() %g seconds\n", ((double)t) + / CLOCKS_PER_SEC / iter); + + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_api_assertion); + mu_run_test(test_create); + mu_run_test(test_create_error); + mu_run_test(test_read_config); + mu_run_test(test_init_error); + mu_run_test(test_destroy_error); + mu_run_test(test_usage_error); + mu_run_test(test_stop); + mu_run_test(test_simple); + mu_run_test(test_simple_error); + mu_run_test(test_output); + mu_run_test(test_output_errors); + mu_run_test(test_errors); + mu_run_test(test_serialize); + mu_run_test(test_restore); + mu_run_test(test_serialize_failure); + mu_run_test(test_sandbox_config); + mu_run_test(test_print); + mu_run_test(test_print_disabled); + mu_run_test(test_print_lsb_test_logger); + mu_run_test(test_serialize_binary); + + mu_run_test(benchmark_counter); + mu_run_test(benchmark_serialize); + mu_run_test(benchmark_deserialize); + mu_run_test(benchmark_lua_types_output); + + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + free(e); + + return result != NULL; +} diff --git a/src/test/test_lua_sandbox.c b/src/test/test_lua_sandbox.c deleted file mode 100644 index 858a2a7..0000000 --- a/src/test/test_lua_sandbox.c +++ /dev/null @@ -1,1565 +0,0 @@ -/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ -/* vim: set ts=2 et sw=2 tw=80: */ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -/** @brief Lua sandbox unit tests @file */ - -#include "lua_sandbox.h" - -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#define snprintf _snprintf -#endif - -#define mu_assert(cond, ...) \ -do { \ - if (!(cond)) { \ - int cnt = snprintf(mu_err, MU_ERR_LEN, "line: %d (%s) ", __LINE__, #cond); \ - if (cnt > 0 && cnt < MU_ERR_LEN) { \ - cnt = snprintf(mu_err+cnt, MU_ERR_LEN-cnt, __VA_ARGS__); \ - if (cnt > 0 && cnt < MU_ERR_LEN) { \ - return mu_err; \ - } \ - } \ - mu_err[MU_ERR_LEN - 1] = 0; \ - return mu_err; \ - } \ -} while (0) - -#define mu_run_test(test) \ -do { \ - char *message = test(); \ - mu_tests_run++; \ - if (message) \ - return message; \ -} while (0) - -#define MU_ERR_LEN 1024 -int mu_tests_run = 0; -char mu_err[MU_ERR_LEN] = { 0 }; -char* e = NULL; -const char* written_data = NULL; -size_t written_data_len = 0; - - -int file_exists(const char* fn) -{ - FILE* fh; - fh = fopen(fn, "r"); - if (fh) { - fclose(fh); - return 1; - } - return 0; -} - - -char* read_file(const char* fn) -{ - char* str = NULL; - FILE* fh; - fh = fopen(fn, "rb"); - mu_assert(fh != NULL, "fopen() failed"); - - int result = fseek(fh, 0, SEEK_END); - mu_assert(result == 0, "fseek(SEEK_END) failed %d", result); - - long pos = ftell(fh); - mu_assert(pos != -1, "ftell() failed %s", strerror(errno)); - rewind(fh); - - str = malloc(pos + 1); - mu_assert(str != NULL, "malloc failed"); - - size_t b = fread(str, 1, pos, fh); - mu_assert((long)b == pos, "fread() failed"); - str[pos] = 0; - - fclose(fh); - return str; -} - - -int process(lua_sandbox* lsb, double ts) -{ - static const char* func_name = "process"; - lua_State* lua = lsb_get_lua(lsb); - if (!lua) return 1; - - if (lsb_pcall_setup(lsb, func_name)) return 1; - - lua_pushnumber(lua, ts); - if (lua_pcall(lua, 1, 2, 0) != 0) { - char err[LSB_ERROR_SIZE]; - int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - lua_tostring(lua, -1)); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - - if (!lua_isnumber(lua, 1)) { - char err[LSB_ERROR_SIZE]; - int len = snprintf(err, LSB_ERROR_SIZE, - "%s() must return a single numeric value", func_name); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - - int status = (int)lua_tointeger(lua, 1); - switch (lua_type(lua, 2)) { - case LUA_TNIL: - lsb_set_error(lsb, NULL); - break; - case LUA_TSTRING: - lsb_set_error(lsb, lua_tostring(lua, 2)); - break; - default: - { - char err[LSB_ERROR_SIZE]; - int len = snprintf(err, LSB_ERROR_SIZE, - "%s() must return a nil or string error message", - func_name); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - break; - } - lua_pop(lua, 2); - - lsb_pcall_teardown(lsb); - - return status; -} - - -int report(lua_sandbox* lsb, double tc) -{ - static const char* func_name = "report"; - lua_State* lua = lsb_get_lua(lsb); - if (!lua) return 1; - - if (lsb_pcall_setup(lsb, func_name)) return 1; - - lua_pushnumber(lua, tc); - if (lua_pcall(lua, 1, 0, 0) != 0) { - char err[LSB_ERROR_SIZE]; - int len = snprintf(err, LSB_ERROR_SIZE, "%s() %s", func_name, - lua_tostring(lua, -1)); - if (len >= LSB_ERROR_SIZE || len < 0) { - err[LSB_ERROR_SIZE - 1] = 0; - } - lsb_terminate(lsb, err); - return 1; - } - - lsb_pcall_teardown(lsb); - lua_gc(lua, LUA_GCCOLLECT, 0); - - return 0; -} - - -int write_output(lua_State* lua) -{ - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - luaL_error(lua, "write_output() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - int n = lua_gettop(lua); - lsb_output(lsb, 1, n, 1); - written_data = lsb_get_output(lsb, &written_data_len); - return 0; -} - - -int write_message(lua_State* lua) -{ - void* luserdata = lua_touserdata(lua, lua_upvalueindex(1)); - if (NULL == luserdata) { - luaL_error(lua, "write_message() invalid lightuserdata"); - } - lua_sandbox* lsb = (lua_sandbox*)luserdata; - - if (lua_gettop(lua) != 1 || lua_type(lua, 1) != LUA_TTABLE) { - luaL_error(lua, "write_message() takes a table argument"); - } - - if (lsb_output_protobuf(lsb, 1, 0) != 0) { - luaL_error(lua, "write_message() could not encode protobuf - %s", - lsb_get_error(lsb)); - } - written_data = lsb_get_output(lsb, &written_data_len); - return 0; -} - - -static char* test_create_error() -{ - lua_sandbox* sb = lsb_create(NULL, NULL, "../../modules", 0, 0, 0); - mu_assert(!sb, "lsb_create() null lua_file"); - - return NULL; -} - - -static char* test_init_error() -{ - // null sandbox - int result = lsb_init(NULL, NULL); - mu_assert(result == 0, "lsb_init() null sandbox ptr"); - - // load error - lua_sandbox* sb = lsb_create(NULL, "lua/simple1.lua", "../../modules", 0, - 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, NULL); - mu_assert(result == 2, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_state s = lsb_get_state(sb); - mu_assert(s == LSB_TERMINATED, "lsb_get_state() received: %d", s); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // out of memory - sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 6000, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, NULL); - mu_assert(result == 2, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s\n", e); - - sb = lsb_create(NULL, "lua/lpeg_date_time.lua", NULL, 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - // disabled external modules - result = lsb_init(sb, NULL); - mu_assert(result == 2, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - const char* expected = "lua/lpeg_date_time.lua:7: require_library() external modules are disabled"; - mu_assert(strcmp(lsb_get_error(sb), expected) == 0, - "lsb_get_error() received: %s", lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_destroy_error() -{ - const char* expected = "preserve_global_data could not open: " - "invaliddir/simple.preserve"; - e = lsb_destroy(NULL, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, "invaliddir/simple.preserve"); - mu_assert(e, "lsb_destroy() received NULL"); - mu_assert(strcmp(e, expected) == 0, - "lsb_destroy() received: %s", e); - free(e); - e = NULL; - - return NULL; -} - - -static char* test_usage_error() -{ - unsigned u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "NULL sandbox memory usage received: %u", u); - - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - u = lsb_usage(NULL, LSB_UT_MAX + 1, LSB_US_CURRENT); - mu_assert(u == 0, "Invalid usage type received: %u", u); - - u = lsb_usage(NULL, LSB_UT_MEMORY, LSB_US_MAX + 1); - mu_assert(u == 0, "Invalid usage stat received: %u", u); - - mu_assert(sb, "lsb_create() received: NULL"); - lsb_terminate(sb, "forced termination"); - u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u == 0, "Terminated memory usage received: %u", u); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_misc() -{ - lsb_state s = lsb_get_state(NULL); - mu_assert(s == LSB_UNKNOWN, "lsb_get_state() received: %d", s); - - const char* le = lsb_get_error(NULL); - mu_assert(strlen(le) == 0, "lsb_get_error() received: %s", le); - - return NULL; -} - - -static char* test_simple() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", - 65765, 1000, 1024); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - unsigned u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_CURRENT); - mu_assert(u > 0, "Current memory usage received: %u", u); - printf("cur_mem %u\n", u); - - u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_MAXIMUM); - mu_assert(u > 0, "Maximum memory usage received: %u", u); - printf("max_mem %u\n", u); - - u = lsb_usage(sb, LSB_UT_MEMORY, LSB_US_LIMIT); - mu_assert(u == 65765, "Memory limit received: %u", u); - - u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_CURRENT); - mu_assert(u == 7, "Current instructions received: %u", u); - - u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_MAXIMUM); - mu_assert(u == 7, "Maximum instructions received: %u", u); - printf("max_ins %u\n", u); - - u = lsb_usage(sb, LSB_UT_INSTRUCTION, LSB_US_LIMIT); - mu_assert(u == 1000, "Instruction limit received: %u", u); - - u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_CURRENT); - mu_assert(u == 0, "Current output received: %u", u); - - u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_MAXIMUM); - mu_assert(u == 0, "Maximum output received: %u", u); - printf("max_out %u\n", u); - - u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); - mu_assert(u == 1024, "Output limit received: %u", u); - - u = lsb_usage(sb, LSB_UT_OUTPUT, LSB_US_LIMIT); - mu_assert(u == 1024, "Output limit received: %u", u); - - lsb_state s = lsb_get_state(sb); - mu_assert(s == LSB_RUNNING, "lsb_get_state() received: %d", s); - - e = lsb_destroy(sb, "simple.preserve"); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - -static char* test_simple_error() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/simple.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 1); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - mu_assert(strcmp("ok", lsb_get_error(sb)) == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - mu_assert(strcmp("", lsb_get_error(sb)) == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 2); - mu_assert(result == 1, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_output() -{ - const char* outputs[] = { - "1.2 string nil true false" - , "" - , "\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65" - , "\x10\x80\x94\xeb\xdc\x03\x52\x12\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f" - , "\x10\x80\x94\xeb\xdc\x03\x52\x24\x0a\x06\x63\x6f\x75\x6e\x74\x73\x10\x03\x3a\x18\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x00\x00\x00\x00\x00\x00\x10\x40" - , "\x10\x80\x94\xeb\xdc\x03\x52\x19\x0a\x05\x63\x6f\x75\x6e\x74\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x39\x00\x00\x00\x00\x00\x00\x14\x40" - , "\x10\x80\x94\xeb\xdc\x03\x52\x2b\x0a\x06\x63\x6f\x75\x6e\x74\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\x18\x40\x00\x00\x00\x00\x00\x00\x1c\x40\x00\x00\x00\x00\x00\x00\x20\x40" -#ifdef LUA_JIT - , "\x10\x80\x94\xeb\xdc\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33" -#else - , "\x10\x80\x94\xeb\xdc\x03\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33" -#endif - , "\x10\x80\x94\xeb\xdc\x03\x52\x8d\x01\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x82\x01\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39" - , NULL - }; - - enum {output_size = 63 * 1024}; - - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", 0, 0 - , output_size); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - lsb_add_function(sb, &write_message, "write_message"); - - for (int x = 0; outputs[x]; ++x) { - result = process(sb, x); - mu_assert(!result, "process() test: %d failed: %d %s", x, result, - lsb_get_error(sb)); - if (outputs[x][0]) { - if (outputs[x][0] == 0x10) { - size_t header = 18; - if (memcmp(outputs[x], written_data + header, written_data_len - header) - != 0) { - char hex_data[output_size + 1]; - size_t z = 0; - for (size_t y = header; y < written_data_len; ++y, z += 3) { - snprintf(hex_data + z, output_size - z, "%02x ", - (unsigned char)written_data[y]); - } - hex_data[z] = 0; - mu_assert(0, "test: %d received: %s", x, hex_data); - } - } else { - mu_assert(strcmp(outputs[x], written_data) == 0, - "test: %d received: %s", x, written_data); - } - } - } - - e = lsb_destroy(sb, "circular_buffer.preserve"); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_output_errors() -{ - const char* tests[] = - { - "process() lua/output_errors.lua:10: bad argument #1 to 'output' (unsuported type)" - , "process() lua/output_errors.lua:16: output_limit exceeded" - , "process() lua/output_errors.lua:19: write_message() could not encode protobuf - array has mixed types" - , "process() lua/output_errors.lua:22: write_message() could not encode protobuf - unsupported type: nil" - , "process() lua/output_errors.lua:24: bad argument #1 to 'write_output' (unknown userdata type)" - , "process() lua/output_errors.lua:27: output_limit exceeded" - , "process() lua/output_errors.lua:30: write_message() could not encode protobuf - unsupported array type: table" - , "process() lua/output_errors.lua:36: strbuf max_size exceeded" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/output_errors.lua", "../../modules", - 0, 0, 128); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - lsb_add_function(sb, &write_message, "write_message"); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - -static char* test_cbuf_errors() -{ - const char* tests[] = - { - "process() lua/circular_buffer_errors.lua:9: bad argument #0 to 'new' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:11: bad argument #1 to 'new' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:13: bad argument #1 to 'new' (rows must be > 1)" - , "process() lua/circular_buffer_errors.lua:15: bad argument #2 to 'new' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:17: bad argument #2 to 'new' (columns must be > 0)" - , "process() lua/circular_buffer_errors.lua:19: bad argument #3 to 'new' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:21: bad argument #3 to 'new' (seconds_per_row is out of range)" - , "process() lua/circular_buffer_errors.lua:23: removed test" - , "process() not enough memory" - , "process() lua/circular_buffer_errors.lua:28: bad argument #2 to 'set' (column out of range)" - , "process() lua/circular_buffer_errors.lua:31: bad argument #2 to 'set' (column out of range)" - , "process() lua/circular_buffer_errors.lua:34: bad argument #2 to 'set' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:37: bad argument #1 to 'set' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:41: bad argument #1 to 'get' (lsb.circular_buffer expected, got number)" - , "process() lua/circular_buffer_errors.lua:44: bad argument #3 to 'set' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:47: bad argument #-1 to 'set' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:50: bad argument #-1 to 'add' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:53: bad argument #-1 to 'get' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:56: bad argument #-1 to 'compute' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:59: bad argument #1 to 'compute' (invalid option 'func')" - , "process() lua/circular_buffer_errors.lua:62: bad argument #2 to 'compute' (column out of range)" - , "process() lua/circular_buffer_errors.lua:65: bad argument #4 to 'compute' (end must be >= start)" - , "process() lua/circular_buffer_errors.lua:68: bad argument #1 to 'format' (invalid option 'invalid')" - , "process() lua/circular_buffer_errors.lua:71: bad argument #-1 to 'format' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:74: bad argument #-1 to 'format' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:77: fromstring() too few values: 0, expected 2" - , "process() lua/circular_buffer_errors.lua:80: fromstring() too few values: 0, expected 2" - , "process() lua/circular_buffer_errors.lua:83: fromstring() too many values, more than: 2" - , "process() lua/circular_buffer_errors.lua:86: bad argument #-1 to 'mannwhitneyu' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:89: bad argument #1 to 'mannwhitneyu' (number expected, got nil)" - , "process() lua/circular_buffer_errors.lua:92: bad argument #1 to 'mannwhitneyu' (column out of range)" - , "process() lua/circular_buffer_errors.lua:95: bad argument #3 to 'mannwhitneyu' (ranges must not overlap)" - , "process() lua/circular_buffer_errors.lua:98: bad argument #3 to 'mannwhitneyu' (end_1 must be >= start_1)" - , "process() lua/circular_buffer_errors.lua:101: bad argument #5 to 'mannwhitneyu' (end_2 must be >= start_2)" - , "process() lua/circular_buffer_errors.lua:104: bad argument #-1 to 'mannwhitneyu' (too many arguments)" - , "process() lua/circular_buffer_errors.lua:107: bad argument #6 to 'mannwhitneyu' (use_continuity must be a boolean)" - , "process() lua/circular_buffer_errors.lua:110: bad argument #-1 to 'get_header' (incorrect number of arguments)" - , "process() lua/circular_buffer_errors.lua:113: bad argument #1 to 'get_header' (column out of range)" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer_errors.lua", - "../../modules", 32767, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - -static char* test_cbuf() -{ - const char* outputs[] = { - "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\tnan\tnan\nnan\tnan\tnan\nnan\tnan\tnan\n" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" - , "{\"time\":2,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n3\t1\t3\nnan\tnan\tnan\n1\t1\t1\n" - , "{\"time\":8,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\nnan\tnan\tnan\nnan\tnan\tnan\n1\t1\t1\n" - , NULL - }; - - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, "error %s", - lsb_get_error(sb)); - mu_assert(strcmp(outputs[0], written_data) == 0, "received: %s", - written_data); - - process(sb, 0); - process(sb, 1e9); - process(sb, 1e9); - process(sb, 2e9); - process(sb, 2e9); - process(sb, 2e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[1], written_data) == 0, "received: %s", - written_data); - - process(sb, 4e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[2], written_data) == 0, "received: %s", - written_data); - - process(sb, 10e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[3], written_data) == 0, "received: %s", - written_data); - - for (int i = 1; i < 18; ++i) { - result = report(sb, i); - mu_assert(result == 0, "report() test: %d received: %d error: %s", i, - result, lsb_get_error(sb)); - } - - e = lsb_destroy(sb, "circular_buffer.preserve"); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_cbuf_delta() -{ - const char* outputs[] = { - "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" -#ifdef LUA_JIT - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n0\t1\t1\t1\n1\t2\t1\t2\n2\t3\t1\t3\n" -#else - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t2\t1\t2\n2\t3\t1\t3\n0\t1\t1\t1\n" -#endif - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n1\t1\t1\n2\t1\t2\n3\t1\t3\n" - , "" - , "{\"time\":0,\"rows\":3,\"columns\":3,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Add_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Set_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Get_column\",\"unit\":\"count\",\"aggregation\":\"sum\"}]}\n2\tnan\tnan\tnan\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t2\t5\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t3\t8\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\tnan\t9\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\tnan\t10\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t1\tnan\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t1\tnan\n" - , "{\"time\":0,\"rows\":2,\"columns\":2,\"seconds_per_row\":1,\"column_info\":[{\"name\":\"Sum_column\",\"unit\":\"count\",\"aggregation\":\"sum\"},{\"name\":\"Min\",\"unit\":\"count\",\"aggregation\":\"min\"}]}\n0\t-2\t0\n" - , "" - , NULL - }; - - lua_sandbox* sb = lsb_create(NULL, "lua/circular_buffer_delta.lua", - "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - process(sb, 0); - process(sb, 1e9); - process(sb, 1e9); - process(sb, 2e9); - process(sb, 2e9); - process(sb, 2e9); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[0], written_data) == 0, "received: %s", - written_data); - - result = report(sb, 1); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(outputs[1], written_data) == 0, "received: %s", - written_data); - - for (int i = 2; outputs[i] != NULL; ++i) { - result = report(sb, i - 2); - mu_assert(result == 0, "report() received: %d error: %s", result, - lsb_get_error(sb)); - mu_assert(strcmp(outputs[i], written_data) == 0, "test: %d received: %s", i, - written_data); - } - - e = lsb_destroy(sb, "circular_buffer_delta.preserve"); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_cjson() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/cjson.lua", "../../modules", 0, 0, 64); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_cjson_unlimited() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/cjson_unlimited.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - mu_assert(written_data_len == 103001, "received %d bytes", (int)written_data_len); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_errors() -{ - const char* tests[] = { -#ifdef _WIN32 - "process() lua/errors.lua:9: cannot open lua\\unknown.lua: No such file or directory" -#else - "process() lua/errors.lua:9: cannot open lua/unknown.lua: No such file or directory" -#endif - , "process() lua/errors.lua:11: bad argument #0 to 'output' (must have at least one argument)" - , "process() not enough memory" - , "process() instruction_limit exceeded" - , "process() lua/errors.lua:20: attempt to perform arithmetic on global 'x' (a nil value)" - , "process() must return a single numeric value" - , "process() must return a single numeric value" - , "process() lua/errors.lua:27: output_limit exceeded" -#ifdef _WIN32 - , "process() lua/errors.lua:30: lua\\bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" -#else - , "process() lua/errors.lua:30: lua/bad_module.lua:1: attempt to perform arithmetic on global 'nilvalue' (a nil value)" -#endif - , "process() lua/errors.lua:32: invalid module name '../invalid'" - , "process() lua/errors.lua:34: require_path exceeded 260" - , "process() lua/errors.lua:37: package table is missing" - , "process() lua/errors.lua:40: package.loaded table is missing" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/errors.lua", "lua", 32767, 1000, - 128); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - -static char* test_lpeg() -{ - const char* tests[] = { - "lua/lpeg.lua" - , "lua/lpeg_clf.lua" - , "lua/lpeg_cbufd.lua" - , "lua/lpeg_date_time.lua" - , "lua/lpeg_ip_address.lua" - , "lua/lpeg_mysql.lua" - , "lua/lpeg_syslog.lua" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, tests[i], "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - - return NULL; -} - - -static char* test_util() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/util_test.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_serialize() -{ - const char* output_file = "serialize.preserve"; - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "../../modules", - 64000, 1000, 64000); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - -#ifdef LUA_JIT - char* expected = read_file("output/serialize.data"); -#else - char* expected = read_file("output/serialize.lua51.data"); -#endif - char* actual = read_file(output_file); - mu_assert(strcmp(expected, actual) == 0, "serialization mismatch"); - free(expected); - free(actual); - - return NULL; -} - - -static char* test_restore() -{ - const char* output_file = "restore.preserve"; - - lua_sandbox* sb = lsb_create(NULL, "lua/restore.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - mu_assert(strcmp("101", written_data) == 0, "test: initial load received: %s", - written_data); - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/restore.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - mu_assert(strcmp("102", written_data) == 0, "test: reload received: %s", - written_data); - result = report(sb, 2); // change the preservation version - mu_assert(result == 0, "report() received: %d", result); - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data with a version change - sb = lsb_create(NULL, "lua/restore.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - mu_assert(strcmp("101", written_data) == 0, - "test: reload with version change received: %s", written_data); - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_serialize_failure() -{ - const char* output_file = "serialize_failure.preserve"; - const char* expected = "serialize_data cannot preserve type 'function'"; - - lua_sandbox* sb = lsb_create(NULL, - "lua/serialize_failure.lua", "../../modules", - 32767, 1000, 1024); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); - mu_assert(e, "lsb_destroy() received: no error"); - mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); - free(e); - e = NULL; - mu_assert(file_exists(output_file) == 0, "output file was not cleaned up"); - - return NULL; -} - - -static char* test_serialize_noglobal() -{ - const char* output_file = "serialize_noglobal.preserve"; - const char* expected = "preserve_global_data cannot access the global table"; - - lua_sandbox* sb = lsb_create(NULL, - "lua/serialize_noglobal.lua", "../../modules", - 32767, 1000, 1024); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); - mu_assert(e, "lsb_destroy() received: no error"); - mu_assert(strcmp(e, expected) == 0, "lsb_destroy() received: %s", e); - free(e); - e = NULL; - mu_assert(file_exists(output_file) == 0, "output file was not cleaned up"); - - return NULL; -} - - -static char* test_bloom_filter_errors() -{ - const char* tests[] = - { - "process() lua/bloom_filter_errors.lua:9: bad argument #0 to 'new' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:11: bad argument #1 to 'new' (number expected, got nil)" - , "process() lua/bloom_filter_errors.lua:13: bad argument #1 to 'new' (items must be > 1)" - , "process() lua/bloom_filter_errors.lua:15: bad argument #2 to 'new' (number expected, got nil)" - , "process() lua/bloom_filter_errors.lua:17: bad argument #2 to 'new' (probability must be between 0 and 1)" - , "process() lua/bloom_filter_errors.lua:19: bad argument #2 to 'new' (probability must be between 0 and 1)" - , "process() lua/bloom_filter_errors.lua:22: bad argument #-1 to 'add' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:25: bad argument #1 to 'add' (must be a string or number)" - , "process() lua/bloom_filter_errors.lua:28: bad argument #-1 to 'query' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:31: bad argument #1 to 'query' (must be a string or number)" - , "process() lua/bloom_filter_errors.lua:34: bad argument #-1 to 'clear' (incorrect number of arguments)" - , "process() lua/bloom_filter_errors.lua:37: bad argument #1 to 'fromstring' (string expected, got table)" - , "process() lua/bloom_filter_errors.lua:40: fromstring() bytes found: 23, expected 24" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, "lua/bloom_filter_errors.lua", - "../../modules", 32767, 1000, 128); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - -static char* test_bloom_filter() -{ - const char* output_file = "bloom_filter.preserve"; - const char* tests[] = { - "1" - , "2" - , "3" - , NULL - }; - - lua_sandbox* sb = lsb_create(NULL, "lua/bloom_filter.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - int i = 0; - for (; tests[i]; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i], written_data) == 0, "test: %d received: %s", i, - written_data); - } - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, - written_data); // count should remain the same - - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/bloom_filter.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - for (int i = 0; tests[i]; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp(tests[i - 1], written_data) == 0, "test: %d received: %s", i, - written_data); // count should remain the same - - // test clear - report(sb, 99); - process(sb, 0); - report(sb, 0); - mu_assert(strcmp("1", written_data) == 0, "test: clear received: %s", - written_data); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_hyperloglog_errors() -{ - const char* tests[] = - { - "process() lua/hyperloglog_errors.lua:9: bad argument #0 to 'new' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:12: bad argument #1 to 'add' (must be a string or number)" - , "process() lua/hyperloglog_errors.lua:15: bad argument #-1 to 'add' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:18: bad argument #-1 to 'count' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:21: bad argument #-1 to 'clear' (incorrect number of arguments)" - , "process() lua/hyperloglog_errors.lua:24: bad argument #1 to 'fromstring' (string expected, got table)" - , "process() lua/hyperloglog_errors.lua:27: fromstring() bytes found: 23, expected 12304" - , NULL - }; - - for (int i = 0; tests[i]; ++i) { - lua_sandbox* sb = lsb_create(NULL, - "lua/hyperloglog_errors.lua", "../../modules", - 32767, 1000, 128); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, i); - mu_assert(result == 1, "test: %d received: %d", i, result); - - const char* le = lsb_get_error(sb); - mu_assert(le, "test: %d received NULL", i); - mu_assert(strcmp(tests[i], le) == 0, "test: %d received: %s", i, le); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - - return NULL; -} - - -static char* test_hyperloglog() -{ - const char* output_file = "hyperloglog.preserve"; - - lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - - for (int i = 0; i < 100000; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, "test: initial received: %s", - written_data); // count should remain the same - - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, "test: cache received: %s", - written_data); // count should remain the same - - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - - // re-load to test the preserved data - sb = lsb_create(NULL, "lua/hyperloglog.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - result = lsb_init(sb, output_file); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, "test: reload received: %s", - written_data); // count should remain the same - - for (int i = 0; i < 100000; ++i) { - result = process(sb, i); - mu_assert(result == 0, "process() received: %d %s", result, - lsb_get_error(sb)); - } - result = report(sb, 0); - mu_assert(result == 0, "report() received: %d", result); - mu_assert(strcmp("100070", written_data) == 0, - "test: data replay received: %s", written_data); - // count should remain the same - - // test clear - report(sb, 99); - report(sb, 0); - mu_assert(strcmp("0", written_data) == 0, "test: clear received: %s", - written_data); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* test_struct() -{ - lua_sandbox* sb = lsb_create(NULL, "lua/struct.lua", "../../modules", - 65765, 1000, 1024); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - result = process(sb, 0); - mu_assert(result == 0, "process() received: %d %s", result, lsb_get_error(sb)); - - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - - return NULL; -} - - -static char* benchmark_counter() -{ - int iter = 10000000; - - lua_sandbox* sb = lsb_create(NULL, "lua/counter.lua", "../../modules", 32000, - 10, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - process(sb, 0); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_counter() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_serialize() -{ - int iter = 1000; - const char* output_file = "serialize.preserve"; - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "../../modules", - 64000, 1000, 1024); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, output_file); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - t = clock() - t; - printf("benchmark_serialize() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_deserialize() -{ - int iter = 1000; - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - lua_sandbox* sb = lsb_create(NULL, "lua/serialize.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - - int result = lsb_init(sb, "output/serialize.data"); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - } - t = clock() - t; - printf("benchmark_deserialize() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_lpeg_decoder() -{ - int iter = 10000; - - lua_sandbox* sb = lsb_create(NULL, "lua/decoder.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - lsb_add_function(sb, &write_message, "write_message"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_lpeg_decoder() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_lua_types_output() -{ - int iter = 1000000; - - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", - 100000, 1000, 1024 * 63); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_lua_types_output() %g seconds\n", ((float)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - -static char* benchmark_cbuf_output() -{ - int iter = 10000; - - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 1), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cbuf_output() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_message_output() -{ - int iter = 1000000; - - lua_sandbox* sb = lsb_create(NULL, "lua/output.lua", "../../modules", 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_message, "write_message"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 7), "%s", lsb_get_error(sb)); - } - t = clock() - t; - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_message_output() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_cbuf_add() -{ - int iter = 1000000; - - lua_sandbox* sb = lsb_create(NULL, - "lua/circular_buffer_add.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - - double ts = 0; - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, 0), "%s", lsb_get_error(sb)); - ts += 1e9; - } - t = clock() - t; - mu_assert(lsb_get_state(sb) == LSB_RUNNING, "benchmark_cbuf_add() failed %s", - lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_cbuf_add() %g seconds\n", ((float)t) / CLOCKS_PER_SEC - / iter); - - return NULL; -} - - -static char* benchmark_bloom_filter_add() -{ - int iter = 1000000; - - lua_sandbox* sb = lsb_create(NULL, - "lua/bloom_filter_benchmark.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - report(sb, 0); - mu_assert(strcmp("1000000", written_data) == 0, "received: %s", written_data); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_bloom_filter_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_bloom_filter_add() %g seconds\n", ((float)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - -static char* benchmark_hyperloglog_add() -{ - int iter = 1000000; - - lua_sandbox* sb = lsb_create(NULL, "lua/hyperloglog.lua", "../../modules", - 0, 0, 0); - mu_assert(sb, "lsb_create() received: NULL"); - int result = lsb_init(sb, NULL); - mu_assert(result == 0, "lsb_init() received: %d %s", result, - lsb_get_error(sb)); - lsb_add_function(sb, &write_output, "write_output"); - - clock_t t = clock(); - for (int x = 0; x < iter; ++x) { - mu_assert(0 == process(sb, x), "%s", lsb_get_error(sb)); // test add speed - } - t = clock() - t; - report(sb, 0); - mu_assert(strcmp("1006268", written_data) == 0, "received: %s", written_data); - mu_assert(lsb_get_state(sb) == LSB_RUNNING, - "benchmark_hyperloglog_add() failed %s", lsb_get_error(sb)); - e = lsb_destroy(sb, NULL); - mu_assert(!e, "lsb_destroy() received: %s", e); - printf("benchmark_hyperloglog_add() %g seconds\n", ((float)t) - / CLOCKS_PER_SEC / iter); - - return NULL; -} - - -static char* all_tests() -{ - - mu_run_test(test_create_error); - mu_run_test(test_init_error); - mu_run_test(test_destroy_error); - mu_run_test(test_usage_error); - mu_run_test(test_misc); - mu_run_test(test_simple); - mu_run_test(test_simple_error); - mu_run_test(test_output); - mu_run_test(test_output_errors); - mu_run_test(test_cbuf_errors); - mu_run_test(test_cbuf); - mu_run_test(test_cbuf_delta); - mu_run_test(test_cjson); - mu_run_test(test_cjson_unlimited); - mu_run_test(test_errors); - mu_run_test(test_lpeg); - mu_run_test(test_util); - mu_run_test(test_serialize); - mu_run_test(test_restore); - mu_run_test(test_serialize_failure); - mu_run_test(test_serialize_noglobal); - mu_run_test(test_bloom_filter_errors); - mu_run_test(test_bloom_filter); - mu_run_test(test_hyperloglog_errors); - mu_run_test(test_hyperloglog); - mu_run_test(test_struct); - - mu_run_test(benchmark_counter); - mu_run_test(benchmark_serialize); - mu_run_test(benchmark_deserialize); - mu_run_test(benchmark_lpeg_decoder); - mu_run_test(benchmark_lua_types_output); - mu_run_test(benchmark_cbuf_output); - mu_run_test(benchmark_message_output); - mu_run_test(benchmark_cbuf_add); - mu_run_test(benchmark_bloom_filter_add); - mu_run_test(benchmark_hyperloglog_add); - - return NULL; -} - - -int main() -{ - char* result = all_tests(); - if (result) { - printf("%s\n", result); - } else { - printf("ALL TESTS PASSED\n"); - } - printf("Tests run: %d\n", mu_tests_run); - free(e); - - return result != 0; -} diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt new file mode 100644 index 0000000..e0df018 --- /dev/null +++ b/src/util/CMakeLists.txt @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set(UTIL_SRC +heka_message.c +heka_message_matcher.c +heka_message_matcher_parser.c +input_buffer.c +output_buffer.c +protobuf.c +running_stats.c +string.c +string_matcher.c +util.c +) + +add_library(luasandboxutil SHARED ${UTIL_SRC}) +set_target_properties(luasandboxutil PROPERTIES VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} SOVERSION 0) +target_compile_definitions(luasandboxutil PRIVATE -Dluasandboxutil_EXPORTS) + +if(LIBM_LIBRARY) + target_link_libraries(luasandboxutil ${LIBM_LIBRARY}) +endif() + +install(TARGETS luasandboxutil DESTINATION ${CMAKE_INSTALL_LIBDIR}) +add_subdirectory(test) diff --git a/src/util/heka_message.c b/src/util/heka_message.c new file mode 100644 index 0000000..6fc8321 --- /dev/null +++ b/src/util/heka_message.c @@ -0,0 +1,589 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight Heka message implementation @file */ + +#include "luasandbox/util/heka_message.h" + +#include +#include +#include +#include + +#include "../luasandbox_defines.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox/util/protobuf.h" + +static size_t decode_header(char *buf, + size_t len, + size_t max_message_size, + lsb_logger *logger) +{ + if (*buf != 0x08) { + return 0; + } + + char *p = buf; + if (p && p < buf + len - 1) { + long long vi; + if (lsb_pb_read_varint(p + 1, buf + len, &vi)) { + if (vi > 0 && vi <= (long long)max_message_size) { + return (size_t)vi; + } else { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, + "maximum (%lld) messages size exceeded: %lld", + (long long)max_message_size, vi); + } + } + } + } + return 0; +} + + +static const char* +read_string(int wiretype, const char *p, const char *e, lsb_const_string *s) +{ + if (wiretype != LSB_PB_WT_LENGTH) { + return NULL; + } + + long long vi; + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || vi > e - p) { + return NULL; + } + s->s = p; + s->len = (size_t)vi; + return p + vi; +} + + +static bool +read_string_value(const char *p, const char *e, int ai, lsb_read_value *val) +{ + int acnt = 0; + int tag = 0; + int wiretype = 0; + while (p && p < e) { + val->type = LSB_READ_NIL; + p = lsb_pb_read_key(p, &tag, &wiretype); + p = read_string(wiretype, p, e, &val->u.s); + if (p) { + if (ai == acnt++) { + val->type = LSB_READ_STRING; + return true; + } + } + } + return false; +} + + +static bool +read_integer_value(const char *p, const char *e, int ai, lsb_read_value *val) +{ + int acnt = 0; + long long ll = 0; + while (p && p < e) { + p = lsb_pb_read_varint(p, e, &ll); + if (p) { + if (ai == acnt++) { + val->type = LSB_READ_NUMERIC; + val->u.d = (double)ll; + return true; + } + } + } + return false; +} + + +static bool +read_double_value(const char *p, const char *e, int ai, lsb_read_value *val) +{ + if (p + (sizeof(double) * (ai + 1)) > e) { + return false; + } + val->type = LSB_READ_NUMERIC; + p += sizeof(double) * ai; + memcpy(&val->u.d, p, sizeof(double)); + return true; +} + + +static const char* +process_varint(int wiretype, const char *p, const char *e, long long *val) +{ + if (wiretype != 0) { + return NULL; + } + p = lsb_pb_read_varint(p, e, val); + return p ? p : NULL; +} + + +static const char* +process_fields(lsb_heka_field *f, const char *p, const char *e) +{ + int tag = 0; + int wiretype = 0; + long long vi = 0; + + memset(f, 0, sizeof(lsb_heka_field)); + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || vi > e - p) { + return NULL; + } + e = p + vi; // only process to the end of the current field record + + do { + p = lsb_pb_read_key(p, &tag, &wiretype); + + switch (tag) { + case LSB_PB_NAME: + p = read_string(wiretype, p, e, &f->name); + break; + + case LSB_PB_VALUE_TYPE: + p = process_varint(wiretype, p, e, &vi); + if (p) { + f->value_type = (int)vi; + } + break; + + case LSB_PB_REPRESENTATION: + p = read_string(wiretype, p, e, &f->representation); + break; + + // don't bother with the value(s) until we actually need them + // since this stream is created by Hindsight + // - tags are guaranteed to be properly ordered (values at the end) + // - there won't be repeated tags for packed values + case LSB_PB_VALUE_STRING: + case LSB_PB_VALUE_BYTES: + if (wiretype != 2) { + p = NULL; + break; + } + f->value.s = p - 1; + f->value.len = e - f->value.s; + p = e; + break; + + case LSB_PB_VALUE_INTEGER: + case LSB_PB_VALUE_BOOL: + if (wiretype != 0 && wiretype != 2) { + p = NULL; + break; + } + // fall thru + case LSB_PB_VALUE_DOUBLE: + if (tag == 7 && wiretype != 1 && wiretype != 2) { + p = NULL; + break; + } + if (wiretype == 2) { + p = lsb_pb_read_varint(p, e, &vi); + if (!p || vi < 0 || vi > e - p) { + p = NULL; + break; + } + } + f->value.s = p; + f->value.len = e - f->value.s; + p = e; + break; + + default: + p = NULL; // don't allow unknown tags + break; + } + } while (p && p < e); + + return p && f->name.s ? p : NULL; +} + + +bool lsb_decode_heka_message(lsb_heka_message *m, + const char *buf, + size_t len, + lsb_logger *logger) +{ + if (!m || !buf || len == 0) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "%s", LSB_ERR_UTIL_NULL); + } + return false; + } + + const char *cp = buf; // current position + const char *lp = buf; // last position + const char *ep = buf + len; // end position + int wiretype = 0; + int tag = 0; + long long val = 0; + bool timestamp = false; + + lsb_clear_heka_message(m); + + do { + cp = lsb_pb_read_key(cp, &tag, &wiretype); + + switch (tag) { + case LSB_PB_UUID: + cp = read_string(wiretype, cp, ep, &m->uuid); + if (m->uuid.len != LSB_UUID_SIZE) cp = NULL; + break; + + case LSB_PB_TIMESTAMP: + cp = process_varint(wiretype, cp, ep, &m->timestamp); + if (cp) timestamp = true; + break; + + case LSB_PB_TYPE: + cp = read_string(wiretype, cp, ep, &m->type); + break; + + case LSB_PB_LOGGER: + cp = read_string(wiretype, cp, ep, &m->logger); + break; + + case LSB_PB_SEVERITY: + cp = process_varint(wiretype, cp, ep, &val); + if (cp) m->severity = (int)val; + break; + + case LSB_PB_PAYLOAD: + cp = read_string(wiretype, cp, ep, &m->payload); + break; + + case LSB_PB_ENV_VERSION: + cp = read_string(wiretype, cp, ep, &m->env_version); + break; + + case LSB_PB_PID: + cp = process_varint(wiretype, cp, ep, &val); + if (cp) m->pid = (int)val; + break; + + case LSB_PB_HOSTNAME: + cp = read_string(wiretype, cp, ep, &m->hostname); + break; + + case LSB_PB_FIELDS: + if (wiretype != 2) { + cp = NULL; + break; + } + if (m->fields_len == m->fields_size) { + int step = 8; + m->fields_size += step; + lsb_heka_field *tmp = realloc(m->fields, + m->fields_size * sizeof(lsb_heka_field)); + if (!tmp) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 0, + "fields reallocation failed"); + } + return false; + } + // the new memory will be initialized as needed Issue #231 + m->fields = tmp; + } + cp = process_fields(&m->fields[m->fields_len], cp, ep); + ++m->fields_len; + break; + + default: + cp = NULL; + break; + } + if (cp) lp = cp; + } while (cp && cp < ep); + + if (!cp) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "tag:%d wiretype:%d position:%d", + tag, wiretype, lp - buf); + } + return false; + } + + if (!m->uuid.s) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "%s", "missing " LSB_UUID); + } + return false; + } + + if (!timestamp) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "%s", "missing " LSB_TIMESTAMP); + } + return false; + } + + m->raw.s = buf; + m->raw.len = len; + return true; +} + + +bool lsb_find_heka_message(lsb_heka_message *m, + lsb_input_buffer *ib, + bool decode, + size_t *discarded_bytes, + lsb_logger *logger) +{ + if (!m || !ib || !discarded_bytes) { + if (logger && logger->cb) { + logger->cb(logger->context, __func__, 4, "%s", LSB_ERR_UTIL_NULL); + } + return false; + } + + *discarded_bytes = 0; + if (ib->readpos == ib->scanpos) { + return false; // empty buffer + } + + char *p = memchr(&ib->buf[ib->scanpos], 0x1e, ib->readpos - ib->scanpos); + if (p) { + if (p != ib->buf + ib->scanpos) { + // partial buffer skipped before locating a possible header + *discarded_bytes += p - ib->buf - ib->scanpos; + } + ib->scanpos = p - ib->buf; + + if (ib->readpos - ib->scanpos < 2) { + return false; // header length is not buf + } + + size_t hlen = (unsigned char)ib->buf[ib->scanpos + 1]; + size_t hend = ib->scanpos + hlen + 3; + if (hend > ib->readpos) { + return false; // header is not in buf + } + if (ib->buf[hend - 1] != 0x1f) { + // invalid header length + ++ib->scanpos; + ++*discarded_bytes; + size_t db; + bool b = lsb_find_heka_message(m, ib, decode, &db, logger); + *discarded_bytes += db; + return b; + } + + if (!ib->msglen) { + ib->msglen = decode_header(&ib->buf[ib->scanpos + 2], hlen, + ib->maxsize - LSB_MAX_HDR_SIZE, logger); + } + + if (ib->msglen) { + size_t mend = hend + ib->msglen; + if (mend > ib->readpos) { + return false; // message is not in buf + } + + if (decode) { + if (lsb_decode_heka_message(m, &ib->buf[hend], ib->msglen, logger)) { + ib->scanpos = mend; + ib->msglen = 0; + return true; + } else { + // message decode failure + ++ib->scanpos; + ++*discarded_bytes; + ib->msglen = 0; + size_t db; + bool b = lsb_find_heka_message(m, ib, decode, &db, logger); + *discarded_bytes += db; + return b; + } + } else { + // allow a framed message is non Heka protobuf format + lsb_clear_heka_message(m); + m->raw.s = &ib->buf[hend]; + m->raw.len = ib->msglen; + ib->scanpos = mend; + ib->msglen = 0; + return true; + } + } else { + // header decode failure + ++ib->scanpos; + ++*discarded_bytes; + size_t db; + bool b = lsb_find_heka_message(m, ib, decode, &db, logger); + *discarded_bytes += db; + return b; + } + } else { + // full buffer skipped since no header was located + *discarded_bytes += ib->readpos - ib->scanpos; + ib->scanpos = ib->readpos = 0; + } + return false; +} + + +lsb_err_value lsb_init_heka_message(lsb_heka_message *m, int num_fields) +{ + if (!m) return LSB_ERR_UTIL_NULL; + if (num_fields < 1) return LSB_ERR_UTIL_PRANGE; + + m->fields = malloc(num_fields * sizeof(lsb_heka_field)); + if (!m->fields) return LSB_ERR_UTIL_OOM; + + m->fields_size = num_fields; + lsb_clear_heka_message(m); + return NULL; +} + + +void lsb_clear_heka_message(lsb_heka_message *m) +{ + if (!m) return; + + lsb_init_const_string(&m->raw); + lsb_init_const_string(&m->uuid); + lsb_init_const_string(&m->type); + lsb_init_const_string(&m->logger); + lsb_init_const_string(&m->payload); + lsb_init_const_string(&m->env_version); + lsb_init_const_string(&m->hostname); + + // The fields will be cleared as they are built out anything beyond fields_len + // should be considered uninitialized Issue #231. + m->timestamp = 0; + m->severity = 7; + m->pid = INT_MIN; + m->fields_len = 0; +} + + +void lsb_free_heka_message(lsb_heka_message *m) +{ + if (!m) return; + + lsb_clear_heka_message(m); + free(m->fields); + m->fields = NULL; + m->fields_size = 0; +} + + +bool lsb_read_heka_field(const lsb_heka_message *m, + lsb_const_string *name, + int fi, + int ai, + lsb_read_value *val) +{ + if (!m || !name || !val) { + return false; + } + + int fcnt = 0; + const char *p, *e; + val->type = LSB_READ_NIL; + + for (int i = 0; i < m->fields_len; ++i) { + if (name->len == m->fields[i].name.len + && strncmp(name->s, m->fields[i].name.s, m->fields[i].name.len) == 0) { + if (fi == fcnt++) { + p = m->fields[i].value.s; + e = p + m->fields[i].value.len; + switch (m->fields[i].value_type) { + case LSB_PB_STRING: + case LSB_PB_BYTES: + return read_string_value(p, e, ai, val); + case LSB_PB_INTEGER: + return read_integer_value(p, e, ai, val); + case LSB_PB_BOOL: + if (read_integer_value(p, e, ai, val)) { + val->type = LSB_READ_BOOL; + return true; + } + return false; + case LSB_PB_DOUBLE: + return read_double_value(p, e, ai, val); + default: + return false; + } + } + } + } + return false; +} + + +lsb_err_value +lsb_write_heka_uuid(lsb_output_buffer *ob, const char *uuid, size_t len) +{ + if (!ob) { + return LSB_ERR_UTIL_NULL; + } + + static const size_t needed = 18; + ob->pos = 0; // writing a uuid will always clear the buffer as it is the + // start of a new message + lsb_err_value ret = lsb_expand_output_buffer(ob, needed); + if (ret) return ret; + + ob->buf[ob->pos++] = 2 | (LSB_PB_UUID << 3); // write key + ob->buf[ob->pos++] = LSB_UUID_SIZE; // write length + if (uuid && len == LSB_UUID_SIZE) { + memcpy(ob->buf + ob->pos, uuid, LSB_UUID_SIZE); + ob->pos += LSB_UUID_SIZE; + } else if (uuid && len == LSB_UUID_STR_SIZE) { + int cnt = sscanf(uuid, "%02hhx%02hhx%02hhx%02hhx" + "-%02hhx%02hhx" + "-%02hhx%02hhx" + "-%02hhx%02hhx" + "-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx", + (unsigned char *)ob->buf + ob->pos, + (unsigned char *)ob->buf + ob->pos + 1, + (unsigned char *)ob->buf + ob->pos + 2, + (unsigned char *)ob->buf + ob->pos + 3, + (unsigned char *)ob->buf + ob->pos + 4, + (unsigned char *)ob->buf + ob->pos + 5, + (unsigned char *)ob->buf + ob->pos + 6, + (unsigned char *)ob->buf + ob->pos + 7, + (unsigned char *)ob->buf + ob->pos + 8, + (unsigned char *)ob->buf + ob->pos + 9, + (unsigned char *)ob->buf + ob->pos + 10, + (unsigned char *)ob->buf + ob->pos + 11, + (unsigned char *)ob->buf + ob->pos + 12, + (unsigned char *)ob->buf + ob->pos + 13, + (unsigned char *)ob->buf + ob->pos + 14, + (unsigned char *)ob->buf + ob->pos + 15); + if (cnt == LSB_UUID_SIZE) { + ob->pos += cnt; + } + } + + if (ob->pos == 2) { // only the header has been written + for (int x = 0; x < LSB_UUID_SIZE; ++x) { + ob->buf[ob->pos++] = rand() % 256; + } + ob->buf[8] = (ob->buf[8] & 0x0F) | 0x40; + ob->buf[10] = (ob->buf[10] & 0x0F) | 0xA0; + } + return NULL; +} + + +size_t lsb_write_heka_header(char *buf, size_t len) +{ + int hlen = lsb_pb_output_varint(buf + 3, len) + 1; + buf[hlen + 2] = '\x1f'; + buf[0] = '\x1e'; + buf[1] = (char)hlen; + buf[2] = '\x08'; + return LSB_HDR_FRAME_SIZE + hlen; +} diff --git a/src/util/heka_message_matcher.c b/src/util/heka_message_matcher.c new file mode 100644 index 0000000..381b81a --- /dev/null +++ b/src/util/heka_message_matcher.c @@ -0,0 +1,228 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight/Heka message matcher implementation @file */ + +#include "heka_message_matcher_impl.h" + +#include +#include +#include +#include + +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" +#include "luasandbox/util/string.h" +#include "luasandbox/util/string_matcher.h" + + +static bool string_test(match_node *mn, lsb_const_string *val) +{ + const char *mn_val = mn->data + mn->var_len; + switch (mn->op) { + case OP_EQ: + if (val->len != mn->val_len || !val->s) return false; + return strncmp(val->s, mn_val, val->len) == 0; + case OP_NE: + if (val->len != mn->val_len || !val->s) return true; + return strncmp(val->s, mn_val, val->len) != 0; + case OP_LT: + { + if (!val->s) return true; + int cmp = strncmp(val->s, mn_val, val->len); + return cmp == 0 ? val->len < mn->val_len : cmp < 0; + } + case OP_LTE: + return val->s ? strncmp(val->s, mn_val, val->len) <= 0 : true; + case OP_GT: + { + if (!val->s) return false; + int cmp = strncmp(val->s, mn_val, val->len); + return cmp == 0 ? val->len > mn->val_len : cmp > 0; + } + case OP_GTE: + { + if (!val->s) return false; + int cmp = strncmp(val->s, mn_val, val->len); + return cmp == 0 ? val->len >= mn->val_len : cmp > 0; + } + case OP_RE: + if (mn->val_mod == PATTERN_MOD_ESC) { + return lsb_string_find(val->s, val->len, mn_val, mn->val_len); + } else { + return lsb_string_match(val->s, val->len, mn_val); + } + case OP_NRE: + if (mn->val_mod == PATTERN_MOD_ESC) { + return !lsb_string_find(val->s, val->len, mn_val, mn->val_len); + } else { + return !lsb_string_match(val->s, val->len, mn_val); + } + default: + break; + } + return false; +} + + +static bool numeric_test(match_node *mn, double val) +{ + double d = 0; + memcpy(&d, mn->data + mn->var_len, sizeof(double)); + switch (mn->op) { + case OP_EQ: + return val == d; + case OP_NE: + return val != d; + case OP_LT: + return val < d; + case OP_LTE: + return val <= d; + case OP_GT: + return val > d; + case OP_GTE: + return val >= d; + default: + break; + } + return false; +} + + +static bool eval_node(match_node *mn, lsb_heka_message *m) +{ + switch (mn->op) { + case OP_TRUE: + return true; + case OP_FALSE: + return false; + default: + switch (mn->field_id) { + case LSB_PB_TIMESTAMP: + return numeric_test(mn, (double)m->timestamp); + case LSB_PB_TYPE: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->type.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } + return string_test(mn, &m->type); + case LSB_PB_LOGGER: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->logger.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } + return string_test(mn, &m->logger); + case LSB_PB_SEVERITY: + return numeric_test(mn, m->severity); + case LSB_PB_PAYLOAD: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->payload.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } + return string_test(mn, &m->payload); + case LSB_PB_ENV_VERSION: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->env_version.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } + return string_test(mn, &m->env_version); + case LSB_PB_PID: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->pid == INT_MIN; + return mn->op == OP_EQ ? is_nil : !is_nil; + } + return numeric_test(mn, m->pid); + case LSB_PB_HOSTNAME: + if (mn->val_type == TYPE_NIL) { + bool is_nil = m->hostname.s == NULL; + return mn->op == OP_EQ ? is_nil : !is_nil; + } + return string_test(mn, &m->hostname); + case LSB_PB_UUID: + return string_test(mn, &m->uuid); + default: + { + lsb_read_value val; + lsb_const_string variable = { .s = mn->data, .len = mn->var_len }; + + if (!lsb_read_heka_field(m, &variable, mn->u.idx.f, mn->u.idx.a, + &val)) { + if (mn->val_type == TYPE_NIL) { + return mn->op == OP_EQ; + } + return false; + } + + switch (mn->val_type) { + case TYPE_STRING: + if (val.type == LSB_READ_STRING) { + return string_test(mn, &val.u.s); + } + break; + case TYPE_NUMERIC: + if (val.type == LSB_READ_NUMERIC) { + return numeric_test(mn, val.u.d); + } + break; + case TYPE_TRUE: + if (val.type == LSB_READ_BOOL || val.type == LSB_READ_NUMERIC) { + return mn->op == OP_EQ ? val.u.d == true : val.u.d != true; + } + break; + case TYPE_FALSE: + if (val.type == LSB_READ_BOOL || val.type == LSB_READ_NUMERIC) { + return mn->op == OP_EQ ? val.u.d == false: val.u.d != false; + } + break; + case TYPE_NIL: + return mn->op == OP_NE; + } + } + break; + } + break; + } + return false; +} + + +void lsb_destroy_message_matcher(lsb_message_matcher *mm) +{ + if (!mm) return; + free(mm->nodes); + free(mm); +} + + +bool lsb_eval_message_matcher(lsb_message_matcher *mm, lsb_heka_message *m) +{ + bool match = false; + if (!mm) return match; + + match_node *s = mm->nodes; + match_node *e = mm->nodes + (mm->bytes / sizeof(match_node)); + for (match_node *p = mm->nodes; p < e;) { + switch (p->op) { + case OP_OR: + if (match) { + p = s + p->u.off; // short circuit + continue; + } + break; + case OP_AND: + if (!match) { + p = s + p->u.off; // short circuit + continue; + } + break; + default: + match = eval_node(p, m); + break; + } + p += p->units; + } + return match; +} diff --git a/src/util/heka_message_matcher_impl.h b/src/util/heka_message_matcher_impl.h new file mode 100644 index 0000000..ebf3543 --- /dev/null +++ b/src/util/heka_message_matcher_impl.h @@ -0,0 +1,74 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Hindsight/Heka message matcher implementation header @file */ + +#ifndef luasandbox_util_heka_message_matcher_impl_h_ +#define luasandbox_util_heka_message_matcher_impl_h_ + +#include +#include + +typedef enum { + OP_EQ, + OP_NE, + OP_GTE, + OP_GT, + OP_LTE, + OP_LT, + OP_RE, + OP_NRE, + OP_TRUE, + OP_FALSE, + OP_OPEN, + OP_OR, + OP_AND +} match_operation; + + +typedef enum { + TYPE_NIL, + TYPE_STRING, + TYPE_NUMERIC, + TYPE_FALSE, + TYPE_TRUE, +} match_type; + + +typedef enum { + PATTERN_MOD_NONE, + PATTERN_MOD_ESC, +} match_pattern_mod; + + +struct indices { + uint8_t f; + uint8_t a; +}; + + +typedef struct match_node { + uint8_t op; + uint8_t units; + uint8_t var_len; + uint8_t val_len; + uint8_t field_id; + uint8_t val_type : 4; + uint8_t val_mod : 4; + union { + struct indices idx; + uint16_t off; + } u; + char data[]; // inlined field variable and value data (when necessary) +} match_node; + + +struct lsb_message_matcher { + size_t bytes; + match_node *nodes; +}; + +#endif diff --git a/src/util/heka_message_matcher_parser.c b/src/util/heka_message_matcher_parser.c new file mode 100644 index 0000000..d9d54ef --- /dev/null +++ b/src/util/heka_message_matcher_parser.c @@ -0,0 +1,2469 @@ +/* A recursive-descent parser generated by peg 0.1.18 */ + +#include +#include +#include +#define YYRULECOUNT 56 +#line 1 "../src/util/heka_message_matcher_parser.leg" + +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight message matcher parser @file */ + +#include +#include +#include +#include +#include + +#include "heka_message_matcher_impl.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" + +#ifndef _MSC_VER +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-function" +#else +#pragma warning( disable : 4267 4244 ) +#endif + +typedef struct match_node_tmp { + uint8_t id; + uint8_t op; + uint8_t val_mod; + uint8_t val_type; + uint8_t val_len; + uint8_t var_len; + uint8_t fi; // left node index for logical op + uint8_t ai; // right node index for logical op + char *var; + + union { + char *s; + double d; + } val; +} match_node_tmp; + + +typedef struct match_node_array { + match_node_tmp *a; + int pos; + int size; +} match_node_array; + + +typedef struct input_string { + const char *s; + size_t pos; + size_t size; +} input_string; + + +typedef struct context { + match_node_array out; + match_node_array ops; + match_node_tmp mn; + struct tm tms; + int cond_cnt; + input_string is; +} context; + + +#define YY_PARSE(T) static T +#define YY_CTX_LOCAL 1 +#define YY_CTX_MEMBERS \ + context ctx; + +#define YY_INPUT(yy, buf, result, max_size) \ +{ \ + input_string *is = &yy->ctx.is; \ + result = is->size - is->pos; \ + if (result > 0) { \ + if (max_size >= result) { \ + memcpy(buf, &is->s[is->pos], result); \ + is->pos += result; \ + } else { \ + memcpy(buf, &is->s[is->pos], max_size); \ + is->pos += max_size; \ + result = max_size; \ + } \ + } \ +} + + +static void init_match_node(match_node_tmp *mn) +{ + memset(mn, 0, sizeof(match_node_tmp)); +} + + +static void move_match_node(match_node_tmp *dest, match_node_tmp *src) +{ + memcpy(dest, src, sizeof(match_node_tmp)); + init_match_node(src); // dest now owns the memory, wipe the pointers +} + + +static void realloc_mna(match_node_array *mna) +{ + size_t bytes = sizeof(match_node_tmp) * ++mna->size; + match_node_tmp *tmp = realloc(mna->a, bytes); + if (tmp) { + mna->a = tmp; + init_match_node(&mna->a[mna->size - 1]); + } else { + fprintf(stderr, "realloc failed\n"); + exit(1); + } +} + + +static void push_output(context *ctx, match_node_tmp *mn) +{ + if (!ctx->out.a || ctx->out.pos == ctx->out.size) { + realloc_mna(&ctx->out); + } + move_match_node(&ctx->out.a[ctx->out.pos++], mn); +} + + +static void push_op(context *ctx, match_operation op) +{ + if (!ctx->ops.a) { + realloc_mna(&ctx->ops); + ctx->ops.a[ctx->ops.pos++].op = op; + return; + } + + if (op == OP_OPEN || op > ctx->ops.a[ctx->ops.pos - 1].op) { + if (ctx->ops.pos == ctx->ops.size) { + realloc_mna(&ctx->ops); + } + ctx->ops.a[ctx->ops.pos++].op = op; + } else { + push_output(ctx, &ctx->ops.a[ctx->ops.pos - 1]); + ctx->ops.a[ctx->ops.pos - 1].op = op; + } +} + + +static void pop_to_paren(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) break; + push_output(ctx, op); + } +} + + +static void pop_all_ops(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) continue; + push_output(ctx, op); + } +} + + +static void update_date(context *ctx, int year, int mon, int day) +{ + ctx->tms.tm_isdst = -1; + ctx->tms.tm_year = year - 1900; + ctx->tms.tm_mon = mon - 1; + ctx->tms.tm_mday = day; +} + + +static void update_time(context *ctx, int hour, int minute, int sec) +{ + ctx->tms.tm_hour = hour; + ctx->tms.tm_min = minute; + ctx->tms.tm_sec = sec; +} + + +static void update_offset(context *ctx, char sign, int hour, int minute) +{ + ctx->mn.val.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); +} + + +static void set_field(context *ctx, char *name) +{ + ctx->mn.id = LSB_PB_FIELDS; + ctx->mn.var_len = strlen(name); + ctx->mn.var = malloc(ctx->mn.var_len + 1); + if (!ctx->mn.var) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.var, name, ctx->mn.var_len + 1); +} + + +static void set_timestamp(context *ctx) +{ + ctx->mn.id = LSB_PB_TIMESTAMP; + ctx->mn.val_type = TYPE_NUMERIC; + if (ctx->tms.tm_isdst == -1) { + ctx->mn.val.d += mktime(&ctx->tms); + ctx->mn.val.d *= 1e9; + } + memset(&ctx->tms, 0, sizeof(struct tm)); +} + + +static void set_numeric_value(context *ctx, char *s) +{ + ctx->mn.val_type = TYPE_NUMERIC; + ctx->mn.val.d = strtod(s, NULL); +} + + +static void set_string_value(context *ctx, char *s) +{ + ctx->mn.val_type = TYPE_STRING; + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { + ++i; + } + s[j] = s[i]; + } + s[j] = 0; + + ctx->mn.val_len = j; + ctx->mn.val.s = malloc(j + 1); + if (!ctx->mn.val.s) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.val.s, s, j + 1); +} + + +static void set_match_mod(context *ctx) +{ + if (ctx->mn.val_mod == PATTERN_MOD_NONE + && strpbrk(ctx->mn.val.s, "^$*+?.[%-") == NULL) { // literal + ctx->mn.val_mod = PATTERN_MOD_ESC; + } +} + + +static bool check_string_len(char *s) +{ + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { + ++i; + } + } + return (j > UCHAR_MAX) ? false : true; +} + + +static int cond_cnt(context *ctx) +{ + return (++ctx->cond_cnt * 2 + 1 > UCHAR_MAX) ? 0 : 1; +} + + +#ifndef YY_MALLOC +#define YY_MALLOC(C, N) malloc(N) +#endif +#ifndef YY_REALLOC +#define YY_REALLOC(C, P, N) realloc(P, N) +#endif +#ifndef YY_FREE +#define YY_FREE(C, P) free(P) +#endif +#ifndef YY_LOCAL +#define YY_LOCAL(T) static T +#endif +#ifndef YY_ACTION +#define YY_ACTION(T) static T +#endif +#ifndef YY_RULE +#define YY_RULE(T) static T +#endif +#ifndef YY_PARSE +#define YY_PARSE(T) T +#endif +#ifndef YYPARSE +#define YYPARSE yyparse +#endif +#ifndef YYPARSEFROM +#define YYPARSEFROM yyparsefrom +#endif +#ifndef YYRELEASE +#define YYRELEASE yyrelease +#endif +#ifndef YY_BEGIN +#define YY_BEGIN ( yy->__begin= yy->__pos, 1) +#endif +#ifndef YY_END +#define YY_END ( yy->__end= yy->__pos, 1) +#endif +#ifdef YY_DEBUG +# define yyprintf(args) fprintf args +#else +# define yyprintf(args) +#endif +#ifndef YYSTYPE +#define YYSTYPE int +#endif +#ifndef YY_STACK_SIZE +#define YY_STACK_SIZE 128 +#endif + +#ifndef YY_BUFFER_SIZE +#define YY_BUFFER_SIZE 1024 +#endif + +#ifndef YY_PART + +typedef struct _yycontext yycontext; +typedef void (*yyaction)(yycontext *yy, char *yytext, int yyleng); +typedef struct _yythunk { int begin, end; yyaction action; struct _yythunk *next; } yythunk; + +struct _yycontext { + char *__buf; + int __buflen; + int __pos; + int __limit; + char *__text; + int __textlen; + int __begin; + int __end; + int __textmax; + yythunk *__thunks; + int __thunkslen; + int __thunkpos; + YYSTYPE __; + YYSTYPE *__val; + YYSTYPE *__vals; + int __valslen; +#ifdef YY_CTX_MEMBERS + YY_CTX_MEMBERS +#endif +}; + +#ifdef YY_CTX_LOCAL +#define YY_CTX_PARAM_ yycontext *yyctx, +#define YY_CTX_PARAM yycontext *yyctx +#define YY_CTX_ARG_ yyctx, +#define YY_CTX_ARG yyctx +#ifndef YY_INPUT +#define YY_INPUT(yy, buf, result, max_size) \ + { \ + int yyc= getchar(); \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ + yyprintf((stderr, "<%c>", yyc)); \ + } +#endif +#else +#define YY_CTX_PARAM_ +#define YY_CTX_PARAM +#define YY_CTX_ARG_ +#define YY_CTX_ARG +yycontext _yyctx= { 0, 0 }; +yycontext *yyctx= &_yyctx; +#ifndef YY_INPUT +#define YY_INPUT(buf, result, max_size) \ + { \ + int yyc= getchar(); \ + result= (EOF == yyc) ? 0 : (*(buf)= yyc, 1); \ + yyprintf((stderr, "<%c>", yyc)); \ + } +#endif +#endif + +YY_LOCAL(int) yyrefill(yycontext *yy) +{ + int yyn; + while (yy->__buflen - yy->__pos < 512) + { + yy->__buflen *= 2; + yy->__buf= (char *)YY_REALLOC(yy, yy->__buf, yy->__buflen); + } +#ifdef YY_CTX_LOCAL + YY_INPUT(yy, (yy->__buf + yy->__pos), yyn, (yy->__buflen - yy->__pos)); +#else + YY_INPUT((yy->__buf + yy->__pos), yyn, (yy->__buflen - yy->__pos)); +#endif + if (!yyn) return 0; + yy->__limit += yyn; + return 1; +} + +YY_LOCAL(int) yymatchDot(yycontext *yy) +{ + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + ++yy->__pos; + return 1; +} + +YY_LOCAL(int) yymatchChar(yycontext *yy, int c) +{ + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + if ((unsigned char)yy->__buf[yy->__pos] == c) + { + ++yy->__pos; + yyprintf((stderr, " ok yymatchChar(yy, %c) @ %s\n", c, yy->__buf+yy->__pos)); + return 1; + } + yyprintf((stderr, " fail yymatchChar(yy, %c) @ %s\n", c, yy->__buf+yy->__pos)); + return 0; +} + +YY_LOCAL(int) yymatchString(yycontext *yy, const char *s) +{ + int yysav= yy->__pos; + while (*s) + { + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + if (yy->__buf[yy->__pos] != *s) + { + yy->__pos= yysav; + return 0; + } + ++s; + ++yy->__pos; + } + return 1; +} + +YY_LOCAL(int) yymatchClass(yycontext *yy, unsigned char *bits) +{ + int c; + if (yy->__pos >= yy->__limit && !yyrefill(yy)) return 0; + c= (unsigned char)yy->__buf[yy->__pos]; + if (bits[c >> 3] & (1 << (c & 7))) + { + ++yy->__pos; + yyprintf((stderr, " ok yymatchClass @ %s\n", yy->__buf+yy->__pos)); + return 1; + } + yyprintf((stderr, " fail yymatchClass @ %s\n", yy->__buf+yy->__pos)); + return 0; +} + +YY_LOCAL(void) yyDo(yycontext *yy, yyaction action, int begin, int end) +{ + while (yy->__thunkpos >= yy->__thunkslen) + { + yy->__thunkslen *= 2; + yy->__thunks= (yythunk *)YY_REALLOC(yy, yy->__thunks, sizeof(yythunk) * yy->__thunkslen); + } + yy->__thunks[yy->__thunkpos].begin= begin; + yy->__thunks[yy->__thunkpos].end= end; + yy->__thunks[yy->__thunkpos].action= action; + ++yy->__thunkpos; +} + +YY_LOCAL(int) yyText(yycontext *yy, int begin, int end) +{ + int yyleng= end - begin; + if (yyleng <= 0) + yyleng= 0; + else + { + while (yy->__textlen < (yyleng + 1)) + { + yy->__textlen *= 2; + yy->__text= (char *)YY_REALLOC(yy, yy->__text, yy->__textlen); + } + memcpy(yy->__text, yy->__buf + begin, yyleng); + } + yy->__text[yyleng]= '\0'; + return yyleng; +} + +YY_LOCAL(void) yyDone(yycontext *yy) +{ + int pos; + for (pos= 0; pos < yy->__thunkpos; ++pos) + { + yythunk *thunk= &yy->__thunks[pos]; + int yyleng= thunk->end ? yyText(yy, thunk->begin, thunk->end) : thunk->begin; + yyprintf((stderr, "DO [%d] %p %s\n", pos, thunk->action, yy->__text)); + thunk->action(yy, yy->__text, yyleng); + } + yy->__thunkpos= 0; +} + +YY_LOCAL(void) yyCommit(yycontext *yy) +{ + if ((yy->__limit -= yy->__pos)) + { + memmove(yy->__buf, yy->__buf + yy->__pos, yy->__limit); + } + yy->__begin -= yy->__pos; + yy->__end -= yy->__pos; + yy->__pos= yy->__thunkpos= 0; +} + +YY_LOCAL(int) yyAccept(yycontext *yy, int tp0) +{ + if (tp0) + { + fprintf(stderr, "accept denied at %d\n", tp0); + return 0; + } + else + { + yyDone(yy); + yyCommit(yy); + } + return 1; +} + +YY_LOCAL(void) yyPush(yycontext *yy, char *text, int count) +{ + yy->__val += count; + while (yy->__valslen <= yy->__val - yy->__vals) + { + long offset= yy->__val - yy->__vals; + int olen = yy->__valslen; + yy->__valslen *= 2; + yy->__vals= (YYSTYPE *)YY_REALLOC(yy, yy->__vals, sizeof(YYSTYPE) * yy->__valslen); + yy->__val= yy->__vals + offset; + memset(yy->__vals + olen, 0, sizeof(YYSTYPE) * olen); + } +} +YY_LOCAL(void) yyPop(yycontext *yy, char *text, int count) { yy->__val -= count; } +YY_LOCAL(void) yySet(yycontext *yy, char *text, int count) { yy->__val[count]= yy->__; } + +#endif /* YY_PART */ + +#define YYACCEPT yyAccept(yy, yythunkpos0) + +YY_RULE(int) yy_second_frac(yycontext *yy); /* 56 */ +YY_RULE(int) yy_second(yycontext *yy); /* 55 */ +YY_RULE(int) yy_minute(yycontext *yy); /* 54 */ +YY_RULE(int) yy_hour(yycontext *yy); /* 53 */ +YY_RULE(int) yy_timeoffset(yycontext *yy); /* 52 */ +YY_RULE(int) yy_partialtime(yycontext *yy); /* 51 */ +YY_RULE(int) yy_fulltime(yycontext *yy); /* 50 */ +YY_RULE(int) yy_day(yycontext *yy); /* 49 */ +YY_RULE(int) yy_month(yycontext *yy); /* 48 */ +YY_RULE(int) yy_year(yycontext *yy); /* 47 */ +YY_RULE(int) yy_fulldate(yycontext *yy); /* 46 */ +YY_RULE(int) yy_rfc3339(yycontext *yy); /* 45 */ +YY_RULE(int) yy_ts_quoted(yycontext *yy); /* 44 */ +YY_RULE(int) yy_zero_to_255(yycontext *yy); /* 43 */ +YY_RULE(int) yy_index(yycontext *yy); /* 42 */ +YY_RULE(int) yy_fields(yycontext *yy); /* 41 */ +YY_RULE(int) yy_exponent(yycontext *yy); /* 40 */ +YY_RULE(int) yy_decimal(yycontext *yy); /* 39 */ +YY_RULE(int) yy_number(yycontext *yy); /* 38 */ +YY_RULE(int) yy_sign(yycontext *yy); /* 37 */ +YY_RULE(int) yy_numeric_value(yycontext *yy); /* 36 */ +YY_RULE(int) yy_string_match_mod(yycontext *yy); /* 35 */ +YY_RULE(int) yy_nil(yycontext *yy); /* 34 */ +YY_RULE(int) yy_string_match(yycontext *yy); /* 33 */ +YY_RULE(int) yy_string_value(yycontext *yy); /* 32 */ +YY_RULE(int) yy_string_headers(yycontext *yy); /* 31 */ +YY_RULE(int) yy_boolean(yycontext *yy); /* 30 */ +YY_RULE(int) yy_false(yycontext *yy); /* 29 */ +YY_RULE(int) yy_true(yycontext *yy); /* 28 */ +YY_RULE(int) yy_relational(yycontext *yy); /* 27 */ +YY_RULE(int) yy_op_lt(yycontext *yy); /* 26 */ +YY_RULE(int) yy_op_lte(yycontext *yy); /* 25 */ +YY_RULE(int) yy_op_gt(yycontext *yy); /* 24 */ +YY_RULE(int) yy_op_gte(yycontext *yy); /* 23 */ +YY_RULE(int) yy_op_sne(yycontext *yy); /* 22 */ +YY_RULE(int) yy_op_seq(yycontext *yy); /* 21 */ +YY_RULE(int) yy_op_ne(yycontext *yy); /* 20 */ +YY_RULE(int) yy_op_eq(yycontext *yy); /* 19 */ +YY_RULE(int) yy_boolean_test(yycontext *yy); /* 18 */ +YY_RULE(int) yy_field_test(yycontext *yy); /* 17 */ +YY_RULE(int) yy_pid(yycontext *yy); /* 16 */ +YY_RULE(int) yy_severity(yycontext *yy); /* 15 */ +YY_RULE(int) yy_optional_string_headers(yycontext *yy); /* 14 */ +YY_RULE(int) yy_timestamp(yycontext *yy); /* 13 */ +YY_RULE(int) yy_uuid(yycontext *yy); /* 12 */ +YY_RULE(int) yy_close(yycontext *yy); /* 11 */ +YY_RULE(int) yy_open(yycontext *yy); /* 10 */ +YY_RULE(int) yy_test(yycontext *yy); /* 9 */ +YY_RULE(int) yy_and(yycontext *yy); /* 8 */ +YY_RULE(int) yy_expr(yycontext *yy); /* 7 */ +YY_RULE(int) yy_or(yycontext *yy); /* 6 */ +YY_RULE(int) yy_anded(yycontext *yy); /* 5 */ +YY_RULE(int) yy_eol(yycontext *yy); /* 4 */ +YY_RULE(int) yy_ored(yycontext *yy); /* 3 */ +YY_RULE(int) yy_sp(yycontext *yy); /* 2 */ +YY_RULE(int) yy_match(yycontext *yy); /* 1 */ + +YY_ACTION(void) yy_1_nil(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_nil\n")); + { +#line 379 + yy->ctx.mn.val_type = TYPE_NIL; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_second_frac(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_second_frac\n")); + { +#line 377 + yy->ctx.mn.val.d += strtod(yytext, NULL); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_second(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_second\n")); + { +#line 376 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_minute(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_minute\n")); + { +#line 372 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_hour(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_hour\n")); + { +#line 371 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_timeoffset(yycontext *yy, char *yytext, int yyleng) +{ +#define m yy->__val[-1] +#define h yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_timeoffset\n")); + { +#line 367 + update_offset(&yy->ctx, yytext[0], h, m); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef m +#undef h +} +YY_ACTION(void) yy_1_timeoffset(yycontext *yy, char *yytext, int yyleng) +{ +#define m yy->__val[-1] +#define h yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_timeoffset\n")); + { +#line 366 + update_offset(&yy->ctx, '+', 0, 0); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef m +#undef h +} +YY_ACTION(void) yy_1_partialtime(yycontext *yy, char *yytext, int yyleng) +{ +#define s yy->__val[-1] +#define m yy->__val[-2] +#define h yy->__val[-3] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_partialtime\n")); + { +#line 365 + update_time(&yy->ctx, h, m, s); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef s +#undef m +#undef h +} +YY_ACTION(void) yy_1_day(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_day\n")); + { +#line 361 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_month(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_month\n")); + { +#line 356 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_year(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_year\n")); + { +#line 353 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_fulldate(yycontext *yy, char *yytext, int yyleng) +{ +#define d yy->__val[-1] +#define m yy->__val[-2] +#define y yy->__val[-3] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_fulldate\n")); + { +#line 352 + update_date(&yy->ctx, y, m, d); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef d +#undef m +#undef y +} +YY_ACTION(void) yy_1_timestamp(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_timestamp\n")); + { +#line 350 + set_timestamp(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_index(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_index\n")); + { +#line 344 + __ = atoi(yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_3_fields(yycontext *yy, char *yytext, int yyleng) +{ +#define a yy->__val[-1] +#define f yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_3_fields\n")); + { +#line 343 + yy->ctx.mn.ai = a; + } +#undef yythunkpos +#undef yypos +#undef yy +#undef a +#undef f +} +YY_ACTION(void) yy_2_fields(yycontext *yy, char *yytext, int yyleng) +{ +#define a yy->__val[-1] +#define f yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_fields\n")); + { +#line 343 + yy->ctx.mn.fi = f; + } +#undef yythunkpos +#undef yypos +#undef yy +#undef a +#undef f +} +YY_ACTION(void) yy_1_fields(yycontext *yy, char *yytext, int yyleng) +{ +#define a yy->__val[-1] +#define f yy->__val[-2] +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_fields\n")); + { +#line 343 + set_field(&yy->ctx, yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +#undef a +#undef f +} +YY_ACTION(void) yy_1_numeric_value(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_numeric_value\n")); + { +#line 333 + set_numeric_value(&yy->ctx, yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_pid(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_pid\n")); + { +#line 331 + yy->ctx.mn.id = LSB_PB_PID; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_severity(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_severity\n")); + { +#line 330 + yy->ctx.mn.id = LSB_PB_SEVERITY; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_string_match_mod(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_match_mod\n")); + { +#line 328 + yy->ctx.mn.val_mod = PATTERN_MOD_ESC; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_string_match(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_match\n")); + { +#line 326 + set_match_mod(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_string_value(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_value\n")); + { +#line 324 + set_string_value(&yy->ctx, yytext); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_5_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_5_string_headers\n")); + { +#line 320 + yy->ctx.mn.id = LSB_PB_PAYLOAD; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_4_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_4_string_headers\n")); + { +#line 319 + yy->ctx.mn.id = LSB_PB_ENV_VERSION; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_3_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_3_string_headers\n")); + { +#line 318 + yy->ctx.mn.id = LSB_PB_HOSTNAME; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_string_headers\n")); + { +#line 317 + yy->ctx.mn.id = LSB_PB_LOGGER; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_string_headers(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_string_headers\n")); + { +#line 316 + yy->ctx.mn.id = LSB_PB_TYPE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_uuid(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_uuid\n")); + { +#line 315 + yy->ctx.mn.id = LSB_PB_UUID; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_close(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_close\n")); + { +#line 311 + pop_to_paren(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_open(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_open\n")); + { +#line 310 + push_op(&yy->ctx, OP_OPEN); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_or(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_or\n")); + { +#line 309 + push_op(&yy->ctx, OP_OR); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_and(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_and\n")); + { +#line 308 + push_op(&yy->ctx, OP_AND); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_boolean(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_boolean\n")); + { +#line 306 + yy->ctx.mn.val_type = TYPE_FALSE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_boolean(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_boolean\n")); + { +#line 305 + yy->ctx.mn.val_type = TYPE_TRUE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_2_boolean_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_2_boolean_test\n")); + { +#line 304 + yy->ctx.mn.op = OP_FALSE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_boolean_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_boolean_test\n")); + { +#line 303 + yy->ctx.mn.op = OP_TRUE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_lt(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_lt\n")); + { +#line 294 + yy->ctx.mn.op = OP_LT; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_lte(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_lte\n")); + { +#line 293 + yy->ctx.mn.op = OP_LTE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_gt(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_gt\n")); + { +#line 292 + yy->ctx.mn.op = OP_GT; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_gte(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_gte\n")); + { +#line 291 + yy->ctx.mn.op = OP_GTE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_sne(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_sne\n")); + { +#line 290 + yy->ctx.mn.op = OP_NRE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_seq(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_seq\n")); + { +#line 289 + yy->ctx.mn.op = OP_RE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_ne(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_ne\n")); + { +#line 288 + yy->ctx.mn.op = OP_NE; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_op_eq(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_op_eq\n")); + { +#line 287 + yy->ctx.mn.op = OP_EQ; + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_test(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_test\n")); + { +#line 285 + push_output(&yy->ctx, &yy->ctx.mn); + } +#undef yythunkpos +#undef yypos +#undef yy +} +YY_ACTION(void) yy_1_match(yycontext *yy, char *yytext, int yyleng) +{ +#define __ yy->__ +#define yypos yy->__pos +#define yythunkpos yy->__thunkpos + yyprintf((stderr, "do yy_1_match\n")); + { +#line 274 + pop_all_ops(&yy->ctx); + } +#undef yythunkpos +#undef yypos +#undef yy +} + +YY_RULE(int) yy_second_frac(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "second_frac")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l1; +#undef yytext +#undef yyleng + } if (!yy_decimal(yy)) goto l1; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l1; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_second_frac, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "second_frac", yy->__buf+yy->__pos)); + return 1; + l1:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "second_frac", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_second(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "second")); + { int yypos3= yy->__pos, yythunkpos3= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l4; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l4; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l4; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l4; +#undef yytext +#undef yyleng + } goto l3; + l4:; yy->__pos= yypos3; yy->__thunkpos= yythunkpos3; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l2; +#undef yytext +#undef yyleng + } if (!yymatchString(yy, "60")) goto l2; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l2; +#undef yytext +#undef yyleng + } + } + l3:; yyDo(yy, yy_1_second, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "second", yy->__buf+yy->__pos)); + return 1; + l2:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "second", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_minute(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "minute")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l5; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l5; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l5; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l5; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_minute, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "minute", yy->__buf+yy->__pos)); + return 1; + l5:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "minute", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_hour(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "hour")); + { int yypos7= yy->__pos, yythunkpos7= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l8; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l8; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l8; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l8; +#undef yytext +#undef yyleng + } goto l7; + l8:; yy->__pos= yypos7; yy->__thunkpos= yythunkpos7; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l6; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '2')) goto l6; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\017\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l6; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l6; +#undef yytext +#undef yyleng + } + } + l7:; yyDo(yy, yy_1_hour, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "hour", yy->__buf+yy->__pos)); + return 1; + l6:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "hour", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_timeoffset(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 2, 0); + yyprintf((stderr, "%s\n", "timeoffset")); + { int yypos10= yy->__pos, yythunkpos10= yy->__thunkpos; if (!yymatchChar(yy, 'Z')) goto l11; yyDo(yy, yy_1_timeoffset, yy->__begin, yy->__end); goto l10; + l11:; yy->__pos= yypos10; yy->__thunkpos= yythunkpos10; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l9; +#undef yytext +#undef yyleng + } if (!yy_sign(yy)) goto l9; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l9; +#undef yytext +#undef yyleng + } if (!yy_hour(yy)) goto l9; yyDo(yy, yySet, -2, 0); if (!yy_minute(yy)) goto l9; yyDo(yy, yySet, -1, 0); yyDo(yy, yy_2_timeoffset, yy->__begin, yy->__end); + } + l10:; + yyprintf((stderr, " ok %s @ %s\n", "timeoffset", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 2, 0); + return 1; + l9:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "timeoffset", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_partialtime(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 3, 0); + yyprintf((stderr, "%s\n", "partialtime")); if (!yy_hour(yy)) goto l12; yyDo(yy, yySet, -3, 0); if (!yymatchChar(yy, ':')) goto l12; if (!yy_minute(yy)) goto l12; yyDo(yy, yySet, -2, 0); if (!yymatchChar(yy, ':')) goto l12; if (!yy_second(yy)) goto l12; yyDo(yy, yySet, -1, 0); + { int yypos13= yy->__pos, yythunkpos13= yy->__thunkpos; if (!yy_second_frac(yy)) goto l13; goto l14; + l13:; yy->__pos= yypos13; yy->__thunkpos= yythunkpos13; + } + l14:; yyDo(yy, yy_1_partialtime, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "partialtime", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 3, 0); + return 1; + l12:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "partialtime", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_fulltime(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "fulltime")); if (!yy_partialtime(yy)) goto l15; if (!yy_timeoffset(yy)) goto l15; + yyprintf((stderr, " ok %s @ %s\n", "fulltime", yy->__buf+yy->__pos)); + return 1; + l15:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "fulltime", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_day(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "day")); + { int yypos17= yy->__pos, yythunkpos17= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l18; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\006\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l18; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l18; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l18; +#undef yytext +#undef yyleng + } goto l17; + l18:; yy->__pos= yypos17; yy->__thunkpos= yythunkpos17; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l19; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '0')) goto l19; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l19; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l19; +#undef yytext +#undef yyleng + } goto l17; + l19:; yy->__pos= yypos17; yy->__thunkpos= yythunkpos17; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l16; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '3')) goto l16; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l16; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l16; +#undef yytext +#undef yyleng + } + } + l17:; yyDo(yy, yy_1_day, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "day", yy->__buf+yy->__pos)); + return 1; + l16:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "day", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_month(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "month")); + { int yypos21= yy->__pos, yythunkpos21= yy->__thunkpos; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l22; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '0')) goto l22; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l22; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l22; +#undef yytext +#undef yyleng + } goto l21; + l22:; yy->__pos= yypos21; yy->__thunkpos= yythunkpos21; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l20; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '1')) goto l20; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\007\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l20; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l20; +#undef yytext +#undef yyleng + } + } + l21:; yyDo(yy, yy_1_month, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "month", yy->__buf+yy->__pos)); + return 1; + l20:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "month", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_year(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "year")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l23; +#undef yytext +#undef yyleng + } if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l23; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l23; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_year, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "year", yy->__buf+yy->__pos)); + return 1; + l23:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "year", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_fulldate(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 3, 0); + yyprintf((stderr, "%s\n", "fulldate")); if (!yy_year(yy)) goto l24; yyDo(yy, yySet, -3, 0); if (!yymatchChar(yy, '-')) goto l24; if (!yy_month(yy)) goto l24; yyDo(yy, yySet, -2, 0); if (!yymatchChar(yy, '-')) goto l24; if (!yy_day(yy)) goto l24; yyDo(yy, yySet, -1, 0); yyDo(yy, yy_1_fulldate, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "fulldate", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 3, 0); + return 1; + l24:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "fulldate", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_rfc3339(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "rfc3339")); if (!yy_fulldate(yy)) goto l25; if (!yymatchChar(yy, 'T')) goto l25; if (!yy_fulltime(yy)) goto l25; + yyprintf((stderr, " ok %s @ %s\n", "rfc3339", yy->__buf+yy->__pos)); + return 1; + l25:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "rfc3339", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_ts_quoted(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "ts_quoted")); + { int yypos27= yy->__pos, yythunkpos27= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l28; if (!yy_rfc3339(yy)) goto l28; if (!yymatchChar(yy, '"')) goto l28; goto l27; + l28:; yy->__pos= yypos27; yy->__thunkpos= yythunkpos27; if (!yymatchChar(yy, '\'')) goto l26; if (!yy_rfc3339(yy)) goto l26; if (!yymatchChar(yy, '\'')) goto l26; + } + l27:; + yyprintf((stderr, " ok %s @ %s\n", "ts_quoted", yy->__buf+yy->__pos)); + return 1; + l26:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "ts_quoted", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_zero_to_255(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "zero_to_255")); + { int yypos30= yy->__pos, yythunkpos30= yy->__thunkpos; if (!yymatchChar(yy, '2')) goto l31; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l31; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\077\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l31; goto l30; + l31:; yy->__pos= yypos30; yy->__thunkpos= yythunkpos30; if (!yymatchChar(yy, '1')) goto l32; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l32; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l32; goto l30; + l32:; yy->__pos= yypos30; yy->__thunkpos= yythunkpos30; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l33; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l33; goto l30; + l33:; yy->__pos= yypos30; yy->__thunkpos= yythunkpos30; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l29; + } + l30:; + yyprintf((stderr, " ok %s @ %s\n", "zero_to_255", yy->__buf+yy->__pos)); + return 1; + l29:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "zero_to_255", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_index(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "index")); if (!yymatchChar(yy, '[')) goto l34; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l34; +#undef yytext +#undef yyleng + } if (!yy_zero_to_255(yy)) goto l34; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l34; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, ']')) goto l34; yyDo(yy, yy_1_index, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "index", yy->__buf+yy->__pos)); + return 1; + l34:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "index", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_fields(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; yyDo(yy, yyPush, 2, 0); + yyprintf((stderr, "%s\n", "fields")); if (!yymatchString(yy, "Fields[")) goto l35; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l35; +#undef yytext +#undef yyleng + } + l36:; + { int yypos37= yy->__pos, yythunkpos37= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\377\377\377\377\377\377\377\377\377\377\377\337\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377\377")) goto l37; goto l36; + l37:; yy->__pos= yypos37; yy->__thunkpos= yythunkpos37; + } yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l35; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, ']')) goto l35; yyDo(yy, yy_1_fields, yy->__begin, yy->__end); + { int yypos38= yy->__pos, yythunkpos38= yy->__thunkpos; if (!yy_index(yy)) goto l38; yyDo(yy, yySet, -2, 0); goto l39; + l38:; yy->__pos= yypos38; yy->__thunkpos= yythunkpos38; + } + l39:; yyDo(yy, yy_2_fields, yy->__begin, yy->__end); + { int yypos40= yy->__pos, yythunkpos40= yy->__thunkpos; if (!yy_index(yy)) goto l40; yyDo(yy, yySet, -1, 0); goto l41; + l40:; yy->__pos= yypos40; yy->__thunkpos= yythunkpos40; + } + l41:; yyDo(yy, yy_3_fields, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "fields", yy->__buf+yy->__pos)); yyDo(yy, yyPop, 2, 0); + return 1; + l35:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "fields", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_exponent(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "exponent")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\000\000\040\000\000\000\040\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l42; + { int yypos43= yy->__pos, yythunkpos43= yy->__thunkpos; if (!yy_sign(yy)) goto l43; goto l44; + l43:; yy->__pos= yypos43; yy->__thunkpos= yythunkpos43; + } + l44:; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l42; + l45:; + { int yypos46= yy->__pos, yythunkpos46= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l46; goto l45; + l46:; yy->__pos= yypos46; yy->__thunkpos= yythunkpos46; + } + yyprintf((stderr, " ok %s @ %s\n", "exponent", yy->__buf+yy->__pos)); + return 1; + l42:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "exponent", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_decimal(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "decimal")); if (!yymatchChar(yy, '.')) goto l47; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l47; + l48:; + { int yypos49= yy->__pos, yythunkpos49= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l49; goto l48; + l49:; yy->__pos= yypos49; yy->__thunkpos= yythunkpos49; + } + yyprintf((stderr, " ok %s @ %s\n", "decimal", yy->__buf+yy->__pos)); + return 1; + l47:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "decimal", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_number(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "number")); + { int yypos51= yy->__pos, yythunkpos51= yy->__thunkpos; if (!yymatchChar(yy, '0')) goto l52; goto l51; + l52:; yy->__pos= yypos51; yy->__thunkpos= yythunkpos51; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\376\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l50; + l53:; + { int yypos54= yy->__pos, yythunkpos54= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\000\377\003\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l54; goto l53; + l54:; yy->__pos= yypos54; yy->__thunkpos= yythunkpos54; + } + } + l51:; + yyprintf((stderr, " ok %s @ %s\n", "number", yy->__buf+yy->__pos)); + return 1; + l50:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "number", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_sign(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "sign")); if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\000\050\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l55; + yyprintf((stderr, " ok %s @ %s\n", "sign", yy->__buf+yy->__pos)); + return 1; + l55:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "sign", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_numeric_value(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "numeric_value")); yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l56; +#undef yytext +#undef yyleng + } + { int yypos57= yy->__pos, yythunkpos57= yy->__thunkpos; if (!yy_sign(yy)) goto l57; goto l58; + l57:; yy->__pos= yypos57; yy->__thunkpos= yythunkpos57; + } + l58:; if (!yy_number(yy)) goto l56; + { int yypos59= yy->__pos, yythunkpos59= yy->__thunkpos; if (!yy_decimal(yy)) goto l59; goto l60; + l59:; yy->__pos= yypos59; yy->__thunkpos= yythunkpos59; + } + l60:; + { int yypos61= yy->__pos, yythunkpos61= yy->__thunkpos; if (!yy_exponent(yy)) goto l61; goto l62; + l61:; yy->__pos= yypos61; yy->__thunkpos= yythunkpos61; + } + l62:; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l56; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_numeric_value, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "numeric_value", yy->__buf+yy->__pos)); + return 1; + l56:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "numeric_value", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_match_mod(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_match_mod")); if (!yymatchChar(yy, '%')) goto l63; yyDo(yy, yy_1_string_match_mod, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); + return 1; + l63:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_match_mod", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_nil(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "nil")); if (!yymatchString(yy, "NIL")) goto l64; yyDo(yy, yy_1_nil, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l64; + yyprintf((stderr, " ok %s @ %s\n", "nil", yy->__buf+yy->__pos)); + return 1; + l64:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "nil", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_match(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_match")); + { int yypos66= yy->__pos, yythunkpos66= yy->__thunkpos; if (!yy_op_seq(yy)) goto l67; goto l66; + l67:; yy->__pos= yypos66; yy->__thunkpos= yythunkpos66; if (!yy_op_sne(yy)) goto l65; + } + l66:; if (!yy_sp(yy)) goto l65; if (!yy_string_value(yy)) goto l65; + { int yypos68= yy->__pos, yythunkpos68= yy->__thunkpos; if (!yy_string_match_mod(yy)) goto l68; goto l69; + l68:; yy->__pos= yypos68; yy->__thunkpos= yythunkpos68; + } + l69:; yyDo(yy, yy_1_string_match, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "string_match", yy->__buf+yy->__pos)); + return 1; + l65:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_match", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_value(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_value")); + { int yypos71= yy->__pos, yythunkpos71= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l72; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l72; +#undef yytext +#undef yyleng + } + l73:; + { int yypos74= yy->__pos, yythunkpos74= yy->__thunkpos; + { int yypos75= yy->__pos, yythunkpos75= yy->__thunkpos; if (!yymatchChar(yy, '\\')) goto l76; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\004\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l76; goto l75; + l76:; yy->__pos= yypos75; yy->__thunkpos= yythunkpos75; + { int yypos77= yy->__pos, yythunkpos77= yy->__thunkpos; if (!yymatchChar(yy, '"')) goto l77; goto l74; + l77:; yy->__pos= yypos77; yy->__thunkpos= yythunkpos77; + } if (!yymatchDot(yy)) goto l74; + } + l75:; goto l73; + l74:; yy->__pos= yypos74; yy->__thunkpos= yythunkpos74; + } yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l72; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '"')) goto l72; goto l71; + l72:; yy->__pos= yypos71; yy->__thunkpos= yythunkpos71; if (!yymatchChar(yy, '\'')) goto l70; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_BEGIN)) goto l70; +#undef yytext +#undef yyleng + } + l78:; + { int yypos79= yy->__pos, yythunkpos79= yy->__thunkpos; + { int yypos80= yy->__pos, yythunkpos80= yy->__thunkpos; if (!yymatchChar(yy, '\\')) goto l81; if (!yymatchClass(yy, (unsigned char *)"\000\000\000\000\200\000\000\000\000\000\000\020\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l81; goto l80; + l81:; yy->__pos= yypos80; yy->__thunkpos= yythunkpos80; + { int yypos82= yy->__pos, yythunkpos82= yy->__thunkpos; if (!yymatchChar(yy, '\'')) goto l82; goto l79; + l82:; yy->__pos= yypos82; yy->__thunkpos= yythunkpos82; + } if (!yymatchDot(yy)) goto l79; + } + l80:; goto l78; + l79:; yy->__pos= yypos79; yy->__thunkpos= yythunkpos79; + } yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(YY_END)) goto l70; +#undef yytext +#undef yyleng + } if (!yymatchChar(yy, '\'')) goto l70; + } + l71:; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(check_string_len(yytext))) goto l70; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_string_value, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "string_value", yy->__buf+yy->__pos)); + return 1; + l70:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_value", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_string_headers(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "string_headers")); + { int yypos84= yy->__pos, yythunkpos84= yy->__thunkpos; if (!yymatchString(yy, "Type")) goto l85; yyDo(yy, yy_1_string_headers, yy->__begin, yy->__end); goto l84; + l85:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Logger")) goto l86; yyDo(yy, yy_2_string_headers, yy->__begin, yy->__end); goto l84; + l86:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Hostname")) goto l87; yyDo(yy, yy_3_string_headers, yy->__begin, yy->__end); goto l84; + l87:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "EnvVersion")) goto l88; yyDo(yy, yy_4_string_headers, yy->__begin, yy->__end); goto l84; + l88:; yy->__pos= yypos84; yy->__thunkpos= yythunkpos84; if (!yymatchString(yy, "Payload")) goto l83; yyDo(yy, yy_5_string_headers, yy->__begin, yy->__end); + } + l84:; + yyprintf((stderr, " ok %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); + return 1; + l83:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "string_headers", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_boolean(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "boolean")); + { int yypos90= yy->__pos, yythunkpos90= yy->__thunkpos; if (!yy_true(yy)) goto l91; yyDo(yy, yy_1_boolean, yy->__begin, yy->__end); goto l90; + l91:; yy->__pos= yypos90; yy->__thunkpos= yythunkpos90; if (!yy_false(yy)) goto l89; yyDo(yy, yy_2_boolean, yy->__begin, yy->__end); + } + l90:; + yyprintf((stderr, " ok %s @ %s\n", "boolean", yy->__buf+yy->__pos)); + return 1; + l89:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "boolean", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_false(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "false")); if (!yymatchString(yy, "FALSE")) goto l92; if (!yy_sp(yy)) goto l92; + yyprintf((stderr, " ok %s @ %s\n", "false", yy->__buf+yy->__pos)); + return 1; + l92:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "false", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_true(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "true")); if (!yymatchString(yy, "TRUE")) goto l93; if (!yy_sp(yy)) goto l93; + yyprintf((stderr, " ok %s @ %s\n", "true", yy->__buf+yy->__pos)); + return 1; + l93:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "true", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_relational(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "relational")); + { int yypos95= yy->__pos, yythunkpos95= yy->__thunkpos; if (!yy_op_eq(yy)) goto l96; goto l95; + l96:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_ne(yy)) goto l97; goto l95; + l97:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_gte(yy)) goto l98; goto l95; + l98:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_gt(yy)) goto l99; goto l95; + l99:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_lte(yy)) goto l100; goto l95; + l100:; yy->__pos= yypos95; yy->__thunkpos= yythunkpos95; if (!yy_op_lt(yy)) goto l94; + } + l95:; + yyprintf((stderr, " ok %s @ %s\n", "relational", yy->__buf+yy->__pos)); + return 1; + l94:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "relational", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_lt(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_lt")); if (!yymatchChar(yy, '<')) goto l101; if (!yy_sp(yy)) goto l101; yyDo(yy, yy_1_op_lt, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); + return 1; + l101:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_lt", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_lte(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_lte")); if (!yymatchString(yy, "<=")) goto l102; if (!yy_sp(yy)) goto l102; yyDo(yy, yy_1_op_lte, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); + return 1; + l102:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_lte", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_gt(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_gt")); if (!yymatchChar(yy, '>')) goto l103; if (!yy_sp(yy)) goto l103; yyDo(yy, yy_1_op_gt, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); + return 1; + l103:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_gt", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_gte(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_gte")); if (!yymatchString(yy, ">=")) goto l104; if (!yy_sp(yy)) goto l104; yyDo(yy, yy_1_op_gte, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); + return 1; + l104:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_gte", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_sne(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_sne")); if (!yymatchString(yy, "!~")) goto l105; if (!yy_sp(yy)) goto l105; yyDo(yy, yy_1_op_sne, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); + return 1; + l105:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_sne", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_seq(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_seq")); if (!yymatchString(yy, "=~")) goto l106; if (!yy_sp(yy)) goto l106; yyDo(yy, yy_1_op_seq, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); + return 1; + l106:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_seq", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_ne(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_ne")); if (!yymatchString(yy, "!=")) goto l107; if (!yy_sp(yy)) goto l107; yyDo(yy, yy_1_op_ne, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); + return 1; + l107:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_ne", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_op_eq(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "op_eq")); if (!yymatchString(yy, "==")) goto l108; if (!yy_sp(yy)) goto l108; yyDo(yy, yy_1_op_eq, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); + return 1; + l108:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "op_eq", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_boolean_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "boolean_test")); + { int yypos110= yy->__pos, yythunkpos110= yy->__thunkpos; if (!yy_true(yy)) goto l111; yyDo(yy, yy_1_boolean_test, yy->__begin, yy->__end); goto l110; + l111:; yy->__pos= yypos110; yy->__thunkpos= yythunkpos110; if (!yy_false(yy)) goto l109; yyDo(yy, yy_2_boolean_test, yy->__begin, yy->__end); + } + l110:; + yyprintf((stderr, " ok %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); + return 1; + l109:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "boolean_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_field_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "field_test")); if (!yy_fields(yy)) goto l112; if (!yy_sp(yy)) goto l112; + { int yypos113= yy->__pos, yythunkpos113= yy->__thunkpos; if (!yy_relational(yy)) goto l114; if (!yy_sp(yy)) goto l114; + { int yypos115= yy->__pos, yythunkpos115= yy->__thunkpos; if (!yy_string_value(yy)) goto l116; goto l115; + l116:; yy->__pos= yypos115; yy->__thunkpos= yythunkpos115; if (!yy_numeric_value(yy)) goto l114; + } + l115:; goto l113; + l114:; yy->__pos= yypos113; yy->__thunkpos= yythunkpos113; if (!yy_string_match(yy)) goto l117; goto l113; + l117:; yy->__pos= yypos113; yy->__thunkpos= yythunkpos113; + { int yypos118= yy->__pos, yythunkpos118= yy->__thunkpos; if (!yy_op_eq(yy)) goto l119; goto l118; + l119:; yy->__pos= yypos118; yy->__thunkpos= yythunkpos118; if (!yy_op_ne(yy)) goto l112; + } + l118:; if (!yy_sp(yy)) goto l112; + { int yypos120= yy->__pos, yythunkpos120= yy->__thunkpos; if (!yy_boolean(yy)) goto l121; goto l120; + l121:; yy->__pos= yypos120; yy->__thunkpos= yythunkpos120; if (!yy_nil(yy)) goto l112; + } + l120:; + } + l113:; + yyprintf((stderr, " ok %s @ %s\n", "field_test", yy->__buf+yy->__pos)); + return 1; + l112:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "field_test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_pid(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "pid")); if (!yymatchString(yy, "Pid")) goto l122; yyDo(yy, yy_1_pid, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l122; + { int yypos123= yy->__pos, yythunkpos123= yy->__thunkpos; if (!yy_relational(yy)) goto l124; if (!yy_sp(yy)) goto l124; if (!yy_numeric_value(yy)) goto l124; goto l123; + l124:; yy->__pos= yypos123; yy->__thunkpos= yythunkpos123; + { int yypos125= yy->__pos, yythunkpos125= yy->__thunkpos; if (!yy_op_eq(yy)) goto l126; goto l125; + l126:; yy->__pos= yypos125; yy->__thunkpos= yythunkpos125; if (!yy_op_ne(yy)) goto l122; + } + l125:; if (!yy_sp(yy)) goto l122; if (!yy_nil(yy)) goto l122; + } + l123:; + yyprintf((stderr, " ok %s @ %s\n", "pid", yy->__buf+yy->__pos)); + return 1; + l122:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "pid", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_severity(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "severity")); if (!yymatchString(yy, "Severity")) goto l127; yyDo(yy, yy_1_severity, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l127; if (!yy_relational(yy)) goto l127; if (!yy_sp(yy)) goto l127; if (!yy_numeric_value(yy)) goto l127; + yyprintf((stderr, " ok %s @ %s\n", "severity", yy->__buf+yy->__pos)); + return 1; + l127:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "severity", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_optional_string_headers(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "optional_string_headers")); if (!yy_string_headers(yy)) goto l128; if (!yy_sp(yy)) goto l128; + { int yypos129= yy->__pos, yythunkpos129= yy->__thunkpos; if (!yy_relational(yy)) goto l130; if (!yy_sp(yy)) goto l130; if (!yy_string_value(yy)) goto l130; goto l129; + l130:; yy->__pos= yypos129; yy->__thunkpos= yythunkpos129; if (!yy_string_match(yy)) goto l131; goto l129; + l131:; yy->__pos= yypos129; yy->__thunkpos= yythunkpos129; + { int yypos132= yy->__pos, yythunkpos132= yy->__thunkpos; if (!yy_op_eq(yy)) goto l133; goto l132; + l133:; yy->__pos= yypos132; yy->__thunkpos= yythunkpos132; if (!yy_op_ne(yy)) goto l128; + } + l132:; if (!yy_sp(yy)) goto l128; if (!yy_nil(yy)) goto l128; + } + l129:; + yyprintf((stderr, " ok %s @ %s\n", "optional_string_headers", yy->__buf+yy->__pos)); + return 1; + l128:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "optional_string_headers", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_timestamp(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "timestamp")); if (!yymatchString(yy, "Timestamp")) goto l134; if (!yy_sp(yy)) goto l134; if (!yy_relational(yy)) goto l134; if (!yy_sp(yy)) goto l134; + { int yypos135= yy->__pos, yythunkpos135= yy->__thunkpos; if (!yy_numeric_value(yy)) goto l136; goto l135; + l136:; yy->__pos= yypos135; yy->__thunkpos= yythunkpos135; if (!yy_ts_quoted(yy)) goto l134; + } + l135:; yyDo(yy, yy_1_timestamp, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "timestamp", yy->__buf+yy->__pos)); + return 1; + l134:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "timestamp", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_uuid(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "uuid")); if (!yymatchString(yy, "Uuid")) goto l137; yyDo(yy, yy_1_uuid, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l137; + { int yypos138= yy->__pos, yythunkpos138= yy->__thunkpos; if (!yy_relational(yy)) goto l139; if (!yy_sp(yy)) goto l139; if (!yy_string_value(yy)) goto l139; goto l138; + l139:; yy->__pos= yypos138; yy->__thunkpos= yythunkpos138; if (!yy_string_match(yy)) goto l137; + } + l138:; + yyprintf((stderr, " ok %s @ %s\n", "uuid", yy->__buf+yy->__pos)); + return 1; + l137:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "uuid", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_close(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "close")); if (!yymatchChar(yy, ')')) goto l140; yyDo(yy, yy_1_close, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l140; + yyprintf((stderr, " ok %s @ %s\n", "close", yy->__buf+yy->__pos)); + return 1; + l140:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "close", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_open(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "open")); if (!yymatchChar(yy, '(')) goto l141; yyDo(yy, yy_1_open, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l141; + yyprintf((stderr, " ok %s @ %s\n", "open", yy->__buf+yy->__pos)); + return 1; + l141:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "open", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_test(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "test")); + { int yypos143= yy->__pos, yythunkpos143= yy->__thunkpos; if (!yy_uuid(yy)) goto l144; goto l143; + l144:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_timestamp(yy)) goto l145; goto l143; + l145:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_optional_string_headers(yy)) goto l146; goto l143; + l146:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_severity(yy)) goto l147; goto l143; + l147:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_pid(yy)) goto l148; goto l143; + l148:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_field_test(yy)) goto l149; goto l143; + l149:; yy->__pos= yypos143; yy->__thunkpos= yythunkpos143; if (!yy_boolean_test(yy)) goto l142; + } + l143:; yyDo(yy, yy_1_test, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l142; + yyprintf((stderr, " ok %s @ %s\n", "test", yy->__buf+yy->__pos)); + return 1; + l142:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "test", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_and(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "and")); if (!yymatchString(yy, "&&")) goto l150; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(cond_cnt(&yy->ctx))) goto l150; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_and, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l150; + yyprintf((stderr, " ok %s @ %s\n", "and", yy->__buf+yy->__pos)); + return 1; + l150:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "and", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_expr(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "expr")); + { int yypos152= yy->__pos, yythunkpos152= yy->__thunkpos; if (!yy_test(yy)) goto l153; goto l152; + l153:; yy->__pos= yypos152; yy->__thunkpos= yythunkpos152; if (!yy_open(yy)) goto l151; if (!yy_ored(yy)) goto l151; if (!yy_close(yy)) goto l151; + } + l152:; + yyprintf((stderr, " ok %s @ %s\n", "expr", yy->__buf+yy->__pos)); + return 1; + l151:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "expr", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_or(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "or")); if (!yymatchString(yy, "||")) goto l154; yyText(yy, yy->__begin, yy->__end); { +#define yytext yy->__text +#define yyleng yy->__textlen +if (!(cond_cnt(&yy->ctx))) goto l154; +#undef yytext +#undef yyleng + } yyDo(yy, yy_1_or, yy->__begin, yy->__end); if (!yy_sp(yy)) goto l154; + yyprintf((stderr, " ok %s @ %s\n", "or", yy->__buf+yy->__pos)); + return 1; + l154:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "or", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_anded(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "anded")); if (!yy_expr(yy)) goto l155; + l156:; + { int yypos157= yy->__pos, yythunkpos157= yy->__thunkpos; if (!yy_and(yy)) goto l157; if (!yy_expr(yy)) goto l157; goto l156; + l157:; yy->__pos= yypos157; yy->__thunkpos= yythunkpos157; + } if (!yy_sp(yy)) goto l155; + yyprintf((stderr, " ok %s @ %s\n", "anded", yy->__buf+yy->__pos)); + return 1; + l155:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "anded", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_eol(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "eol")); + { int yypos159= yy->__pos, yythunkpos159= yy->__thunkpos; if (!yymatchDot(yy)) goto l159; goto l158; + l159:; yy->__pos= yypos159; yy->__thunkpos= yythunkpos159; + } + yyprintf((stderr, " ok %s @ %s\n", "eol", yy->__buf+yy->__pos)); + return 1; + l158:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "eol", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_ored(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "ored")); if (!yy_anded(yy)) goto l160; + l161:; + { int yypos162= yy->__pos, yythunkpos162= yy->__thunkpos; if (!yy_or(yy)) goto l162; if (!yy_anded(yy)) goto l162; goto l161; + l162:; yy->__pos= yypos162; yy->__thunkpos= yythunkpos162; + } if (!yy_sp(yy)) goto l160; + yyprintf((stderr, " ok %s @ %s\n", "ored", yy->__buf+yy->__pos)); + return 1; + l160:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "ored", yy->__buf+yy->__pos)); + return 0; +} +YY_RULE(int) yy_sp(yycontext *yy) +{ + yyprintf((stderr, "%s\n", "sp")); + l164:; + { int yypos165= yy->__pos, yythunkpos165= yy->__thunkpos; if (!yymatchClass(yy, (unsigned char *)"\000\046\000\000\001\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000")) goto l165; goto l164; + l165:; yy->__pos= yypos165; yy->__thunkpos= yythunkpos165; + } + yyprintf((stderr, " ok %s @ %s\n", "sp", yy->__buf+yy->__pos)); + return 1; +} +YY_RULE(int) yy_match(yycontext *yy) +{ int yypos0= yy->__pos, yythunkpos0= yy->__thunkpos; + yyprintf((stderr, "%s\n", "match")); if (!yy_sp(yy)) goto l166; if (!yy_ored(yy)) goto l166; if (!yy_eol(yy)) goto l166; yyDo(yy, yy_1_match, yy->__begin, yy->__end); + yyprintf((stderr, " ok %s @ %s\n", "match", yy->__buf+yy->__pos)); + return 1; + l166:; yy->__pos= yypos0; yy->__thunkpos= yythunkpos0; + yyprintf((stderr, " fail %s @ %s\n", "match", yy->__buf+yy->__pos)); + return 0; +} + +#ifndef YY_PART + +typedef int (*yyrule)(yycontext *yy); + +YY_PARSE(int) YYPARSEFROM(YY_CTX_PARAM_ yyrule yystart) +{ + int yyok; + if (!yyctx->__buflen) + { + yyctx->__buflen= YY_BUFFER_SIZE; + yyctx->__buf= (char *)YY_MALLOC(yyctx, yyctx->__buflen); + yyctx->__textlen= YY_BUFFER_SIZE; + yyctx->__text= (char *)YY_MALLOC(yyctx, yyctx->__textlen); + yyctx->__thunkslen= YY_STACK_SIZE; + yyctx->__thunks= (yythunk *)YY_MALLOC(yyctx, sizeof(yythunk) * yyctx->__thunkslen); + yyctx->__valslen= YY_STACK_SIZE; + yyctx->__vals= (YYSTYPE *)YY_MALLOC(yyctx, sizeof(YYSTYPE) * yyctx->__valslen); + memset(yyctx->__vals, 0, sizeof(YYSTYPE) * yyctx->__valslen); + yyctx->__begin= yyctx->__end= yyctx->__pos= yyctx->__limit= yyctx->__thunkpos= 0; + } + yyctx->__begin= yyctx->__end= yyctx->__pos; + yyctx->__thunkpos= 0; + yyctx->__val= yyctx->__vals; + yyok= yystart(yyctx); + if (yyok) yyDone(yyctx); + yyCommit(yyctx); + return yyok; +} + +YY_PARSE(int) YYPARSE(YY_CTX_PARAM) +{ + return YYPARSEFROM(YY_CTX_ARG_ yy_match); +} + +YY_PARSE(yycontext *) YYRELEASE(yycontext *yyctx) +{ + if (yyctx->__buflen) + { + yyctx->__buflen= 0; + YY_FREE(yyctx, yyctx->__buf); + YY_FREE(yyctx, yyctx->__text); + YY_FREE(yyctx, yyctx->__thunks); + YY_FREE(yyctx, yyctx->__vals); + } + return yyctx; +} + +#endif +#line 385 "../src/util/heka_message_matcher_parser.leg" + + +static match_node* copy_node(unsigned char parent, match_node *mn, + match_node_tmp *mnt) +{ + mn->op = mnt->op; + mn->val_len = 0; + mn->var_len = 0; + mn->field_id = mnt->id; + mn->val_mod = mnt->val_mod; + mn->val_type = mnt->val_type; + + if (mn->op == OP_AND || mn->op == OP_OR) { + mn->u.off = parent; + } else { + mn->u.idx.f = mnt->fi; + mn->u.idx.a = mnt->ai; + } + + if (mnt->id == LSB_PB_FIELDS) { + mn->var_len = mnt->var_len; + } + if (mn->var_len) { + memcpy(mn->data, mnt->var, mn->var_len); // no NUL terminator + free(mnt->var); + mnt->var = NULL; + mnt->var_len = 0; + } + + size_t val_len = 0; + switch (mnt->val_type) { + case TYPE_STRING: + val_len = mnt->val_len + 1; + memcpy(mn->data + mn->var_len, mnt->val.s, val_len); + free(mnt->val.s); + mnt->val.s = NULL; + mnt->val_len = 0; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + memcpy(mn->data + mn->var_len, &mnt->val.d, val_len); + break; + default: + break; + } + + mn->units = 1 + ((sizeof(match_node) - 1 + mn->var_len + val_len) + / sizeof(match_node)); + if (val_len && mnt->val_type == TYPE_STRING) { + mn->val_len = val_len - 1; + } else { + mn->val_len = val_len; + } + return mn + mn->units; +} + + +static match_node* inorder_traverse(unsigned char parent, match_node **offsets, + match_node *mn, match_node_tmp *root, + match_node_tmp *mnt) +{ + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + unsigned char idx = mnt->val_len; + mnt->val_len = 0; + mn = inorder_traverse(idx, offsets, mn, root, root + mnt->fi); + offsets[idx] = mn; + mn = copy_node(parent, mn, mnt); + } else { + mn = copy_node(parent, mn, mnt); + } + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + mn = inorder_traverse(parent, offsets, mn, root, root + mnt->ai); + } + return mn; +} + + +static size_t get_matcher_bytes(match_node_tmp nodes[], size_t size) +{ + size_t len = 0; + for (unsigned i = 0; i < size; ++i) { + size_t val_len = 0; + switch (nodes[i].val_type) { + case TYPE_STRING: + val_len = nodes[i].val_len + 1; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + break; + default: + break; + } + + size_t var_len = 0; + if (nodes[i].id == LSB_PB_FIELDS) { + var_len = nodes[i].var_len; + } + + len += (sizeof(match_node) * 2 + val_len + var_len - 1) + / sizeof(match_node) * sizeof(match_node); + + if (nodes[i].op == OP_OR || nodes[i].op == OP_AND) { + // squirrel away the position for the short-circuit calculation + nodes[i].val_len = i; + } + } + return len; +} + + +static void make_tree(match_node_tmp nodes[], size_t size) +{ + // turn the postfix stack into a traversable tree + match_node_tmp *stack[size]; + memset(stack, 0, sizeof(stack)); + int top = 0; + for (unsigned i = 0; i < size; ++i) { + if (nodes[i].op != OP_AND && nodes[i].op != OP_OR) { + stack[top++] = &nodes[i]; + } else { + nodes[i].ai = stack[--top] - nodes; + nodes[i].fi = stack[--top] - nodes; + stack[top++] = &nodes[i]; + } + } +} + + +static lsb_message_matcher* make_matcher(match_node_tmp nodes[], size_t size) +{ + lsb_message_matcher *mm = malloc(sizeof(lsb_message_matcher)); + if (!mm) { return NULL; } + + mm->bytes = get_matcher_bytes(nodes, size); + mm->nodes = calloc(mm->bytes, 1); + if (!mm->nodes) { + free(mm); + return NULL; + } + + match_node *offsets[size]; + memset(offsets, 0, sizeof(offsets)); + + inorder_traverse(size, offsets, mm->nodes, nodes, nodes + (size - 1)); + + // populate the short-circuit offsets + match_node *e = mm->nodes + (mm->bytes / sizeof(match_node)); + for (match_node *p = mm->nodes; p < e;){ + if (p->op == OP_AND || p->op == OP_OR) { + if (p->u.off < size) { + p->u.off = offsets[p->u.off] - mm->nodes; + } else { + p->u.off = mm->bytes / sizeof(match_node); + } + } + p += p->units; + } + return mm; +} + + +lsb_message_matcher* lsb_create_message_matcher(const char *exp) +{ + if (!exp) { return NULL; } + + lsb_message_matcher *mm = NULL; + yycontext yy; + memset(&yy, 0, sizeof(yy)); + yy.ctx.is.s = exp; + yy.ctx.is.size = strlen(exp); + int ret = yyparse(&yy); + if (ret) { + make_tree(yy.ctx.out.a, yy.ctx.out.pos); + mm = make_matcher(yy.ctx.out.a, yy.ctx.out.pos); + } + free(yy.ctx.out.a); + free(yy.ctx.ops.a); + yyrelease(&yy); + return mm; +} + diff --git a/src/util/heka_message_matcher_parser.leg b/src/util/heka_message_matcher_parser.leg new file mode 100644 index 0000000..0c46d3c --- /dev/null +++ b/src/util/heka_message_matcher_parser.leg @@ -0,0 +1,564 @@ +%{ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight message matcher parser @file */ + +#include +#include +#include +#include +#include + +#include "heka_message_matcher_impl.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" + +#ifndef _MSC_VER +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wunused-function" +#else +#pragma warning( disable : 4267 4244 ) +#endif + +typedef struct match_node_tmp { + uint8_t id; + uint8_t op; + uint8_t val_mod; + uint8_t val_type; + uint8_t val_len; + uint8_t var_len; + uint8_t fi; // left node index for logical op + uint8_t ai; // right node index for logical op + char *var; + + union { + char *s; + double d; + } val; +} match_node_tmp; + + +typedef struct match_node_array { + match_node_tmp *a; + int pos; + int size; +} match_node_array; + + +typedef struct input_string { + const char *s; + size_t pos; + size_t size; +} input_string; + + +typedef struct context { + match_node_array out; + match_node_array ops; + match_node_tmp mn; + struct tm tms; + int cond_cnt; + input_string is; +} context; + + +#define YY_PARSE(T) static T +#define YY_CTX_LOCAL 1 +#define YY_CTX_MEMBERS \ + context ctx; + +#define YY_INPUT(yy, buf, result, max_size) \ +{ \ + input_string *is = &yy->ctx.is; \ + result = is->size - is->pos; \ + if (result > 0) { \ + if (max_size >= result) { \ + memcpy(buf, &is->s[is->pos], result); \ + is->pos += result; \ + } else { \ + memcpy(buf, &is->s[is->pos], max_size); \ + is->pos += max_size; \ + result = max_size; \ + } \ + } \ +} + + +static void init_match_node(match_node_tmp *mn) +{ + memset(mn, 0, sizeof(match_node_tmp)); +} + + +static void move_match_node(match_node_tmp *dest, match_node_tmp *src) +{ + memcpy(dest, src, sizeof(match_node_tmp)); + init_match_node(src); // dest now owns the memory, wipe the pointers +} + + +static void realloc_mna(match_node_array *mna) +{ + size_t bytes = sizeof(match_node_tmp) * ++mna->size; + match_node_tmp *tmp = realloc(mna->a, bytes); + if (tmp) { + mna->a = tmp; + init_match_node(&mna->a[mna->size - 1]); + } else { + fprintf(stderr, "realloc failed\n"); + exit(1); + } +} + + +static void push_output(context *ctx, match_node_tmp *mn) +{ + if (!ctx->out.a || ctx->out.pos == ctx->out.size) { + realloc_mna(&ctx->out); + } + move_match_node(&ctx->out.a[ctx->out.pos++], mn); +} + + +static void push_op(context *ctx, match_operation op) +{ + if (!ctx->ops.a) { + realloc_mna(&ctx->ops); + ctx->ops.a[ctx->ops.pos++].op = op; + return; + } + + if (op == OP_OPEN || op > ctx->ops.a[ctx->ops.pos - 1].op) { + if (ctx->ops.pos == ctx->ops.size) { + realloc_mna(&ctx->ops); + } + ctx->ops.a[ctx->ops.pos++].op = op; + } else { + push_output(ctx, &ctx->ops.a[ctx->ops.pos - 1]); + ctx->ops.a[ctx->ops.pos - 1].op = op; + } +} + + +static void pop_to_paren(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) break; + push_output(ctx, op); + } +} + + +static void pop_all_ops(context *ctx) +{ + for (; ctx->ops.pos > 0; --ctx->ops.pos) { + match_node_tmp *op = &ctx->ops.a[ctx->ops.pos - 1]; + if (op->op == OP_OPEN) continue; + push_output(ctx, op); + } +} + + +static void update_date(context *ctx, int year, int mon, int day) +{ + ctx->tms.tm_isdst = -1; + ctx->tms.tm_year = year - 1900; + ctx->tms.tm_mon = mon - 1; + ctx->tms.tm_mday = day; +} + + +static void update_time(context *ctx, int hour, int minute, int sec) +{ + ctx->tms.tm_hour = hour; + ctx->tms.tm_min = minute; + ctx->tms.tm_sec = sec; +} + + +static void update_offset(context *ctx, char sign, int hour, int minute) +{ + ctx->mn.val.d += (hour * 3600 + minute * 60) * (sign == '-' ? -1 : 1); +} + + +static void set_field(context *ctx, char *name) +{ + ctx->mn.id = LSB_PB_FIELDS; + ctx->mn.var_len = strlen(name); + ctx->mn.var = malloc(ctx->mn.var_len + 1); + if (!ctx->mn.var) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.var, name, ctx->mn.var_len + 1); +} + + +static void set_timestamp(context *ctx) +{ + ctx->mn.id = LSB_PB_TIMESTAMP; + ctx->mn.val_type = TYPE_NUMERIC; + if (ctx->tms.tm_isdst == -1) { + ctx->mn.val.d += mktime(&ctx->tms); + ctx->mn.val.d *= 1e9; + } + memset(&ctx->tms, 0, sizeof(struct tm)); +} + + +static void set_numeric_value(context *ctx, char *s) +{ + ctx->mn.val_type = TYPE_NUMERIC; + ctx->mn.val.d = strtod(s, NULL); +} + + +static void set_string_value(context *ctx, char *s) +{ + ctx->mn.val_type = TYPE_STRING; + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { + ++i; + } + s[j] = s[i]; + } + s[j] = 0; + + ctx->mn.val_len = j; + ctx->mn.val.s = malloc(j + 1); + if (!ctx->mn.val.s) { + fprintf(stderr, "malloc failed\n"); + exit(1); + } + memcpy(ctx->mn.val.s, s, j + 1); +} + + +static void set_match_mod(context *ctx) +{ + if (ctx->mn.val_mod == PATTERN_MOD_NONE + && strpbrk(ctx->mn.val.s, "^$*+?.[%-") == NULL) { // literal + ctx->mn.val_mod = PATTERN_MOD_ESC; + } +} + + +static bool check_string_len(char *s) +{ + int i, j; + for (i = 0, j = 0; s[i]; ++i, ++j) { + if (s[i] == '\\' + && (s[i + 1] == '"' || s[i + 1] == '\'' || s[i + 1] == '\\')) { + ++i; + } + } + return (j > UCHAR_MAX) ? false : true; +} + + +static int cond_cnt(context *ctx) +{ + return (++ctx->cond_cnt * 2 + 1 > UCHAR_MAX) ? 0 : 1; +} + +%} + +match = sp ored eol {pop_all_ops(&yy->ctx)} +ored = anded (or anded)* sp +anded = expr (and expr)* sp +expr = test | open ored close +test = ( uuid + | timestamp + | optional_string_headers + | severity + | pid + | field_test + | boolean_test + ) {push_output(&yy->ctx, &yy->ctx.mn)} sp + +op_eq = "==" sp {yy->ctx.mn.op = OP_EQ} +op_ne = "!=" sp {yy->ctx.mn.op = OP_NE} +op_seq = "=~" sp {yy->ctx.mn.op = OP_RE} +op_sne = "!~" sp {yy->ctx.mn.op = OP_NRE} +op_gte = ">=" sp {yy->ctx.mn.op = OP_GTE} +op_gt = ">" sp {yy->ctx.mn.op = OP_GT} +op_lte = "<=" sp {yy->ctx.mn.op = OP_LTE} +op_lt = "<" sp {yy->ctx.mn.op = OP_LT} + +relational = op_eq + | op_ne + | op_gte + | op_gt + | op_lte + | op_lt + +boolean_test = true {yy->ctx.mn.op = OP_TRUE} + | false {yy->ctx.mn.op = OP_FALSE} +boolean = true {yy->ctx.mn.val_type = TYPE_TRUE} + | false {yy->ctx.mn.val_type = TYPE_FALSE} + +and = "&&" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_AND)} sp +or = "||" &{cond_cnt(&yy->ctx)} {push_op(&yy->ctx, OP_OR)} sp +open = "(" {push_op(&yy->ctx, OP_OPEN)} sp +close = ")" {pop_to_paren(&yy->ctx)} sp + +optional_string_headers = string_headers sp (relational sp string_value | string_match | (op_eq | op_ne) sp nil) + +uuid = "Uuid" {yy->ctx.mn.id = LSB_PB_UUID} sp (relational sp string_value | string_match) +string_headers = "Type" {yy->ctx.mn.id = LSB_PB_TYPE} + | "Logger" {yy->ctx.mn.id = LSB_PB_LOGGER} + | "Hostname" {yy->ctx.mn.id = LSB_PB_HOSTNAME} + | "EnvVersion" {yy->ctx.mn.id = LSB_PB_ENV_VERSION} + | "Payload" {yy->ctx.mn.id = LSB_PB_PAYLOAD} + +string_value = ( '"' < (("\\" ["\\]) | (!'"' .))* > '"' + | "'" < (("\\" ['\\]) | (!"'" .))* > "'" + ) &{check_string_len(yytext)} {set_string_value(&yy->ctx, yytext)} + +string_match = (op_seq | op_sne) sp string_value string_match_mod? {set_match_mod(&yy->ctx)} + +string_match_mod = "%" {yy->ctx.mn.val_mod = PATTERN_MOD_ESC} + +severity = "Severity" {yy->ctx.mn.id = LSB_PB_SEVERITY} sp relational sp numeric_value +pid = "Pid" {yy->ctx.mn.id = LSB_PB_PID} sp (relational sp numeric_value | (op_eq | op_ne) sp nil) + +numeric_value = < sign? number decimal? exponent? > {set_numeric_value(&yy->ctx, yytext)} +sign = [-+] +number = "0" + | [1-9] [0-9]* +decimal = "." [0-9]+ +exponent = [eE] sign? [0-9]+ + +field_test = fields sp ((relational sp (string_value | numeric_value)) + | string_match + | (op_eq | op_ne) sp (boolean | nil)) +fields = "Fields[" < [^\]]* > "]" {set_field(&yy->ctx, yytext)} f:index? {yy->ctx.mn.fi = f} a:index? {yy->ctx.mn.ai = a} +index = "[" < zero_to_255 > "]" {$$ = atoi(yytext)} +zero_to_255 = "2" [0-5] [0-5] + | "1" [0-9] [0-9] + | [1-9] [0-9] + | [0-9] + +timestamp = ("Timestamp" sp relational sp (numeric_value | ts_quoted)) {set_timestamp(&yy->ctx)} +ts_quoted = '"' rfc3339 '"' | "'" rfc3339 "'" +fulldate = (y:year "-" m:month "-" d:day) {update_date(&yy->ctx, y, m, d)} +year = < [0-9] [0-9] [0-9] [0-9] > {$$ = atoi(yytext)} +month = ( < "0" [1-9] > + | < "1" [0-2] > + ) {$$ = atoi(yytext)} +day = ( + < [1-2] [0-9] > + | < "0" [1-9] > + | < "3" [0-1] > + ) {$$ = atoi(yytext)} + +rfc3339 = fulldate "T" fulltime +fulltime = partialtime timeoffset +partialtime = h:hour ":" m:minute ":" s:second second_frac? {update_time(&yy->ctx, h, m, s)} +timeoffset = "Z" {update_offset(&yy->ctx, '+', 0, 0)} + | < sign > h:hour m:minute {update_offset(&yy->ctx, yytext[0], h, m)} +hour = ( + < [0-1] [0-9] > + | < "2" [0-3] > + ) {$$ = atoi(yytext)} +minute = < [0-5] [0-9] > {$$ = atoi(yytext)} +second = ( + < [0-5] [0-9] > + | < "60" > + ) {$$ = atoi(yytext)} +second_frac = < decimal > {yy->ctx.mn.val.d += strtod(yytext, NULL)} + +nil = "NIL" {yy->ctx.mn.val_type = TYPE_NIL} sp +true = "TRUE" sp +false = "FALSE" sp +sp = [ \r\n\t]* +eol = !. + +%% + +static match_node* copy_node(unsigned char parent, match_node *mn, + match_node_tmp *mnt) +{ + mn->op = mnt->op; + mn->val_len = 0; + mn->var_len = 0; + mn->field_id = mnt->id; + mn->val_mod = mnt->val_mod; + mn->val_type = mnt->val_type; + + if (mn->op == OP_AND || mn->op == OP_OR) { + mn->u.off = parent; + } else { + mn->u.idx.f = mnt->fi; + mn->u.idx.a = mnt->ai; + } + + if (mnt->id == LSB_PB_FIELDS) { + mn->var_len = mnt->var_len; + } + if (mn->var_len) { + memcpy(mn->data, mnt->var, mn->var_len); // no NUL terminator + free(mnt->var); + mnt->var = NULL; + mnt->var_len = 0; + } + + size_t val_len = 0; + switch (mnt->val_type) { + case TYPE_STRING: + val_len = mnt->val_len + 1; + memcpy(mn->data + mn->var_len, mnt->val.s, val_len); + free(mnt->val.s); + mnt->val.s = NULL; + mnt->val_len = 0; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + memcpy(mn->data + mn->var_len, &mnt->val.d, val_len); + break; + default: + break; + } + + mn->units = 1 + ((sizeof(match_node) - 1 + mn->var_len + val_len) + / sizeof(match_node)); + if (val_len && mnt->val_type == TYPE_STRING) { + mn->val_len = val_len - 1; + } else { + mn->val_len = val_len; + } + return mn + mn->units; +} + + +static match_node* inorder_traverse(unsigned char parent, match_node **offsets, + match_node *mn, match_node_tmp *root, + match_node_tmp *mnt) +{ + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + unsigned char idx = mnt->val_len; + mnt->val_len = 0; + mn = inorder_traverse(idx, offsets, mn, root, root + mnt->fi); + offsets[idx] = mn; + mn = copy_node(parent, mn, mnt); + } else { + mn = copy_node(parent, mn, mnt); + } + if (mnt->id == 0 && (mnt->op == OP_AND || mnt->op == OP_OR)) { + mn = inorder_traverse(parent, offsets, mn, root, root + mnt->ai); + } + return mn; +} + + +static size_t get_matcher_bytes(match_node_tmp nodes[], size_t size) +{ + size_t len = 0; + for (unsigned i = 0; i < size; ++i) { + size_t val_len = 0; + switch (nodes[i].val_type) { + case TYPE_STRING: + val_len = nodes[i].val_len + 1; + break; + case TYPE_NUMERIC: + val_len = sizeof(double); + break; + default: + break; + } + + size_t var_len = 0; + if (nodes[i].id == LSB_PB_FIELDS) { + var_len = nodes[i].var_len; + } + + len += (sizeof(match_node) * 2 + val_len + var_len - 1) + / sizeof(match_node) * sizeof(match_node); + + if (nodes[i].op == OP_OR || nodes[i].op == OP_AND) { + // squirrel away the position for the short-circuit calculation + nodes[i].val_len = i; + } + } + return len; +} + + +static void make_tree(match_node_tmp nodes[], size_t size) +{ + // turn the postfix stack into a traversable tree + match_node_tmp *stack[size]; + memset(stack, 0, sizeof(stack)); + int top = 0; + for (unsigned i = 0; i < size; ++i) { + if (nodes[i].op != OP_AND && nodes[i].op != OP_OR) { + stack[top++] = &nodes[i]; + } else { + nodes[i].ai = stack[--top] - nodes; + nodes[i].fi = stack[--top] - nodes; + stack[top++] = &nodes[i]; + } + } +} + + +static lsb_message_matcher* make_matcher(match_node_tmp nodes[], size_t size) +{ + lsb_message_matcher *mm = malloc(sizeof(lsb_message_matcher)); + if (!mm) { return NULL; } + + mm->bytes = get_matcher_bytes(nodes, size); + mm->nodes = calloc(mm->bytes, 1); + if (!mm->nodes) { + free(mm); + return NULL; + } + + match_node *offsets[size]; + memset(offsets, 0, sizeof(offsets)); + + inorder_traverse(size, offsets, mm->nodes, nodes, nodes + (size - 1)); + + // populate the short-circuit offsets + match_node *e = mm->nodes + (mm->bytes / sizeof(match_node)); + for (match_node *p = mm->nodes; p < e;){ + if (p->op == OP_AND || p->op == OP_OR) { + if (p->u.off < size) { + p->u.off = offsets[p->u.off] - mm->nodes; + } else { + p->u.off = mm->bytes / sizeof(match_node); + } + } + p += p->units; + } + return mm; +} + + +lsb_message_matcher* lsb_create_message_matcher(const char *exp) +{ + if (!exp) { return NULL; } + + lsb_message_matcher *mm = NULL; + yycontext yy; + memset(&yy, 0, sizeof(yy)); + yy.ctx.is.s = exp; + yy.ctx.is.size = strlen(exp); + int ret = yyparse(&yy); + if (ret) { + make_tree(yy.ctx.out.a, yy.ctx.out.pos); + mm = make_matcher(yy.ctx.out.a, yy.ctx.out.pos); + } + free(yy.ctx.out.a); + free(yy.ctx.ops.a); + yyrelease(&yy); + return mm; +} diff --git a/src/util/input_buffer.c b/src/util/input_buffer.c new file mode 100644 index 0000000..c17ecbc --- /dev/null +++ b/src/util/input_buffer.c @@ -0,0 +1,78 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream input buffer implementation @file */ + +#include "luasandbox/util/input_buffer.h" + +#include +#include +#include +#include + +#include "luasandbox/util/util.h" +#include "luasandbox/util/heka_message.h" + +lsb_err_value +lsb_init_input_buffer(lsb_input_buffer *b, size_t max_message_size) +{ + if (!b) return LSB_ERR_UTIL_NULL; + b->buf = NULL; + if (max_message_size == 0) return LSB_ERR_UTIL_PRANGE; + max_message_size += LSB_MAX_HDR_SIZE; + b->size = max_message_size < BUFSIZ ? max_message_size : BUFSIZ; + b->maxsize = max_message_size; + b->readpos = 0; + b->scanpos = 0; + b->msglen = 0; + b->buf = malloc(b->size); + return b->buf ? NULL : LSB_ERR_UTIL_OOM; +} + + +void lsb_free_input_buffer(lsb_input_buffer *b) +{ + if (!b) return; + + free(b->buf); + b->buf = NULL; + b->size = 0; + b->readpos = 0; + b->scanpos = 0; + b->msglen = 0; +} + + +lsb_err_value lsb_expand_input_buffer(lsb_input_buffer *b, size_t len) +{ + if (!b) return LSB_ERR_UTIL_NULL; + + if (b->scanpos != 0) { // shift the data to the beginning of the buffer + if (b->scanpos == b->readpos) { + b->scanpos = b->readpos = 0; + } else { + memmove(b->buf, b->buf + b->scanpos, b->readpos - b->scanpos); + b->readpos = b->readpos - b->scanpos; + b->scanpos = 0; + } + } + + if (b->readpos + len > b->size) { + size_t newsize = b->readpos + len; + if (newsize > b->maxsize) return LSB_ERR_UTIL_FULL; + + newsize = lsb_lp2(newsize); + if (newsize > b->maxsize) newsize = b->maxsize; + char *tmp = realloc(b->buf, newsize); + if (tmp) { + b->buf = tmp; + b->size = newsize; + } else { + return LSB_ERR_UTIL_OOM; + } + } + return NULL; +} diff --git a/src/util/output_buffer.c b/src/util/output_buffer.c new file mode 100644 index 0000000..8eff264 --- /dev/null +++ b/src/util/output_buffer.c @@ -0,0 +1,248 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Data stream output buffer implementation @file */ + +#include "luasandbox/util/output_buffer.h" + +#include +#include +#include +#include +#include +#include + +#include "luasandbox/util/util.h" + +#ifdef _MSC_VER +// To silence the +/-INFINITY warning +#pragma warning( disable : 4756 ) +#pragma warning( disable : 4056 ) +#endif + +lsb_err_value +lsb_init_output_buffer(lsb_output_buffer *b, size_t max_message_size) +{ + if (!b) return LSB_ERR_UTIL_NULL; + if (max_message_size && max_message_size < LSB_OUTPUT_SIZE) { + b->size = max_message_size; + } else { + b->size = LSB_OUTPUT_SIZE; + } + b->maxsize = max_message_size; + b->pos = 0; + b->buf = malloc(b->size); + return b->buf ? NULL : LSB_ERR_UTIL_OOM; +} + + +void lsb_free_output_buffer(lsb_output_buffer *b) +{ + if (!b) return; + free(b->buf); + b->buf = NULL; + b->size = 0; + b->pos = 0; +} + + +lsb_err_value lsb_expand_output_buffer(lsb_output_buffer *b, size_t needed) +{ + if (!b) return LSB_ERR_UTIL_NULL; + + if (needed <= b->size - b->pos) return NULL; + + if (b->maxsize && needed + b->pos > b->maxsize) { + return LSB_ERR_UTIL_FULL; + } + + size_t newsize = lsb_lp2(b->pos + needed); + if (b->maxsize && newsize > b->maxsize) { + newsize = b->maxsize; + } + + void *ptr = realloc(b->buf, newsize); + if (!ptr) { + return LSB_ERR_UTIL_OOM; + } + + b->buf = ptr; + b->size = newsize; + return NULL; +} + + +lsb_err_value lsb_outputc(lsb_output_buffer *b, char ch) +{ + if (!b) { + return LSB_ERR_UTIL_NULL; + } + lsb_err_value ret = lsb_expand_output_buffer(b, 2); + if (ret) return ret; + + b->buf[b->pos++] = ch; + b->buf[b->pos] = 0; + return NULL; +} + + +lsb_err_value lsb_outputf(lsb_output_buffer *b, const char *fmt, ...) +{ + if (!b || !fmt) { + return LSB_ERR_UTIL_NULL; + } + + va_list args; + int remaining = 0; + char *ptr = NULL, *old_ptr = NULL; + do { + ptr = b->buf + b->pos; + remaining = (int)(b->size - b->pos); + va_start(args, fmt); + int needed = vsnprintf(ptr, remaining, fmt, args); + va_end(args); + if (needed == -1) { + // Windows and Unix have different return values for this function + // -1 on Unix is a format error + // -1 on Windows means the buffer is too small and the required len + // is not returned + needed = remaining; + } + if (needed >= remaining) { + if (b->maxsize && (b->size >= b->maxsize + || b->pos + needed >= b->maxsize)) { + return LSB_ERR_UTIL_FULL; + } + size_t newsize = b->size * 2; + while ((size_t)needed >= newsize - b->pos) { + newsize *= 2; + } + if (b->maxsize && newsize > b->maxsize) { + newsize = b->maxsize; + } + void *p = malloc(newsize); + if (p != NULL) { + memcpy(p, b->buf, b->pos); + old_ptr = b->buf; + b->buf = p; + b->size = newsize; + } else { + return LSB_ERR_UTIL_OOM; + } + } else { + b->pos += needed; + break; + } + } while (1); + + free(old_ptr); + return NULL; +} + + +lsb_err_value lsb_outputs(lsb_output_buffer *b, const char *str, size_t len) +{ + if (!b) { + return LSB_ERR_UTIL_NULL; + } + lsb_err_value ret = lsb_expand_output_buffer(b, len + 1); + if (ret) return ret; + + memcpy(b->buf + b->pos, str, len); + b->pos += len; + b->buf[b->pos] = 0; + return ret; +} + + +lsb_err_value lsb_outputd(lsb_output_buffer *b, double d) +{ + if (!b) return LSB_ERR_UTIL_NULL; + + if (isnan(d)) { + return lsb_outputs(b, "nan", 3); + } + if (d == INFINITY) { + return lsb_outputs(b, "inf", 3); + } + if (d == -INFINITY) { + return lsb_outputs(b, "-inf", 4); + } + return lsb_outputfd(b, d); +} + + +lsb_err_value lsb_outputfd(lsb_output_buffer *b, double d) +{ + if (!b) return LSB_ERR_UTIL_NULL; + + if (d < INT_MIN || d > INT_MAX) { + return lsb_outputf(b, "%0.17g", d); + } + + const int precision = 8; + const unsigned magnitude = 100000000; + char buffer[20]; + char *p = buffer; + int negative = 0; + + if (d < 0) { + negative = 1; + d = -d; + } + + int number = (int)d; + double tmp = (d - number) * magnitude; + unsigned fraction = (unsigned)tmp; + double diff = tmp - fraction; + + if (diff > 0.5) { + ++fraction; + if (fraction >= magnitude) { + fraction = 0; + ++number; + } + } else if (diff == 0.5 && ((fraction == 0) || (fraction & 1))) { + // bankers rounding + ++fraction; + } + + // decimal fraction + if (fraction != 0) { + int nodigits = 1; + char c = 0; + for (int x = 0; x < precision; ++x) { + c = fraction % 10; + if (!(c == 0 && nodigits)) { + *p++ = c + '0'; + nodigits = 0; + } + fraction /= 10; + } + *p++ = '.'; + } + + // number + do { + *p++ = (number % 10) + '0'; + number /= 10; + } while (number > 0); + + lsb_err_value ret = lsb_expand_output_buffer(b, (p - buffer) + negative + 1); + if (!ret) { + if (negative) { + b->buf[b->pos++] = '-'; + } + + do { + --p; + b->buf[b->pos++] = *p; + } while (p != buffer); + + b->buf[b->pos] = 0; + } + return ret; +} diff --git a/src/util/protobuf.c b/src/util/protobuf.c new file mode 100644 index 0000000..d510586 --- /dev/null +++ b/src/util/protobuf.c @@ -0,0 +1,151 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** Generic protobuf utility functions @file */ + +#include "luasandbox/util/protobuf.h" + +#include +#include +#include + +const char* lsb_pb_read_key(const char *p, int *tag, int *wiretype) +{ + if (!p || !tag || !wiretype) return NULL; + + *wiretype = 7 & (unsigned char)*p; + *tag = (unsigned char)*p >> 3; + return ++p; +} + + +lsb_err_value lsb_pb_write_key(lsb_output_buffer *ob, unsigned char tag, + unsigned char wiretype) +{ + lsb_err_value ret = lsb_expand_output_buffer(ob, 1); + if (!ret) { + ob->buf[ob->pos++] = wiretype | (tag << 3); + } + return ret; +} + + +const char* lsb_pb_read_varint(const char *p, const char *e, long long *vi) +{ + if (!p || !e || !vi) { + return NULL; + } + + *vi = 0; + int i, shift = 0; + for (i = 0; p != e && i < LSB_MAX_VARINT_BYTES; ++i, ++p) { + *vi |= ((unsigned long long)*p & 0x7f) << shift; + shift += 7; + if ((*p & 0x80) == 0) break; + } + if (i == LSB_MAX_VARINT_BYTES || p == e) { + return NULL; + } + return ++p; +} + + +int lsb_pb_output_varint(char *buf, unsigned long long i) +{ + int pos = 0; + if (!buf) return pos; + + if (i == 0) { + buf[pos++] = 0; + return pos; + } + + while (i) { + buf[pos++] = (i & 0x7F) | 0x80; + i >>= 7; + } + buf[pos - 1] &= 0x7F; // end the varint + return pos; +} + + +lsb_err_value lsb_pb_write_varint(lsb_output_buffer *ob, unsigned long long i) +{ + lsb_err_value ret = lsb_expand_output_buffer(ob, LSB_MAX_VARINT_BYTES); + if (!ret) { + ob->pos += lsb_pb_output_varint(ob->buf + ob->pos, i); + } + return ret; +} + + +lsb_err_value lsb_pb_write_bool(lsb_output_buffer *ob, int i) +{ + lsb_err_value ret = lsb_expand_output_buffer(ob, 1); + if (!ret) { + if (i) { + ob->buf[ob->pos++] = 1; + } else { + ob->buf[ob->pos++] = 0; + } + } + return ret; +} + + +lsb_err_value lsb_pb_write_double(lsb_output_buffer *ob, double i) +{ + static const size_t needed = sizeof(double); + + lsb_err_value ret = lsb_expand_output_buffer(ob, needed); + if (!ret) { + // todo add big endian support if necessary + memcpy(&ob->buf[ob->pos], &i, needed); + ob->pos += needed; + } + return ret; +} + + +lsb_err_value +lsb_pb_write_string(lsb_output_buffer *ob, char tag, const char *s, size_t len) +{ + lsb_err_value ret = lsb_pb_write_key(ob, tag, LSB_PB_WT_LENGTH); + if (!ret) ret = lsb_pb_write_varint(ob, len); + if (!ret) ret = lsb_expand_output_buffer(ob, len); + if (!ret) { + memcpy(&ob->buf[ob->pos], s, len); + ob->pos += len; + } + return ret; +} + + +lsb_err_value lsb_pb_update_field_length(lsb_output_buffer *ob, size_t len_pos) +{ + if (len_pos >= ob->pos) { + return LSB_ERR_UTIL_PRANGE; + } + + size_t len = ob->pos - len_pos - 1; + if (len < 128) { + ob->buf[len_pos] = (char)len; + return NULL; + } + size_t l = len, cnt = 0; + while (l) { + l >>= 7; + ++cnt; // compute the number of bytes needed for the varint length + } + size_t needed = cnt - 1; + lsb_err_value ret = lsb_expand_output_buffer(ob, needed); + if (!ret) { + ob->pos += needed; + memmove(&ob->buf[len_pos + cnt], &ob->buf[len_pos + 1], len); + lsb_pb_output_varint(ob->buf + len_pos, len); + } + return ret; +} diff --git a/src/util/running_stats.c b/src/util/running_stats.c new file mode 100644 index 0000000..c3fe00d --- /dev/null +++ b/src/util/running_stats.c @@ -0,0 +1,42 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Running stats implementation @file */ + +#include "luasandbox/util/running_stats.h" + +#include + +void lsb_init_running_stats(lsb_running_stats *s) +{ + s->count = 0.0; + s->mean = 0.0; + s->sum = 0.0; +} + + +void lsb_update_running_stats(lsb_running_stats *s, double d) +{ + if (!isfinite(d)) return; + + double old_mean = s->mean; + double old_sum = s->sum; + + if (++s->count == 1) { + s->mean = d; + } else { + s->mean = old_mean + (d - old_mean) / s->count; + s->sum = old_sum + (d - old_mean) * (d - s->mean); + } +} + + +double lsb_sd_running_stats(lsb_running_stats *s) +{ + if (s->count < 2) return 0.0; + return sqrt(s->sum / (s->count - 1)); +} + diff --git a/src/util/string.c b/src/util/string.c new file mode 100644 index 0000000..edfe5b0 --- /dev/null +++ b/src/util/string.c @@ -0,0 +1,83 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** String functions @file */ + +#include +#include +#include + +#include "luasandbox/util/string.h" + +void lsb_init_const_string(lsb_const_string *s) +{ + s->s = NULL; + s->len = 0; +} + + +char* lsb_lua_string_unescape(char *d, const char *s, size_t *dlen) +{ + if (!s || !d || !dlen || *dlen <= strlen(s)) { + return NULL; + } + + int x = 0; + int y = 0; + while (s[x]) { + switch (s[x]) { + case '\\': + ++x; + switch (s[x]) { + case 'a': + d[y++] = '\a'; break; + case 'b': + d[y++] = '\b'; break; + case 'f': + d[y++] = '\f'; break; + case 'n': + d[y++] = '\n'; break; + case 'r': + d[y++] = '\r'; break; + case 't': + d[y++] = '\t'; break; + case 'v': + d[y++] = '\v'; break; + default: + if (!isdigit(s[x])) { + switch (s[x]) { + case '"': + case '\'': + case '?': + case '\\': + break; + default: + return NULL; + } + d[y++] = s[x]; + } else { /* \xxx */ + int n = 0; + int c = 0; + do { + c = 10 * c + (s[x++] - '0'); + } while (++n < 3 && isdigit(s[x])); + if (c > UCHAR_MAX) return NULL; + d[y++] = (char)c; + --x; + } + } + ++x; + break; + + default: + d[y++] = s[x++]; + break; + } + } + d[y] = 0; + *dlen = y; + return d; +} diff --git a/src/util/string_matcher.c b/src/util/string_matcher.c new file mode 100644 index 0000000..69d3230 --- /dev/null +++ b/src/util/string_matcher.c @@ -0,0 +1,307 @@ +/* +** Modified Lua lstrlib.c for the Lua sandbox message matcher pattern-matching +* +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. +* +* Permission is hereby granted, free of charge, to any person obtaining +* a copy of this software and associated documentation files (the +* "Software"), to deal in the Software without restriction, including +* without limitation the rights to use, copy, modify, merge, publish, +* distribute, sublicense, and/or sell copies of the Software, and to +* permit persons to whom the Software is furnished to do so, subject to +* the following conditions: +* +* The above copyright notice and this permission notice shall be +* included in all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +******************************************************************************/ + +#include "luasandbox/util/string_matcher.h" + +#include +#include + +/* macro to `unsign' a character */ +#define uchar(c) ((unsigned char)(c)) +#define L_ESC '%' + +typedef struct MatchState { + const char *src_init; /* init of source string */ + const char *src_end; /* end (`\0') of source string */ +} MatchState; + + +static const char* classend(const char *p) +{ + switch (*p++) { + case L_ESC: + { + if (*p == '\0') return NULL; // error pattern ends with a % + return p + 1; + } + case '[': + { + if (*p == '^') p++; + do { /* look for a `]' */ + if (*p == '\0') return NULL; // error missing closing ] + if (*(p++) == L_ESC && *p != '\0') p++; /* skip escapes (e.g. `%]') */ + }while (*p != ']'); + return p + 1; + } + default: + { + return p; + } + } +} + + +static int match_class(int c, int cl) +{ + int res; + switch (tolower(cl)) { + case 'a' : + res = isalpha(c); break; + case 'c' : + res = iscntrl(c); break; + case 'd' : + res = isdigit(c); break; + case 'l' : + res = islower(c); break; + case 'p' : + res = ispunct(c); break; + case 's' : + res = isspace(c); break; + case 'u' : + res = isupper(c); break; + case 'w' : + res = isalnum(c); break; + case 'x' : + res = isxdigit(c); break; + case 'z' : + res = (c == 0); break; + default: + return (cl == c); + } + return (islower(cl) ? res : !res); +} + + +static int matchbracketclass(int c, const char *p, const char *ec) +{ + int sig = 1; + if (*(p + 1) == '^') { + sig = 0; + p++; /* skip the `^' */ + } + while (++p < ec) { + if (*p == L_ESC) { + p++; + if (match_class(c, uchar(*p))) return sig; + } else if ((*(p + 1) == '-') && (p + 2 < ec)) { + p += 2; + if (uchar(*(p - 2)) <= c && c <= uchar(*p)) return sig; + } else if (uchar(*p) == c) return sig; + } + return !sig; +} + + +static int singlematch(int c, const char *p, const char *ep) +{ + switch (*p) { + case '.': + return 1; /* matches any char */ + case L_ESC: + return match_class(c, uchar(*(p + 1))); + case '[': + return matchbracketclass(c, p, ep - 1); + default: + return (uchar(*p) == c); + } +} + + +static const char* match(MatchState *ms, const char *s, const char *p); + + +static const char* matchbalance(MatchState *ms, const char *s, + const char *p) +{ + if (*p == 0 || *(p + 1) == 0) return NULL; // ubalanced pattern; + if (*s != *p) return NULL; + else { + int b = *p; + int e = *(p + 1); + int cont = 1; + while (++s < ms->src_end) { + if (*s == e) { + if (--cont == 0) return s + 1; + } else if (*s == b) cont++; + } + } + return NULL; /* string ends out of balance */ +} + + +static const char* max_expand(MatchState *ms, const char *s, + const char *p, const char *ep) +{ + ptrdiff_t i = 0; /* counts maximum expand for item */ + while ((s + i) < ms->src_end && singlematch(uchar(*(s + i)), p, ep)) i++; + /* keeps trying to match with the maximum repetitions */ + while (i >= 0) { + const char *res = match(ms, (s + i), ep + 1); + if (res) return res; + i--; /* else didn't match; reduce 1 repetition to try again */ + } + return NULL; +} + + +static const char* min_expand(MatchState *ms, const char *s, + const char *p, const char *ep) +{ + for (;;) { + const char *res = match(ms, s, ep + 1); + if (res != NULL) return res; + else if (s < ms->src_end && singlematch(uchar(*s), p, ep)) s++; /* try with one more repetition */ + else return NULL; + } +} + + +static const char* match(MatchState *ms, const char *s, const char *p) +{ +init: /* using goto's to optimize tail recursion */ + switch (*p) { + case L_ESC: + { + switch (*(p + 1)) { + case 'b': + { /* balanced string? */ + s = matchbalance(ms, s, p + 2); + if (s == NULL) return NULL; + p += 4; goto init; /* else return match(ms, s, p+4); */ + } + case 'f': + { /* frontier? */ + const char *ep; char previous; + p += 2; + if (*p != '[') return NULL; // missing [ after %f + ep = classend(p); /* points to what is next */ + if (ep == NULL) return NULL; + previous = (s == ms->src_init) ? '\0' : *(s - 1); + if (matchbracketclass(uchar(previous), p, ep - 1) || + !matchbracketclass(uchar(*s), p, ep - 1)) return NULL; + p = ep; goto init; /* else return match(ms, s, ep); */ + } + default: + { + goto dflt; /* case default */ + } + } + } + case '\0': + { /* end of pattern */ + return s; /* match succeeded */ + } + case '$': + { + if (*(p + 1) == '\0') /* is the `$' the last char in pattern? */ + return (s == ms->src_end) ? s : NULL; /* check end of string */ + else goto dflt; + } + default: + dflt: + { /* it is a pattern item */ + const char *ep = classend(p); /* points to what is next */ + if (ep == NULL) return NULL; + + int m = s < ms->src_end && singlematch(uchar(*s), p, ep); + switch (*ep) { + case '?': + { /* optional */ + const char *res; + if (m && ((res = match(ms, s + 1, ep + 1)) != NULL)) return res; + p = ep + 1; goto init; /* else return match(ms, s, ep+1); */ + } + case '*': + { /* 0 or more repetitions */ + return max_expand(ms, s, p, ep); + } + case '+': + { /* 1 or more repetitions */ + return (m ? max_expand(ms, s + 1, p, ep) : NULL); + } + case '-': + { /* 0 or more repetitions (minimum) */ + return min_expand(ms, s, p, ep); + } + default: + { + if (!m) return NULL; + s++; p = ep; goto init; /* else return match(ms, s+1, ep); */ + } + } + } + } +} + + +static const char* lmemfind(const char *s1, size_t l1, + const char *s2, size_t l2) +{ + if (l2 == 0) return s1; /* empty strings are everywhere */ + else if (l2 > l1) return NULL; /* avoids a negative `l1' */ + else { + const char *init; /* to search for a `*s2' inside `s1' */ + l2--; /* 1st char will be checked by `memchr' */ + l1 = l1 - l2; /* `s2' cannot be found after that */ + while (l1 > 0 && (init = (const char *)memchr(s1, *s2, l1)) != NULL) { + init++; /* 1st char is already checked */ + if (memcmp(init, s2 + 1, l2) == 0) return init - 1; + else { /* correct `l1' and `s1' to try again */ + l1 -= init - s1; + s1 = init; + } + } + return NULL; /* not found */ + } +} + + + +bool lsb_string_match(const char *s, size_t len, const char *p) +{ + if (!s || !p) { return false; } + MatchState ms; + int anchor = (*p == '^') ? (p++, 1) : 0; + const char *s1 = s; + ms.src_init = s; + ms.src_end = s + len; + do { + const char *res; + if ((res = match(&ms, s1, p)) != NULL) { + return true; + } + } while (s1++ < ms.src_end && !anchor); + return false; +} + + +bool lsb_string_find(const char *s, size_t ls, const char *p, size_t lp) +{ + if (!s || !p) { return false; } + if (lmemfind(s, ls, p, lp)) { + return true; + } + return false; +} diff --git a/src/util/test/CMakeLists.txt b/src/util/test/CMakeLists.txt new file mode 100644 index 0000000..1ac3bf0 --- /dev/null +++ b/src/util/test/CMakeLists.txt @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +add_executable(test_input_buffer test_input_buffer.c) +target_link_libraries(test_input_buffer luasandboxutil) +add_test(NAME test_input_buffer COMMAND test_input_buffer) + +add_executable(test_output_buffer test_output_buffer.c) +target_link_libraries(test_output_buffer luasandboxutil) +add_test(NAME test_output_buffer COMMAND test_output_buffer) + +add_executable(test_protobuf test_protobuf.c) +target_link_libraries(test_protobuf luasandboxutil) +add_test(NAME test_protobuf COMMAND test_protobuf) + +add_executable(test_string_matcher test_string_matcher.c) +target_link_libraries(test_string_matcher luasandboxutil) +add_test(NAME test_string_matcher COMMAND test_string_matcher) + +add_executable(test_running_stats test_running_stats.c) +target_link_libraries(test_running_stats luasandboxutil) +add_test(NAME test_running_stats COMMAND test_running_stats) + +add_executable(test_util test_util.c) +target_link_libraries(test_util luasandboxutil) +add_test(NAME test_util COMMAND test_util) + +add_executable(test_heka_message test_heka_message.c) +target_link_libraries(test_heka_message luasandboxutil) +add_test(NAME test_heka_message COMMAND test_heka_message) + +add_executable(test_heka_message_matcher test_heka_message_matcher.c) +target_link_libraries(test_heka_message_matcher luasandboxutil) +add_test(NAME test_heka_message_matcher COMMAND test_heka_message_matcher) + +add_executable(test_string test_string.c) +target_link_libraries(test_string luasandboxutil) +add_test(NAME test_string COMMAND test_string) + +if(WIN32) + set(LIBRARY_PATHS "${CMAKE_BINARY_DIR}/src/util") + set_tests_properties(test_input_buffer PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_output_buffer PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_protobuf PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_string_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_running_stats PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_util PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_heka_message PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_heka_message_matcher PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) + set_tests_properties(test_string PROPERTIES ENVIRONMENT PATH=${LIBRARY_PATHS}) +endif() diff --git a/src/util/test/test_heka_message.c b/src/util/test/test_heka_message.c new file mode 100644 index 0000000..a509f28 --- /dev/null +++ b/src/util/test/test_heka_message.c @@ -0,0 +1,408 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief heka_message unit tests @file */ + +#include +#include +#include + +#include + +#include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/heka_message.h" + +#define TEST_UUID "\x0a\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" +#define TEST_NS "\x10\x01" + +// {Uuid="" Timestamp = 1e9, Type="type", Logger="logger", Payload="payload", EnvVersion="env_version", Hostname="hostname", Severity=9, Fields = {number=1,numbers={value={1,2,3}, representation="count"},string="string",strings={"s1","s2","s3"}, bool=true, bools={true,false,false}}} +static char pb[] = "\x0a\x10\x73\x1e\x36\x84\xec\x25\x42\x76\xa4\x01\x79\x6f\x17\xdd\x20\x63\x10\x80\x94\xeb\xdc\x03\x1a\x04\x74\x79\x70\x65\x22\x06\x6c\x6f\x67\x67\x65\x72\x28\x09\x32\x07\x70\x61\x79\x6c\x6f\x61\x64\x3a\x0b\x65\x6e\x76\x5f\x76\x65\x72\x73\x69\x6f\x6e\x4a\x08\x68\x6f\x73\x74\x6e\x61\x6d\x65\x52\x13\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x03\x39\x00\x00\x00\x00\x00\x00\xf0\x3f\x52\x2c\x0a\x07\x6e\x75\x6d\x62\x65\x72\x73\x10\x03\x1a\x05\x63\x6f\x75\x6e\x74\x3a\x18\x00\x00\x00\x00\x00\x00\xf0\x3f\x00\x00\x00\x00\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x08\x40\x52\x0e\x0a\x05\x62\x6f\x6f\x6c\x73\x10\x04\x42\x03\x01\x00\x00\x52\x0a\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x40\x01\x52\x10\x0a\x06\x73\x74\x72\x69\x6e\x67\x22\x06\x73\x74\x72\x69\x6e\x67\x52\x15\x0a\x07\x73\x74\x72\x69\x6e\x67\x73\x22\x02\x73\x31\x22\x02\x73\x32\x22\x02\x73\x33"; + + +struct log_message { + const char *component; + int severity; + char msg[1024]; +}; + +static struct log_message lm = { .component = NULL, .severity = 0, .msg = { 0 } }; + +void log_cb(void *context, const char *component, int severity, const char *fmt, + ...) +{ + (void)context; + lm.component = component; + lm.severity = severity; + va_list args; + va_start(args, fmt); + vsnprintf(lm.msg, sizeof lm.msg, fmt, args); + va_end(args); +} +static lsb_logger logger = { .context = NULL, .cb = log_cb }; + +static char* test_stub() +{ + return NULL; +} + + +static char* test_init() +{ + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 10), "failed"); + lsb_free_heka_message(&m); + lsb_free_heka_message(NULL); + return NULL; +} + + +static char* test_init_failure() +{ + lsb_heka_message m; + mu_assert(lsb_init_heka_message(&m, 0), "suceeded"); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_decode() +{ + +#define add_string(str) {.s = str, .len = sizeof str - 1}, + lsb_const_string tests[] = { + add_string(TEST_UUID TEST_NS) // required fields + add_string(TEST_UUID TEST_NS "\x1a\x04" "Type") + add_string(TEST_UUID TEST_NS "\x22\x06" "Logger") + add_string(TEST_UUID TEST_NS "\x28\x07") // Severity + add_string(TEST_UUID TEST_NS "\x32\x07" "Payload") + add_string(TEST_UUID TEST_NS "\x3a\x0a" "EnvVersion") + add_string(TEST_UUID TEST_NS "\x40\x11") // Pid + add_string(TEST_UUID TEST_NS "\x4a\x08" "Hostname") + add_string(TEST_UUID TEST_NS "\x52\x11\x0a\x03" "foo\x10\x00\x1a\x03" "rep\x22\x03" "bar") // string + add_string(TEST_UUID TEST_NS "\x52\x11\x0a\x03" "foo\x10\x01\x1a\x03" "rep\x2a\x03" "bar") // bytes + add_string(TEST_UUID TEST_NS "\x52\x0e\x0a\x03" "foo\x10\x02\x1a\x03" "rep\x30\x11") // integer + add_string(TEST_UUID TEST_NS "\x52\x15\x0a\x03" "foo\x10\x03\x1a\x03" "rep\x39\x00\x00\x00\x00\x00\x00\x00\x00") // double + add_string(TEST_UUID TEST_NS "\x52\x0e\x0a\x03" "foo\x10\x04\x1a\x03" "rep\x40\x01") // bool + }; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 1), "failed"); + for (unsigned i = 0; i < sizeof tests / sizeof(lsb_const_string); ++i){ + bool ok = lsb_decode_heka_message(&m, tests[i].s, tests[i].len, &logger); + mu_assert(ok, "test: %d failed err: %s", i, lm.msg); + } + mu_assert(!lsb_decode_heka_message(NULL, NULL, 0, NULL), "succeeded"); + mu_assert(!lsb_decode_heka_message(&m, NULL, 0, NULL), "succeeded"); + mu_assert(!lsb_decode_heka_message(&m, tests[0].s, 0, NULL), "succeeded"); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_decode_failure() +{ + struct decode_failure { + const char *s; + const char *e; + }; + + struct decode_failure tests[] = { + { TEST_NS, "missing Uuid" } // required test + , { TEST_UUID, "missing Timestamp" } // require test + , { "\x0a\x01\xff", "tag:1 wiretype:2 position:0" } // invalid UUID length + , { "\xf0", "tag:30 wiretype:0 position:0" } // unknown message tag + , { "\x0b", "tag:1 wiretype:3 position:0" } // uuid invalid wiretype + , { "\x11", "tag:2 wiretype:1 position:0" } // timestamp invalid wiretype + , { "\x1b", "tag:3 wiretype:3 position:0" } // type invalid wiretype + , { "\x23", "tag:4 wiretype:3 position:0" } // logger invalid wiretype + , { "\x2b", "tag:5 wiretype:3 position:0" } // severity invalid wiretype + , { "\x33", "tag:6 wiretype:3 position:0" } // payload invalid wiretype + , { "\x3b", "tag:7 wiretype:3 position:0" } // env_version invalid wiretype + , { "\x43", "tag:8 wiretype:3 position:0" } // pid invalid wiretype + , { "\x4b", "tag:9 wiretype:3 position:0" } // hostname invalid wiretype + , { "\x53", "tag:10 wiretype:3 position:0" } // fields invalid wiretype + , { "\x52\x10", "tag:10 wiretype:2 position:0" } // invalid field length + , { "\x52\x01\x0b", "tag:10 wiretype:2 position:0" } // invalid name wiretype + , { "\x52\x01\x13", "tag:10 wiretype:2 position:0" } // invalid value_type wiretype + , { "\x52\x01\x1b", "tag:10 wiretype:2 position:0" } // invalid representation wiretype + , { "\x52\x01\x23", "tag:10 wiretype:2 position:0" } // invalid string wiretype + , { "\x52\x01\x2b", "tag:10 wiretype:2 position:0" } // invalid bytes wiretype + , { "\x52\x01\x33", "tag:10 wiretype:2 position:0" } // invalid integer wiretype + , { "\x52\x01\x3b", "tag:10 wiretype:2 position:0" } // invalid bool wiretype + , { "\x52\x01\x43", "tag:10 wiretype:2 position:0" } // invalid double wiretype + , { "\x52\x01\x4b", "tag:10 wiretype:2 position:0" } // unknown field tag + , { "\x52\xc\x10\x00\x1a\x03" "rep\x22\x03" "bar", "tag:10 wiretype:2 position:0" } // no name + , { "\x10\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "tag:2 wiretype:0 position:0" } // invalid varint + , { "\x0a\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", "tag:1 wiretype:2 position:0" } // invalid varint length + }; + + lsb_heka_message m; + mu_assert(!lsb_init_heka_message(&m, 10), "failed"); + for (unsigned i = 0; i < sizeof(tests) / sizeof(struct decode_failure); ++i) { + bool ok = lsb_decode_heka_message(&m, tests[i].s, strlen(tests[i].s), &logger); + mu_assert(!ok, "test: %u no error generated", i); + mu_assert(!strcmp(lm.msg, tests[i].e), "test: %u expected: %s received: %s", + i, tests[i].e, lm.msg); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_find_message() +{ + struct find_message { + lsb_const_string s; + bool b; + size_t d; + }; + +#define ONE_TWENTY_SIX "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +#define add_input(str, result, discard) {.s = {.s = str, .len = sizeof str - 1}, .b = result, .d = discard}, + struct find_message tests[] = { + add_input("\x1e\x02\x08\x14\x1f" TEST_UUID TEST_NS, true, 0) // full message + add_input("\x1e\x80\x08\x14" ONE_TWENTY_SIX "\x1f" TEST_UUID TEST_NS, true, 0) // large header + add_input("\x1e", false, 0) + add_input("\x02", false, 0) + add_input("\x08", false, 0) + add_input("\x14", false, 0) + add_input("\x1f", false, 0) + add_input(TEST_UUID, false, 0) + add_input(TEST_NS "\x1e\x02\x08\x14\x1f", true, 0) // completion of an incremental message and the start of another + add_input(TEST_UUID TEST_NS, true, 0) + add_input(TEST_UUID TEST_NS, false, 20) // no framing + add_input("\x1e\x02\x08\x15\x1f" TEST_UUID TEST_NS "\x00", false, 26) // message decode failure + add_input("\x1e\x02\x08\x00\x1f", false, 5) // header decoder failure + add_input("\x1e\x02\x08\x14\x01\x1f", false, 6) // invalid header length + add_input("\x1e\x02\x09\x14\x1f", false, 5) // invalid protobuf header incorrect tag + add_input("\x1e\x02\x08\x65\x1f", false, 5) // invalid header message too long + }; + + lsb_heka_message m; + lsb_input_buffer ib; + size_t db; + mu_assert(!lsb_init_heka_message(&m, 1), "failed"); + mu_assert(!lsb_init_input_buffer(&ib, 100), "failed"); + for (unsigned i = 0; i < sizeof(tests) / sizeof(struct find_message); ++i) { + mu_assert(!lsb_expand_input_buffer(&ib, tests[i].s.len), "buffer exhausted"); + memcpy(ib.buf + ib.readpos, tests[i].s.s, tests[i].s.len); + ib.readpos += tests[i].s.len; + bool b = lsb_find_heka_message(&m, &ib, true, &db, NULL); + mu_assert(tests[i].b == b, "test: %u failed", i); + mu_assert(tests[i].d == db, "test: %u failed expected: %" PRIuSIZE " received: %" PRIuSIZE, i, tests[i].d, db); + } + mu_assert(!lsb_find_heka_message(NULL, NULL, true, NULL, NULL), "succeeded"); + mu_assert(!lsb_find_heka_message(&m, NULL, true, NULL, NULL), "succeeded"); + mu_assert(!lsb_find_heka_message(&m, &ib, true, NULL, NULL), "succeeded"); + lsb_free_input_buffer(&ib); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_read_heka_field() +{ + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, sizeof pb - 1, NULL), "decode failed"); + + lsb_read_value v; + lsb_const_string cs; + cs.s = "string"; + cs.len = 6; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "standalone"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "string", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + cs.s = "strings"; + cs.len = 7; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "item 0"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "s1", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 1, &v), "item 1"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "s2", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 2, &v), "item 2"); + mu_assert(v.type == LSB_READ_STRING, "%d", v.type); + mu_assert(strncmp(v.u.s.s, "s3", v.u.s.len) == 0, "invalid value: %.*s", + (int)v.u.s.len, v.u.s.s); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 3, &v) == false, + "no item 3"); + mu_assert(v.type == LSB_READ_NIL, "%d", v.type); + + cs.s = "number"; + cs.len = 6; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "standalone"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + + cs.s = "numbers"; + cs.len = 7; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "item 0"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 1, &v), "item 1"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 2, "invalid value: %g", v.u.d); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 2, &v), "item 2"); + mu_assert(v.type == LSB_READ_NUMERIC, "%d", v.type); + mu_assert(v.u.d == 3, "invalid value: %g", v.u.d); + + mu_assert(lsb_read_heka_field(&m, &cs, 0, 3, &v) == false, + "no item 3"); + mu_assert(v.type == LSB_READ_NIL, "%d", v.type); + + cs.s = "bool"; + cs.len = 4; + mu_assert(lsb_read_heka_field(&m, &cs, 0, 0, &v), "standalone"); + mu_assert(v.type == LSB_READ_BOOL, "%d", v.type); + mu_assert(v.u.d == 1, "invalid value: %g", v.u.d); + + mu_assert(!lsb_read_heka_field(NULL, NULL, 0, 0, NULL), "succeeded"); + mu_assert(!lsb_read_heka_field(&m, NULL, 0, 0, NULL), "succeeded"); + mu_assert(!lsb_read_heka_field(&m, &cs, 0, 0, NULL), "succeeded"); + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_write_heka_uuid() +{ + lsb_err_value ret; + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_UUID_SIZE + 2); + + const char header[2] = "\x0a\x10"; + const char bin_uuid[LSB_UUID_SIZE] = { 0 }; + ret = lsb_write_heka_uuid(&ob, bin_uuid, LSB_UUID_SIZE); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); + mu_assert(memcmp(ob.buf + 2, bin_uuid, LSB_UUID_SIZE) == 0, "invalid"); + + const char str_uuid[] = "00000000-0000-0000-0000-" + "000000000000"; + ret = lsb_write_heka_uuid(&ob, str_uuid, LSB_UUID_STR_SIZE); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); + mu_assert(memcmp(ob.buf + 2, bin_uuid, LSB_UUID_SIZE) == 0, "invalid"); + + const char err_uuid[] = "00000000+0000-0000-0000-" + "000000000000"; + ret = lsb_write_heka_uuid(&ob, err_uuid, LSB_UUID_STR_SIZE); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp(ob.buf, header, sizeof header) == 0, "invalid header"); + mu_assert(ob.buf[8] & 0x40, "invalid format should create a type 4 uuid"); + + ret = lsb_write_heka_uuid(&ob, NULL, 0); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(ob.buf[8] & 0x40, "null string should create a type 4 uuid"); + lsb_free_output_buffer(&ob); + + ret = lsb_write_heka_uuid(&ob, bin_uuid, 10); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.pos == LSB_UUID_SIZE + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(ob.buf[8] & 0x40, "unexpected length should create a type 4 uuid"); + lsb_free_output_buffer(&ob); + + lsb_output_buffer sob; + lsb_init_output_buffer(&sob, LSB_UUID_SIZE); + ret = lsb_write_heka_uuid(&sob, NULL, 0); + mu_assert(ret, "received "); + mu_assert(sob.pos == 0, "received: %" PRIuSIZE, sob.pos); + lsb_free_output_buffer(&sob); + + ret = lsb_write_heka_uuid(NULL, bin_uuid, LSB_UUID_SIZE); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received %s", lsb_err_string(ret)); + + return NULL; +} + +#if SIZE_MAX < 65536 +static char* test_write_heka_header() +{ + char header[LSB_MIN_HDR_SIZE]; + size_t hlen; + + for (unsigned i = 0; i < sizeof(size_t); ++i) { + hlen = lsb_write_heka_header(header, (size_t)1 << (i * 8)); + mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); + } + hlen = lsb_write_heka_header(header, (size_t)1 << 15); + mu_assert(hlen == 7, "received %" PRIuSIZE, hlen); + return NULL; +} +#elif SIZE_MAX < 4294967296 +static char* test_write_heka_header() +{ + char header[LSB_MIN_HDR_SIZE]; + size_t hlen; + + for (unsigned i = 0; i < sizeof(size_t); ++i) { + hlen = lsb_write_heka_header(header, (size_t)1 << (i * 8)); + mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); + } + hlen = lsb_write_heka_header(header, (size_t)1 << 31); + mu_assert(hlen == 9, "received %" PRIuSIZE, hlen); + return NULL; +} +#else +static char* test_write_heka_header() // limit to 8 byte test +{ + char header[LSB_MIN_HDR_SIZE]; + size_t hlen; + + for (unsigned i = 0; i < 7; ++i) { + hlen = lsb_write_heka_header(header, (size_t)1 << (i * 8)); + mu_assert(hlen == 5 + i, "i: %u received %" PRIuSIZE, i, hlen); + } + hlen = lsb_write_heka_header(header, (size_t)1 << 56); + mu_assert(hlen == 13, "received %" PRIuSIZE, hlen); + hlen = lsb_write_heka_header(header, (size_t)1 << 63); + mu_assert(hlen == 14, "received %" PRIuSIZE, hlen); + return NULL; +} +#endif + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init); + mu_run_test(test_init_failure); + mu_run_test(test_decode); + mu_run_test(test_decode_failure); + mu_run_test(test_find_message); + mu_run_test(test_read_heka_field); + mu_run_test(test_write_heka_uuid); + mu_run_test(test_write_heka_header); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_heka_message_matcher.c b/src/util/test/test_heka_message_matcher.c new file mode 100644 index 0000000..899039d --- /dev/null +++ b/src/util/test/test_heka_message_matcher.c @@ -0,0 +1,468 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief Hindsight/Heka message matcher unit tests @file */ + +#include +#include +#include +#include +#include + +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/heka_message_matcher.h" + +// {"Logger":"GoSpec","Uuid":"xxx","Pid":32157,"Severity":6,"EnvVersion":"0.8","Fields":[{""value":["bar"],"name":"foo","value_type":0},{"value":[64],"name":"number","value_type":2},{"value":["data"],"name":"bytes","value_type":1},{"value":[999,1024],"name":"int","value_type":2},{"value":[99.9],"name":"double","value_type":3},{"value":[true],"name":"bool","value_type":4},{"value":["alternate"],"name":"foo","value_type":0},{"value":["name=test;type=web;"],"name":"Payload","value_type":0},{"representation":"date-time","value":["Mon Jan 02 15:04:05 -0700 2006"],"name":"Timestamp","value_type":0},{"value":[0],"name":"zero","value_type":2},{"value":["43"],"name":"string","value_type":0}],"Payload":"Test Payload with a longer string to attempt to create a difference in pattern match time versus the string literal match time for a unique-item","Timestamp":1.428773426113e+18,"Hostname":"trink-x230","Type":"TEST"} + +char pb[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\xe4\x9e\xf1\xff\xc6\xbb\x81\xea\x13\x1a\x04\x54\x45\x53\x54\x22\x06\x47\x6f\x53\x70\x65\x63\x28\x06\x32\x90\x01Test Payload with a longer string to attempt to create a difference in pattern match time versus the string literal match time for a unique-item\x3a\x03\x30\x2e\x38\x40\x9d\xfb\x01\x4a\x0a\x74\x72\x69\x6e\x6b\x2d\x78\x32\x33\x30\x52\x0c\x0a\x03\x66\x6f\x6f\x10\x00\x22\x03\x62\x61\x72\x52\x0d\x0a\x06\x6e\x75\x6d\x62\x65\x72\x10\x02\x32\x01\x40\x52\x0f\x0a\x05\x62\x79\x74\x65\x73\x10\x01\x2a\x04\x64\x61\x74\x61\x52\x0d\x0a\x03\x69\x6e\x74\x10\x02\x32\x04\xe7\x07\x80\x08\x52\x14\x0a\x06\x64\x6f\x75\x62\x6c\x65\x10\x03\x3a\x08\x9a\x99\x99\x99\x99\xf9\x58\x40\x52\x0b\x0a\x04\x62\x6f\x6f\x6c\x10\x04\x42\x01\x01\x52\x12\x0a\x03\x66\x6f\x6f\x10\x00\x22\x09\x61\x6c\x74\x65\x72\x6e\x61\x74\x65\x52\x20\x0a\x07\x50\x61\x79\x6c\x6f\x61\x64\x10\x00\x22\x13\x6e\x61\x6d\x65\x3d\x74\x65\x73\x74\x3b\x74\x79\x70\x65\x3d\x77\x65\x62\x3b\x52\x38\x0a\x09\x54\x69\x6d\x65\x73\x74\x61\x6d\x70\x10\x00\x1a\x09\x64\x61\x74\x65\x2d\x74\x69\x6d\x65\x22\x1e\x4d\x6f\x6e\x20\x4a\x61\x6e\x20\x30\x32\x20\x31\x35\x3a\x30\x34\x3a\x30\x35\x20\x2d\x30\x37\x30\x30\x20\x32\x30\x30\x36\x52\x0b\x0a\x04\x7a\x65\x72\x6f\x10\x02\x32\x01\x00\x52\x0e\x0a\x06\x73\x74\x72\x69\x6e\x67\x10\x00\x22\x02\x34\x33"; +size_t pblen = sizeof(pb); + +char pbmin[] = "\x0a\x10\x23\x00\x81\xdc\x32\x6f\x4e\x3f\x9a\x5a\x93\x86\xa3\x7e\x24\x6f\x10\x00"; +size_t pbminlen = sizeof(pbmin); + + +static char* test_stub() +{ + return NULL; +} + + +static char* test_api_assertion() +{ + mu_assert(NULL == lsb_create_message_matcher(NULL), "not null"); + lsb_destroy_message_matcher(NULL); + return NULL; +} + +#define T128 "Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' &&" \ +"Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.' && Type =~ '.'" + +#define S255 "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" \ +"0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789" \ +"0123456789012345678901234567890123456789012345678901234" + +static char* test_true_matcher() +{ + char *tests[] = { + "TRUE" + , "Timestamp > 1.428773420000e+18" + , "Timestamp < 1.428773426999e18" + , "Timestamp == 1428773426113040228" + , "Timestamp > '2015-04-11T17:30:26Z'" // 1428773426000000000 + , "Timestamp > '2015-04-11T17:30:26.112Z'" + , "Timestamp < '2015-04-11T17:30:26.114Z'" + , "(Severity == 7 || Logger == 'GoSpec') && Type == 'TEST'" + , "EnvVersion == \"0.8\"" + , "EnvVersion == '0.8'" + , "EnvVersion != '0.9'" + , "EnvVersion > '0.7'" + , "EnvVersion >= '0.8'" + , "EnvVersion < '0.9'" + , "EnvVersion <= '0.8'" + , "Hostname != ''" + , "Logger == 'GoSpec'" + , "Pid != 0" + , "Pid != NIL" + , "Severity != 5" + , "Severity < 7" + , "Severity <= 7" + , "Severity <= 6" + , "Severity == 6" + , "Severity > 5" + , "Severity >= 5" + , "Severity >= 6" + , "Timestamp > 0" + , "Type != 'test'" + , "Type == 'TEST' && Severity == 6" + , "Type == 'test' && Severity == 7 || Logger == 'GoSpec'" + , "Type == 'TEST'" + , "Type == 'foo' || Type == 'bar' || Type == 'TEST'" + , "Fields[foo] == 'bar'" + , "Fields[foo][0] == 'bar'" + , "Fields[foo][0][0] == 'bar'" + , "Fields[foo][1] == 'alternate'" + , "Fields[foo][1][0] == 'alternate'" + , "Fields[foo] <= 'barx'" + , "Fields[foo] < 'barx'" + , "Fields[foo] >= 'bar'" + , "Uuid > '#'" + , "Uuid >= '#'" + , "Fields[foo] > 'baq'" + , "Fields[foo] != 'bara'" + , "Fields[bytes] == 'data'" + , "Fields[int] == 999" + , "Fields[int][0][1] == 1024" + , "Fields[double] == 99.9" + , "Fields[bool] == TRUE" + , "Fields[bool] != FALSE" + , "Fields[int] != NIL" + , "Fields[int][0][1] != NIL" + , "Fields[int][0][2] == NIL" + , "Fields[missing] == NIL" + , "Type =~ 'TEST'" + , "Type !~ 'bogus'" + , "Type =~ 'TEST' && Payload =~ 'Payload'" + , "Fields[foo][1] =~ 'alt'" + , "Fields[Payload] =~ 'name=%w+'" + , "Type =~ 'ST'" + , "Type =~ '^TE'" + , "Type =~ 'ST$'" + , "Type !~ '^te'" + , "Type !~ 'st$'" + , "Fields[foo][255] == NIL" + , "Fields[foo][0][255] == NIL" + , T128 + , "Payload =~ 'unique-item'%" + , "Fields[Timestamp] =~ ' -0700'%" + , "Type != NIL" + , "Hostname != NIL" + , "EnvVersion != NIL" + , "Pid != NIL" + , "Logger != NIL" + , "Payload != NIL" + , "Pid == 32157" + , "Uuid < '\\\\'" + , "Uuid < \"\\\\\"" + , "(Severity == 7 || Logger == 'GoSpec') \r\n\t&& Type == 'TEST'" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 16); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m), "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_nil_header_true_matcher() +{ + char *tests[] = { + "Type == NIL" + , "Type != ''" + , "Type < ''" + , "Type <= ''" + , "Type !~ '.'" + , "Type !~ 'foo'" + , "Hostname == NIL" + , "EnvVersion == NIL" + , "Pid == NIL" + , "Logger == NIL" + , "Payload == NIL" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 1); + mu_assert(lsb_decode_heka_message(&m, pbmin, pbminlen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m), "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_nil_header_false_matcher() +{ + char *tests[] = { + "Type != NIL" + , "Type == '%b()'" + , "Type == ''" + , "Type > ''" + , "Type >= ''" + , "Type =~ '.'" + , "Type =~ 'foo'" + , "Hostname != NIL" + , "Hostname == ''" + , "EnvVersion != NIL" + , "EnvVersion == ''" + , "Pid != NIL" + , "Logger != NIL" + , "Logger == ''" + , "Payload != NIL" + , "Payload == ''" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 1); + mu_assert(lsb_decode_heka_message(&m, pbmin, pbminlen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m) == false, "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_false_matcher() +{ + char *tests[] = { + "FALSE" + , "Timestamp == 1e9" + , "Timestamp > '2015-04-11T17:30:27Z'" + , "Type == 'test'&&(Severity==7||Payload=='Test Payload')" + , "EnvVersion == '0.9'" + , "EnvVersion != '0.8'" + , "EnvVersion > '0.9'" + , "EnvVersion >= '0.9'" + , "EnvVersion < '0.8'" + , "EnvVersion <= '0.7'" + , "Severity == 5" + , "Severity != 6" + , "Severity < 6" + , "Severity <= 5" + , "Severity > 6" + , "Severity >= 7" + , "Fields[foo] == 'ba'" + , "Fields[foo][1] == 'bar'" + , "Fields[foo][0][1] == 'bar'" + , "Fields[bool] == FALSE" + , "Fields[bool] != TRUE" + , "Fields[foo] > 'bara'" + , "Fields[foo] >= 'bara'" + , "Fields[foo] == 'bara'" + , "Type =~ 'Test'" + , "Type !~ 'TEST'" + , "Payload =~ '^Payload'" + , "Type == \"te'st\"" + , "Type == 'te\"st'" + , "Fields[int] =~ '999'" + , "Fields[zero] == \"0\"" + , "Fields[string] == 43" + , "Fields[int] == NIL" + , "Fields[int][0][1] == NIL" + , "Fields[missing] != NIL" + , "Type =~ '^te'" + , "Type =~ 'st$'" + , "Type !~ '^TE'" + , "Type !~ 'ST$'" + , "Logger =~ '.' && Type =~ '^anything'" + , "Type == '" S255 "'" + , "Payload =~ 'not.found'%" + , "Fields[foo][1] =~ 'not.found'%" + , "Type == NIL" + , "Hostname == NIL" + , "EnvVersion == NIL" + , "Pid == NIL" + , "Logger == NIL" + , "Payload == NIL" + , "Uuid > '\\\\'" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "failed to create the matcher %s", tests[i]); + mu_assert(lsb_eval_message_matcher(mm, &m) == false, "%s", tests[i]); + lsb_destroy_message_matcher(mm); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* test_malformed_matcher() +{ + char *tests[] = { + "" + , "bogus" + , "Type = 'test'" // invalid operator + , "Pid == 'test='" // Pid is not a string + , "Type == 'test' && (Severity==7 || Payload == 'Test Payload'" // missing paren + , "Invalid == 'bogus'" // unknown variable name + , "Fields[]" // empty name key + , "Fields[test][]" // empty field index + , "Fields[test][a]" // non numeric field index + , "Fields[test][0][]" // empty array index + , "Fields[test][0][a]" // non numeric array index + , "Fields[test][0][0][]" // extra index dimension + , "Fields[test][xxxx" // unmatched bracket + , "Pid =~ '6'" // string match not allowed on numeric + , "Pid !~ '6'" // string match not allowed on numeric + , "Type =~ 'test" // unmatched quote + , "Type != 'test\"" // mis matched quote types + , "Pid =~ 6" // incorrect type for the operator + , "NIL" // invalid use of constant + , "Type > NIL" // existence check only works with equals and not equals + , "Pid > NIL" + , "Fields[test] > NIL" + , "Uuid == NIL" // required header, cannot be nil + , "Severity == NIL" // defaulted header, cannot be nil + , "TRUE FALSE" // missing operator + , "Timestamp == '20150411T173026'" // non rfc3339 timestamp + , T128 " && Type =~ '.'" // too many tests + , "Type == '" S255 "5'" // string too long + , "Fields[test][256] == 1" // field index out of bounds + , "Fields[test][0][256] == 1" // array index out of bounds + , "Payload =~ 'foo'i" // invalid string match pattern modifier + , "Uuid < '\\'" // unescaped backslash leaving an open string '\' + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + for (int i = 0; tests[i]; ++i) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm == NULL, "created malformed matcher"); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* benchmark_matcher_create() +{ + int iter = 100000; + const char *exp = "Type == 'TEST' && Severity == 6"; + + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_message_matcher *mm = lsb_create_message_matcher(exp); + mu_assert(mm, "lsb_create_message_matcher failed"); + lsb_destroy_message_matcher(mm); + } + t = clock() - t; + printf("benchmark_matcher_create: %g\n", ((double)t) / CLOCKS_PER_SEC + / iter); + return NULL; +} + + +static char* benchmark_match_hs() +{ + // see what a single sample looks like for a better comparison with Hindsight + char *tests[] = { + "TRUE" + , "Type == 'TEST' && Severity == 6" + , "Fields[foo] == 'bar' && Severity == 6" + , "Fields[number] == 64 && Severity == 6" + , "Fields[missing] == NIL" + , "Fields[int] != NIL" + , "Type =~ '^[Tt]EST' && Severity == 6" + , "Payload =~ '^Test'" + , "Payload =~ 'item$'" + , "Payload =~ 'unique%-item'" + , "Payload =~ 'unique-item'%" + , "Payload =~ 'unique'" + , "Payload =~ 'unique'%" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + + unsigned long long start, end; + for (int i = 0; tests[i]; i++) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "lsb_create_message_matcher failed: %s", tests[i]); + start = lsb_get_time(); + mu_assert(lsb_eval_message_matcher(mm, &m), + "lsb_eval_message_matcher failed"); + end = lsb_get_time(); + lsb_destroy_message_matcher(mm); + printf("matcher (lsb_get_time): '%s': %g\n", tests[i], + (double)(end - start) / 1e9); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* benchmark_match() +{ + int iter = 1000000; + char *tests[] = { + "TRUE" + , "Type == 'TEST' && Severity == 6" + , "Fields[foo] == 'bar' && Severity == 6" + , "Fields[number] == 64 && Severity == 6" + , "Fields[missing] == NIL" + , "Fields[int] != NIL" + , "Type =~ '^[Tt]EST' && Severity == 6" + , "Payload =~ '^Test'" + , "Payload =~ 'item$'" + , "Payload =~ 'unique%-item'" + , "Payload =~ 'unique-item'%" + , "Payload =~ 'unique'" + , "Payload =~ 'unique'%" + , NULL }; + + lsb_heka_message m; + lsb_init_heka_message(&m, 8); + mu_assert(lsb_decode_heka_message(&m, pb, pblen - 1, NULL), "decode failed"); + + for (int i = 0; tests[i]; i++) { + lsb_message_matcher *mm = lsb_create_message_matcher(tests[i]); + mu_assert(mm, "lsb_create_message_matcher failed: %s", tests[i]); + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + mu_assert(lsb_eval_message_matcher(mm, &m), + "lsb_eval_message_matcher failed"); + } + t = clock() - t; + lsb_destroy_message_matcher(mm); + printf("matcher: '%s': %g\n", tests[i], ((double)t) / CLOCKS_PER_SEC + / iter); + } + lsb_free_heka_message(&m); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_api_assertion); + mu_run_test(test_true_matcher); + mu_run_test(test_nil_header_true_matcher); + mu_run_test(test_false_matcher); + mu_run_test(test_nil_header_false_matcher); + mu_run_test(test_malformed_matcher); + + mu_run_test(benchmark_match_hs); + mu_run_test(benchmark_matcher_create); + mu_run_test(benchmark_match); + return NULL; +} + + +int main() +{ + lsb_set_tz(NULL); + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_input_buffer.c b/src/util/test/test_input_buffer.c new file mode 100644 index 0000000..3ecaabc --- /dev/null +++ b/src/util/test/test_input_buffer.c @@ -0,0 +1,159 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include +#include + +#include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/input_buffer.h" + +static char* test_stub() +{ + return NULL; +} + +static char* test_init_small_buf() +{ + size_t size = 100; + lsb_input_buffer b; + + lsb_err_value ret = lsb_init_input_buffer(NULL, size); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert(b.size == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + lsb_free_input_buffer(&b); + lsb_free_input_buffer(NULL); + return NULL; +} + + +static char* test_init_large_buf() +{ + size_t size = 1024 * 1024; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert(b.size == BUFSIZ, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_init_zero_buf() +{ + lsb_input_buffer b; + lsb_err_value ret = lsb_init_input_buffer(&b, 0); + mu_assert(ret == LSB_ERR_UTIL_PRANGE, "received: %s", lsb_err_string(ret)); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_expand_buf() +{ + size_t size = 1024 * 1024; + size_t rsize = 1024 * 16; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + mu_assert(b.size == BUFSIZ, "received: %" PRIuSIZE, b.size); + + lsb_err_value ret = lsb_expand_input_buffer(NULL, 0); + mu_assert(LSB_ERR_UTIL_NULL == ret, "received: %s", lsb_err_string(ret)); + + mu_assert(!lsb_expand_input_buffer(&b, 1024 * 9), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + + mu_assert(!lsb_expand_input_buffer(&b, 1024), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size + LSB_MAX_HDR_SIZE, "received: %" PRIuSIZE, + b.size); + + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_expand_failure() +{ + size_t size = 1024; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + lsb_err_value ret = lsb_expand_input_buffer(&b, size + LSB_MAX_HDR_SIZE + 1); + mu_assert(LSB_ERR_UTIL_FULL == ret, "received: %s", lsb_err_string(ret)); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_shift_empty() +{ + size_t size = 16; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + b.readpos = 10; + b.scanpos = 10; + mu_assert(!lsb_expand_input_buffer(&b, 1), "expand failed"); + mu_assert(b.scanpos == 0, "received: %" PRIuSIZE, b.scanpos); + mu_assert(b.readpos == 0, "received: %" PRIuSIZE, b.readpos); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* test_shift_partial() +{ + size_t size = 16; + lsb_input_buffer b; + mu_assert(!lsb_init_input_buffer(&b, size), "init failed"); + memset(b.buf, '#', size); + b.readpos = 10; + b.scanpos = 9; + b.buf[b.scanpos] = 'A'; + mu_assert(!lsb_expand_input_buffer(&b, 1), "expand failed"); + mu_assert(b.scanpos == 0, "received: %" PRIuSIZE, b.scanpos); + mu_assert(b.readpos == 1, "received: %" PRIuSIZE, b.readpos); + mu_assert(b.buf[0] == 'A', "received: %c", b.buf[0]); + lsb_free_input_buffer(&b); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init_small_buf); + mu_run_test(test_init_large_buf); + mu_run_test(test_init_zero_buf); + mu_run_test(test_expand_buf); + mu_run_test(test_expand_failure); + mu_run_test(test_shift_empty); + mu_run_test(test_shift_partial); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_output_buffer.c b/src/util/test/test_output_buffer.c new file mode 100644 index 0000000..c1b9bb9 --- /dev/null +++ b/src/util/test/test_output_buffer.c @@ -0,0 +1,280 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_output_buffer unit tests @file */ + +#include +#include +#include +#include + +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/output_buffer.h" +#include "luasandbox/util/heka_message.h" + +#ifdef _MSC_VER +// To silence the +/-INFINITY warning +#pragma warning( disable : 4756 ) +#pragma warning( disable : 4056 ) +#endif + +static char* test_stub() +{ + return NULL; +} + + +static char* test_init_small_buf() +{ + size_t size = 512; + lsb_output_buffer b; + + lsb_err_value ret = lsb_init_output_buffer(NULL, size); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(b.size == size, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); + lsb_free_output_buffer(&b); + lsb_free_output_buffer(NULL); + return NULL; +} + + +static char* test_init_large_buf() +{ + size_t size = 1024 * 1024; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size , "received: %" PRIuSIZE, b.size); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_init_zero_buf() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == 0 , "received: %" PRIuSIZE, b.size); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_expand_buf() +{ + size_t size = 1024 * 1024; + size_t rsize = 1024 * 16; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(b.size == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.size); + + lsb_err_value ret = lsb_expand_output_buffer(NULL, 0); + mu_assert(LSB_ERR_UTIL_NULL == ret, "received: %s", lsb_err_string(ret)); + + mu_assert(!lsb_expand_output_buffer(&b, 1024 * 9), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); + + mu_assert(!lsb_expand_output_buffer(&b, 1024), "expand failed"); + mu_assert(b.size == rsize, "received: %" PRIuSIZE, b.size); + mu_assert(b.maxsize == size, "received: %" PRIuSIZE, b.size); + + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_expand_failure() +{ + size_t size = 1024; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, size), "init failed"); + mu_assert(lsb_expand_output_buffer(&b, size + 1), "expand succeeded"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputc() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputc(&b, 'a'); + mu_assert(strcmp("a", b.buf) == 0, "received: %s", b.buf); + lsb_outputc(&b, 'b'); + mu_assert(strcmp("ab", b.buf) == 0, "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputf() +{ + lsb_err_value ret; + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 10), "init failed"); + lsb_outputf(&b, "%s", "foo"); + mu_assert(strcmp("foo", b.buf) == 0, "received: %s", b.buf); + lsb_outputf(&b, " %s", "bar"); + mu_assert(strcmp("foo bar", b.buf) == 0, "received: %s", b.buf); + ret = lsb_outputf(&b, " %s", "exceed the buffer"); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received: %s", lsb_err_string(ret)); + ret = lsb_outputf(NULL, "%s", "bar"); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + ret = lsb_outputf(&b, NULL, "bar"); + mu_assert(ret == LSB_ERR_UTIL_NULL, "received: %s", lsb_err_string(ret)); + lsb_free_output_buffer(&b); + + size_t len = 2000; + mu_assert(!lsb_init_output_buffer(&b, len), "init failed"); + for (size_t i = 0; i < len - 1; ++i) { + ret = lsb_outputf(&b, "%c", 'a'); + mu_assert(!ret, "received: %s", ret); + } + ret = lsb_outputf(&b, "%c", 'a'); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received: %s", lsb_err_string(ret)); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputs() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputs(&b, "foo", 3); + mu_assert(strcmp("foo", b.buf) == 0, "received: %s", b.buf); + lsb_outputf(&b, " bar", 4); + mu_assert(strcmp("foo bar", b.buf) == 0, "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputd() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + lsb_outputd(&b, 10.1); + mu_assert(strcmp("10.1", b.buf) == 0, "received: %s", b.buf); + double d = INT_MAX; + lsb_outputd(&b, d + 1); + mu_assert(strcmp("10.12147483648", b.buf) == 0, "received: %s", b.buf); + d = INT_MIN; + lsb_outputd(&b, d - 1); + mu_assert(strcmp("10.12147483648-2147483649", b.buf) == 0, + "received: %s", b.buf); + lsb_outputd(&b, NAN); + mu_assert(strcmp("10.12147483648-2147483649nan", b.buf) == 0, + "received: %s", b.buf); + lsb_outputd(&b, INFINITY); + mu_assert(strcmp("10.12147483648-2147483649naninf", b.buf) == 0, + "received: %s", b.buf); + lsb_outputd(&b, -INFINITY); + mu_assert(strcmp("10.12147483648-2147483649naninf-inf", b.buf) == 0, + "received: %s", b.buf); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputc_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputc(&b, 'a'); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputf_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputf(&b, "%s", "f"); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputs_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputs(&b, "s", 1); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* test_outputd_full() +{ + lsb_output_buffer b; + mu_assert(!lsb_init_output_buffer(&b, 0), "init failed"); + for (int i = 0; i < LSB_OUTPUT_SIZE; ++i) { + lsb_outputd(&b, 1); + } + mu_assert(b.size == LSB_OUTPUT_SIZE * 2, "received: %" PRIuSIZE, b.size); + mu_assert(b.pos == LSB_OUTPUT_SIZE, "received: %" PRIuSIZE, b.pos); + mu_assert(b.buf[b.pos] == '\0', "missing NUL"); + lsb_free_output_buffer(&b); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init_small_buf); + mu_run_test(test_init_large_buf); + mu_run_test(test_init_zero_buf); + mu_run_test(test_expand_buf); + mu_run_test(test_expand_failure); + mu_run_test(test_outputc); + mu_run_test(test_outputf); + mu_run_test(test_outputs); + mu_run_test(test_outputd); + + // make sure the terminating NUL is included in the size of the buffer + mu_run_test(test_outputc_full); + mu_run_test(test_outputf_full); + mu_run_test(test_outputs_full); + mu_run_test(test_outputd_full); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_protobuf.c b/src/util/test/test_protobuf.c new file mode 100644 index 0000000..38b2eb6 --- /dev/null +++ b/src/util/test/test_protobuf.c @@ -0,0 +1,256 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief util protobuf unit tests @file */ + +#include +#include + +#include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/heka_message.h" +#include "luasandbox/util/protobuf.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_lsb_pb_read_key() +{ + int tag, wt; + const char input[] = "\x0b"; + const char *p = lsb_pb_read_key(input, &tag, &wt); + mu_assert(p = input + 1, "received: %p", (void *)p); + mu_assert(tag == 1, "received: %d", tag); + mu_assert(wt == 3, "received: %d", wt); + p = lsb_pb_read_key(NULL, &tag, &wt); + mu_assert(!p, "not null"); + p = lsb_pb_read_key(input, NULL, &wt); + mu_assert(!p, "not null"); + p = lsb_pb_read_key(input, &tag, NULL); + mu_assert(!p, "not null"); + return NULL; +} + + +static char* test_lsb_pb_write_key() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); + + lsb_err_value ret = lsb_pb_write_key(&ob, 1, 3); + mu_assert(!ret, "received %s", ret); + mu_assert(memcmp(ob.buf, "\x0b", 1) == 0, "received: %02hhx", ob.buf[0]); + + ob.pos = ob.maxsize; + ret = lsb_pb_write_key(&ob, 1, 3); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_varint() +{ + struct test_data { + long long v; + const char *s; + }; + + struct test_data tests[] = { + { 0, "\x00" }, + { 16, "\x10" }, + { 300, "\xac\x02" }, + { -1, "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01" }, + { 5000000000LL, "\x80\xe4\x97\xd0\x12" } + }; + + lsb_err_value ret; + long long vi; + const char *p; + for (unsigned i = 0; i < sizeof tests / sizeof tests[0]; ++i){ + long long v = tests[i].v; + const char *s = tests[i].s; + size_t len = strlen(s); + len = len ? len : 1; + p = lsb_pb_read_varint(s, s + len, &vi); + mu_assert(p = s + len, "received: %p", (void *)p); + mu_assert(v == vi, "expected: %lld received: %lld", v, vi); + + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); + ret = lsb_pb_write_varint(&ob, v); + mu_assert(!ret, "received %s", ret); + if (strncmp(s, ob.buf, len) != 0) { + char expected[LSB_MAX_VARINT_BYTES + 1] = { 0 }; + char received[LSB_MAX_VARINT_BYTES + 1] = { 0 }; + for (unsigned i = 0; i < len; ++i) { + sprintf(expected + i * 2, "%02hhx", s[i]); + sprintf(received + i * 2, "%02hhx", ob.buf[i]); + } + mu_assert(false, "expected: %s received: %s", expected, received); + } + lsb_free_output_buffer(&ob); + } + + const char tl[] = "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"; + mu_assert(!lsb_pb_read_varint(tl, tl + sizeof(tl) - 1, &vi), + "parsed invalid varint (too long)"); + + const char ts[] = "\xff"; + mu_assert(!lsb_pb_read_varint(ts, ts + sizeof(ts) - 1, &vi), + "parsed invalid varint (too short)"); + + mu_assert(!lsb_pb_read_varint(NULL, ts + sizeof(ts) - 1, &vi), "null start"); + mu_assert(!lsb_pb_read_varint(ts, NULL, &vi), "null end"); + mu_assert(!lsb_pb_read_varint(ts, ts + sizeof(ts) - 1, NULL), "null ret"); + + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, LSB_MAX_VARINT_BYTES); + ob.pos = ob.maxsize; + ret = lsb_pb_write_varint(&ob, 1); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + lsb_free_output_buffer(&ob); + + return NULL; +} + + +static char* test_lsb_pb_write_bool() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, 1); + lsb_err_value ret = lsb_pb_write_bool(&ob, 7); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.buf[0] == 1, "received: %02hhx", ob.buf[0]); + mu_assert(lsb_pb_write_bool(&ob, 7), "buffer should be full"); + + ob.pos = 0; + ret = lsb_pb_write_bool(&ob, 0); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.buf[0] == 0, "received: %02hhx", ob.buf[0]); + + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_lsb_pb_write_double() +{ + double d = 7.13; + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, sizeof d); + lsb_err_value ret = lsb_pb_write_double(&ob, d); + mu_assert(!ret, "received %s", ret); + mu_assert(memcmp(ob.buf, &d, sizeof d) == 0, "received: %g", + *((double *)ob.buf)); + ret = lsb_pb_write_double(&ob, d); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_lsb_pb_write_string() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, 64); + const char foo[] = "foo"; + const size_t len = sizeof foo - 1; + + // failure writing the key + ob.pos = ob.maxsize; + lsb_err_value ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + + // failure writing the len + ob.pos = ob.maxsize - 1; + ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + + // failure writing the string + ob.pos = ob.maxsize - 3; + ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + + ob.pos = 0; + ret = lsb_pb_write_string(&ob, 1, foo, len); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.pos = len + 2, "received: %" PRIuSIZE, ob.pos); + mu_assert(memcmp("\x0a\x03" "foo", ob.buf, 5) == 0, "received: " + "%02hhx%02hhx%02hhx%02hhx2%hhx", ob.buf[0], ob.buf[1], ob.buf[2], + ob.buf[3], ob.buf[4]); + + lsb_free_output_buffer(&ob); + return NULL; +} + + +static char* test_lsb_pb_update_field_length() +{ + lsb_output_buffer ob; + lsb_init_output_buffer(&ob, 1024); + memset(ob.buf, 0, 1024); + ob.buf[0] = 'a'; + ob.buf[2] = 'b'; + ob.pos = 3; + lsb_err_value ret = lsb_pb_update_field_length(&ob, 1); + mu_assert(!ret, "received %s", ret); + mu_assert(ob.buf[1] == 1, "received: %d", ob.buf[1]); + mu_assert(ob.buf[2] == 'b', "received: %02hhx", ob.buf[2]); + mu_assert(ob.pos == 3, "received: %" PRIuSIZE, ob.pos); + + // buffer full + ob.pos = 1024; + ret = lsb_pb_update_field_length(&ob, 1); + mu_assert(ret == LSB_ERR_UTIL_FULL, "received %s", lsb_err_string(ret)); + + // position is out of bounds + ob.pos = 512; + ret = lsb_pb_update_field_length(&ob, 512); + mu_assert(ret == LSB_ERR_UTIL_PRANGE, "received %s", lsb_err_string(ret)); + + ob.buf[301] = 'x'; + ob.pos = 302; + ret = lsb_pb_update_field_length(&ob, 1); + mu_assert(!ret, "received %s", ret); + mu_assert((unsigned char)ob.buf[1] == 0xac, "received: %02hhx", ob.buf[1]); + mu_assert(ob.buf[2] == 0x02, "received: %02hhx", ob.buf[2]); + mu_assert(ob.buf[3] == 'b', "received: %02hhx", ob.buf[3]); + mu_assert(ob.buf[302] == 'x', "received: %02hhx", ob.buf[302]); + mu_assert(ob.pos == 303, "received: %" PRIuSIZE, ob.pos); + + lsb_free_output_buffer(&ob); + return NULL; +} + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_lsb_pb_read_key); + mu_run_test(test_lsb_pb_write_key); + mu_run_test(test_varint); + mu_run_test(test_lsb_pb_write_bool); + mu_run_test(test_lsb_pb_write_double); + mu_run_test(test_lsb_pb_write_string); + mu_run_test(test_lsb_pb_update_field_length); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_running_stats.c b/src/util/test/test_running_stats.c new file mode 100644 index 0000000..6a1b4e5 --- /dev/null +++ b/src/util/test/test_running_stats.c @@ -0,0 +1,90 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include + +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/running_stats.h" + +#ifdef _MSC_VER +// To silence the +/-INFINITY warning +#pragma warning( disable : 4756 ) +#pragma warning( disable : 4056 ) +#endif + +static char* test_stub() +{ + return NULL; +} + + +static char* test_init() +{ + lsb_running_stats stats; + lsb_init_running_stats(&stats); + mu_assert(stats.count == 0, "received: %g", stats.count); + mu_assert(stats.mean == 0, "received: %g", stats.mean); + mu_assert(stats.sum == 0, "received: %g", stats.sum); + return NULL; +} + + +static char* test_calculation() +{ + lsb_running_stats stats; + lsb_init_running_stats(&stats); + double sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 0, "received: %g", sd); + lsb_update_running_stats(&stats, 1.0); + lsb_update_running_stats(&stats, 2.0); + lsb_update_running_stats(&stats, 3.0); + mu_assert(stats.count == 3, "received: %g", stats.count); + mu_assert(stats.mean == 2, "received: %g", stats.mean); + sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 1.0, "received: %g", sd); + return NULL; +} + + +static char* test_nan_inf() +{ + lsb_running_stats stats; + lsb_init_running_stats(&stats); + double sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 0, "received: %g", sd); + lsb_update_running_stats(&stats, INFINITY); + lsb_update_running_stats(&stats, NAN); + lsb_update_running_stats(&stats, -INFINITY); + mu_assert(stats.count == 0, "received: %g", stats.count); + sd = lsb_sd_running_stats(&stats); + mu_assert(sd == 0, "received: %g", sd); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_init); + mu_run_test(test_calculation); + mu_run_test(test_nan_inf); + return NULL; +} + + +int main() +{ + char* result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_string.c b/src/util/test/test_string.c new file mode 100644 index 0000000..179534f --- /dev/null +++ b/src/util/test/test_string.c @@ -0,0 +1,110 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include +#include + +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/string.h" + +static char* test_stub() +{ + return NULL; +} + +typedef struct { + char *s; + char *r; + size_t len; +} testcase; + + +static char* test_success() +{ + testcase tests[] = { + {"TRUE" ,"TRUE" ,4}, + {"\\a" ,"\a" ,1}, + {"\\b" ,"\b" ,1}, + {"\\f" ,"\f" ,1}, + {"\\n" ,"\n" ,1}, + {"\\r" ,"\r" ,1}, + {"\\t" ,"\t" ,1}, + {"\\v" ,"\v" ,1}, + {"\\\"" ,"\"" ,1}, + {"\\'" ,"'" ,1}, + {"\\\\" ,"\\" ,1}, + {"\\?" ,"?" ,1}, + {"\\1" ,"\1" ,1}, + {"\\33" ,"!" ,1}, + {"\\109" ,"m" ,1}, + {"+\\99\\1009+" ,"+cd9+" ,5}, + {"1\\000M" ,"1\x00M" ,3}, + }; + + for (unsigned i = 0; i < sizeof tests / sizeof(testcase); ++i){ + testcase *tc = &tests[i]; + size_t len = strlen(tc->s) + 1; + char *d = malloc(len); + char *us = lsb_lua_string_unescape(d, tc->s, &len); + mu_assert(us, "test: %d valid string: %s", i, tc->s); + mu_assert(len == tc->len, "test: %d string: %s expected len: %" PRIuSIZE + " received: %" PRIuSIZE, i, tc->s, tc->len, len); + mu_assert(memcmp(us, tc->r, len) == 0, "test: %d memcmp string: %s", i, + tc->s); + free(d); + } + return NULL; +} + + +static char* test_failure() +{ + testcase tests[] = { + {"\\p" ,NULL ,2}, // invalid escape char + {"\\999" ,NULL ,4}, // invalid char value + {"foo" ,NULL ,2}, // destination too small + {NULL ,NULL ,2}, // NULL source + }; + + for (unsigned i = 0; i < sizeof tests / sizeof(testcase); ++i){ + testcase *tc = &tests[i]; + size_t len = tc->len + 1; + char *d = malloc(len); + mu_assert(!lsb_lua_string_unescape(d, tc->s, &len), + "test: %d invalid string: %s", i, tc->s); + free(d); + } + mu_assert(!lsb_lua_string_unescape(NULL, tests[0].s, &tests[0].len), + "NULL destination"); + + char d[3]; + mu_assert(!lsb_lua_string_unescape(d, tests[0].s, NULL), "NULL length"); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_success); + mu_run_test(test_failure); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_string_matcher.c b/src/util/test/test_string_matcher.c new file mode 100644 index 0000000..bf70f40 --- /dev/null +++ b/src/util/test/test_string_matcher.c @@ -0,0 +1,192 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_input_buffer unit tests @file */ + +#include + +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/string_matcher.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_success() +{ + char *tests[] = { + " " , "." + , "foobar", "foo" + , "foobar", "bar$" + , "a" , "%a" + , "1" , "%d" + , "a" , "%l" + , "," , "%p" + , "\t" , "%s" + , "A" , "%u" + , "a" , "%w" + , "f" , "%x" + , "%" , "%%" + , "a" , "[a-c]" + , "d" , "[^a-c]" + , "abc" , "^%a+$" + , "(" , "(" + , "|foo|" , "%b||" + , "?" , "%?" + , "+" , "%+" + , "-" , "%-" + , "*" , "%*" + , "%" , "%%" + , "." , "%." + , "^" , "%^" + , "$" , "%$" + , "[" , "%[" + , "]" , "%]" + , "]" , "]" + , "(" , "%(" + , ")" , "%)" + , ")" , ")" + , "abc" , "^%a-$" + , "a$c" , "a$c" + , "a" , "%a?" + , "abc" , "%a*" + , "123ab" , "%f[%a]" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(lsb_string_match(tests[i], strlen(tests[i]), tests[i + 1]), + "no match test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + mu_assert(lsb_string_match("\0", 1, "%z"), "NULL match"); + return NULL; +} + + +static char* test_find_success() +{ + char *tests[] = { + "foo." , "." + , "foobar", "foo" + , "foo-bar", "o-b" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(lsb_string_find(tests[i], strlen(tests[i]), tests[i + 1], strlen(tests[i + 1])), + "not found test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + return NULL; +} + + +static char* test_failure() +{ + char *tests[] = { + "" , "." + , "_foobar", "^foo" + , "foobar_", "bar$" + , "1" , "%a" + , "a" , "%d" + , "A" , "%l" + , "a" , "%p" + , "." , "%s" + , "a" , "%u" + , "." , "%w" + , "p" , "%x" + , "a" , "%%" + , "d" , "[a-c]" + , "a" , "[^a-c]" + , "a." , "^%a+$" + , "foo()" , "bar()" + , "|foo" , "%b||" + , "aa" , "(%a)%1*" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(!lsb_string_match(tests[i], strlen(tests[i]), tests[i + 1]), + "match test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + return NULL; +} + + +static char* test_find_failure() +{ + char *tests[] = { + "foo" , "." + , "foobar" , "longer string" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(!lsb_string_find(tests[i], strlen(tests[i]), tests[i + 1], strlen(tests[i + 1])), + "find test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + mu_assert(!lsb_string_find(NULL, 0, tests[0], strlen(tests[1])), + "find test: string: NULL pattern: %s", tests[1]); + mu_assert(!lsb_string_find(tests[0], strlen(tests[0]), NULL, 0), + "find test: string: %s pattern: NULL", tests[0]); + return NULL; +} + + +static char* test_invalid() +{ + char *tests[] = { + "" , "[f" + , "" , "%b|" + , "" , "?" + , "" , "+" + , "" , "-" + , "" , "*" + , "" , "%" + , NULL + }; + + for (int i = 0; tests[i]; i += 2) { + mu_assert(!lsb_string_match(tests[i], strlen(tests[i]), tests[i + 1]), + "invalid test: %d string: %s pattern: %s", i / 2, tests[i], + tests[i + 1]); + } + mu_assert(!lsb_string_match(NULL, 0, tests[1]), + "invalid test: string: NULL pattern: %s", tests[1]); + mu_assert(!lsb_string_match(tests[0], strlen(tests[0]), NULL), + "invalid test: string: %s pattern: NULL", tests[0]); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_success); + mu_run_test(test_find_success); + mu_run_test(test_failure); + mu_run_test(test_find_failure); + mu_run_test(test_invalid); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/test/test_util.c b/src/util/test/test_util.c new file mode 100644 index 0000000..f258e2f --- /dev/null +++ b/src/util/test/test_util.c @@ -0,0 +1,134 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** @brief lsb_util unit tests @file */ + +#include +#include +#include +#include + +#include "luasandbox/error.h" +#include "luasandbox/test/mu_test.h" +#include "luasandbox/util/util.h" + +static char* test_stub() +{ + return NULL; +} + + +static char* test_lsb_lp2() +{ + size_t lp2; + + lp2 = lsb_lp2(0); + mu_assert(lp2 == 0, "received: %" PRIuSIZE, lp2); + + lp2 = lsb_lp2(1); + mu_assert(lp2 == 1, "received: %" PRIuSIZE, lp2); + + lp2 = lsb_lp2(1000); + mu_assert(lp2 == 1024, "received: %" PRIuSIZE, lp2); + return NULL; +} + + +static char* test_lsb_read_file() +{ + char *s = lsb_read_file("Makefile"); + mu_assert(s, "read file failed"); + free(s); + + s = lsb_read_file("_foo_bar_"); + free(s); // if it succeeded don't leak + mu_assert(!s, "read file succeeded"); + return NULL; +} + + +static char* test_lsb_set_tz() +{ + mu_assert(lsb_set_tz(NULL), "set_tz failed"); + mu_assert(lsb_set_tz("America/Los_Angeles"), "set_tz failed"); + static const char too_long[] = "01234567890123456789012345678901"; +#ifdef _WIN32 + mu_assert(!lsb_set_tz(too_long), "set_tz succeeded"); +#else + mu_assert(lsb_set_tz(too_long), "set_tz failed"); +#endif + return NULL; +} + + +static char* benchmark_lsb_get_time() +{ + int iter = 1000000; + unsigned long long start, end; + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_get_time(); + } + t = clock() - t; + printf("benchmark_lsb_get_time(%d) - clock %g seconds\n", iter, + (double)t / CLOCKS_PER_SEC / iter); + + start = lsb_get_time(); + for (int x = 0; x < iter; ++x) { + lsb_get_time(); + } + end = lsb_get_time(); + printf("benchmark_lsb_get_time(%d) - self %g seconds\n", iter, + (double)(end - start) / iter / 1e9); + + start = lsb_get_time(); + lsb_get_time(); + end = lsb_get_time(); + printf("benchmark_lsb_get_time(1) %llu\n", end - start); + return NULL; +} + + +static char* benchmark_lsb_get_timestamp() +{ + int iter = 1000000; + clock_t t = clock(); + for (int x = 0; x < iter; ++x) { + lsb_get_timestamp(); + } + t = clock() - t; + printf("benchmark_lsb_get_timestamp(%d) - clock %g seconds\n", iter, + (double)t / CLOCKS_PER_SEC / iter); + long long delta = lsb_get_timestamp() / 1000000000LL - time(NULL); + mu_assert(delta > 0 ? delta <= 1 : delta >= -1, "delta %lld", delta); + return NULL; +} + + +static char* all_tests() +{ + mu_run_test(test_stub); + mu_run_test(test_lsb_lp2); + mu_run_test(test_lsb_read_file); + mu_run_test(test_lsb_set_tz); + + mu_run_test(benchmark_lsb_get_time); + mu_run_test(benchmark_lsb_get_timestamp); + return NULL; +} + + +int main() +{ + char *result = all_tests(); + if (result) { + printf("%s\n", result); + } else { + printf("ALL TESTS PASSED\n"); + } + printf("Tests run: %d\n", mu_tests_run); + return result != NULL; +} diff --git a/src/util/util.c b/src/util/util.c new file mode 100644 index 0000000..d14643a --- /dev/null +++ b/src/util/util.c @@ -0,0 +1,160 @@ +/* -*- Mode: C; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** General purpose utility functions @file */ + +#include "luasandbox/util/util.h" +#include "../luasandbox_defines.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#endif + +#if defined(__MACH__) && defined(__APPLE__) +#include +#include +#include +#endif + +lsb_err_id LSB_ERR_UTIL_NULL = "pointer is NULL"; +lsb_err_id LSB_ERR_UTIL_OOM = "memory allocation failed"; +lsb_err_id LSB_ERR_UTIL_FULL = "buffer full"; +lsb_err_id LSB_ERR_UTIL_PRANGE = "parameter out of range"; + +size_t lsb_lp2(unsigned long long x) +{ + if (x == 0) return 0; + x = x - 1; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >> 16); + x = x | (x >> 32); + return (size_t)(x + 1); +} + + +char* lsb_read_file(const char *fn) +{ + char *str = NULL; + size_t b; + FILE *fh = fopen(fn, "rb" CLOSE_ON_EXEC); + if (!fh) return str; + + if (fseek(fh, 0, SEEK_END)) goto cleanup; + long pos = ftell(fh); + if (pos == -1) goto cleanup; + rewind(fh); + + str = malloc(pos + 1); + if (!str) goto cleanup; + + b = fread(str, 1, pos, fh); + if ((long)b == pos) { + str[pos] = 0; + } + +cleanup: + fclose(fh); + return str; +} + + +unsigned long long lsb_get_time() +{ +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000000000ULL + ts.tv_nsec; +#elif defined(__MACH__) && defined(__APPLE__) + static unsigned long long convert = 0; + if (convert == 0) { + mach_timebase_info_data_t tbi; + (void)mach_timebase_info(&tbi); + convert = tbi.numer / tbi.denom; + } + return mach_absolute_time() * convert; +#elif defined(_WIN32) + static unsigned long long qpf = ULLONG_MAX; + static_assert(sizeof(LARGE_INTEGER) == sizeof qpf, "size mismatch"); + + unsigned long long t; + if (qpf == ULLONG_MAX) QueryPerformanceFrequency((LARGE_INTEGER *)&qpf); + if (qpf) { + QueryPerformanceCounter((LARGE_INTEGER *)&t); + return (t / qpf * 1000000000ULL) + ((t % qpf) * 1000000000ULL / qpf); + } else { + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + t = ft.dwHighDateTime; + t <<= 32; + t |= ft.dwLowDateTime; + return t * 100ULL; + } +#else + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000000ULL + tv.tv_usec * 1000ULL; +#endif +} + + +long long lsb_get_timestamp() +{ +#ifdef HAVE_CLOCK_GETTIME + struct timespec ts; + clock_gettime(CLOCK_REALTIME, &ts); + return ts.tv_sec * 1000000000LL + ts.tv_nsec; +#elif defined(__MACH__) && defined(__APPLE__) + clock_serv_t cclock; + mach_timespec_t mts; + host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &cclock); + clock_get_time(cclock, &mts); + mach_port_deallocate(mach_task_self(), cclock); + return mts.tv_sec * 1000000000LL + mts.tv_nsec; +#elif defined(_WIN32) + FILETIME ft; + GetSystemTimeAsFileTime(&ft); + unsigned long long t = ft.dwHighDateTime; + t <<= 32; + t |= ft.dwLowDateTime; + t -= 116444736000000000ULL; // convert from Jan 1 1601 to Jan 1 1970 + return t * 100LL; +#else + struct timeval tv; + gettimeofday(&tv, NULL); + return tv.tv_sec * 1000000000LL + tv.tv_usec * 1000LL; +#endif +} + + +bool lsb_set_tz(const char *tz) +{ + if (!tz) { + tz = "UTC"; + } +#if _WIN32 + char s[32]; + int n = _snprintf(s, sizeof(s), "TZ=%s", tz); + if (n < 0 || n >= sizeof(s) || _putenv(s) != 0) { + return false; + } +#else + if (setenv("TZ", tz, 1) != 0) { + return false; + } +#endif + return true; +} diff --git a/src/xxhash.c b/src/xxhash.c deleted file mode 100644 index e39ac2e..0000000 --- a/src/xxhash.c +++ /dev/null @@ -1,476 +0,0 @@ -/* -xxHash - Fast Hash algorithm -Copyright (C) 2012-2014, Yann Collet. -BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -* Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -You can contact the author at : -- xxHash source repository : http://code.google.com/p/xxhash/ -*/ - - -//************************************** -// Tuning parameters -//************************************** -// Unaligned memory access is automatically enabled for "common" CPU, such as x86. -// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected. -// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance. -// You can also enable this parameter if you know your input data will always be aligned (boundaries of 4, for U32). -#if defined(__ARM_FEATURE_UNALIGNED) || defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) -# define XXH_USE_UNALIGNED_ACCESS 1 -#endif - -// XXH_ACCEPT_NULL_INPUT_POINTER : -// If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. -// When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. -// This option has a very small performance cost (only measurable on small inputs). -// By default, this option is disabled. To enable it, uncomment below define : -//#define XXH_ACCEPT_NULL_INPUT_POINTER 1 - -// XXH_FORCE_NATIVE_FORMAT : -// By default, xxHash library provides endian-independant Hash values, based on little-endian convention. -// Results are therefore identical for little-endian and big-endian CPU. -// This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. -// Should endian-independance be of no importance for your application, you may set the #define below to 1. -// It will improve speed for Big-endian CPU. -// This option has no impact on Little_Endian CPU. -#define XXH_FORCE_NATIVE_FORMAT 0 - - -//************************************** -// Compiler Specific Options -//************************************** -// Disable some Visual warning messages -#ifdef _MSC_VER // Visual Studio -# pragma warning(disable : 4127) // disable: C4127: conditional expression is constant -#endif - -#ifdef _MSC_VER // Visual Studio -# define FORCE_INLINE static __forceinline -#else -# ifdef __GNUC__ -# define FORCE_INLINE static inline __attribute__((always_inline)) -# else -# define FORCE_INLINE static inline -# endif -#endif - - -//************************************** -// Includes & Memory related functions -//************************************** -#include "xxhash.h" -// Modify the local functions below should you wish to use some other memory related routines -// for malloc(), free() -#include -FORCE_INLINE void* XXH_malloc(size_t s) { return malloc(s); } -FORCE_INLINE void XXH_free (void* p) { free(p); } -// for memcpy() -#include -FORCE_INLINE void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } - - -//************************************** -// Basic Types -//************************************** -#if defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L // C99 -# include - typedef uint8_t BYTE; - typedef uint16_t U16; - typedef uint32_t U32; - typedef int32_t S32; - typedef uint64_t U64; -#else - typedef unsigned char BYTE; - typedef unsigned short U16; - typedef unsigned int U32; - typedef signed int S32; - typedef unsigned long long U64; -#endif - -#if defined(__GNUC__) && !defined(XXH_USE_UNALIGNED_ACCESS) -# define _PACKED __attribute__ ((packed)) -#else -# define _PACKED -#endif - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# ifdef __IBMC__ -# pragma pack(1) -# else -# pragma pack(push, 1) -# endif -#endif - -typedef struct _U32_S { U32 v; } _PACKED U32_S; - -#if !defined(XXH_USE_UNALIGNED_ACCESS) && !defined(__GNUC__) -# pragma pack(pop) -#endif - -#define A32(x) (((U32_S *)(x))->v) - - -//*************************************** -// Compiler-specific Functions and Macros -//*************************************** -#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) - -// Note : although _rotl exists for minGW (GCC under windows), performance seems poor -#if defined(_MSC_VER) -# define XXH_rotl32(x,r) _rotl(x,r) -#else -# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) -#endif - -#if defined(_MSC_VER) // Visual Studio -# define XXH_swap32 _byteswap_ulong -#elif GCC_VERSION >= 403 -# define XXH_swap32 __builtin_bswap32 -#else -static inline U32 XXH_swap32 (U32 x) { - return ((x << 24) & 0xff000000 ) | - ((x << 8) & 0x00ff0000 ) | - ((x >> 8) & 0x0000ff00 ) | - ((x >> 24) & 0x000000ff );} -#endif - - -//************************************** -// Constants -//************************************** -#define PRIME32_1 2654435761U -#define PRIME32_2 2246822519U -#define PRIME32_3 3266489917U -#define PRIME32_4 668265263U -#define PRIME32_5 374761393U - - -//************************************** -// Architecture Macros -//************************************** -typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; -#ifndef XXH_CPU_LITTLE_ENDIAN // It is possible to define XXH_CPU_LITTLE_ENDIAN externally, for example using a compiler switch - static const int one = 1; -# define XXH_CPU_LITTLE_ENDIAN (*(char*)(&one)) -#endif - - -//************************************** -// Macros -//************************************** -#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(!!(c)) }; } // use only *after* variable declarations - - -//**************************** -// Memory reads -//**************************** -typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; - -FORCE_INLINE U32 XXH_readLE32_align(const U32* ptr, XXH_endianess endian, XXH_alignment align) -{ - if (align==XXH_unaligned) - return endian==XXH_littleEndian ? A32(ptr) : XXH_swap32(A32(ptr)); - else - return endian==XXH_littleEndian ? *ptr : XXH_swap32(*ptr); -} - -FORCE_INLINE U32 XXH_readLE32(const U32* ptr, XXH_endianess endian) { return XXH_readLE32_align(ptr, endian, XXH_unaligned); } - - -//**************************** -// Simple Hash Functions -//**************************** -FORCE_INLINE U32 XXH32_endian_align(const void* input, int len, U32 seed, XXH_endianess endian, XXH_alignment align) -{ - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - U32 h32; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (p==NULL) { len=0; p=(const BYTE*)(size_t)16; } -#endif - - if (len>=16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = seed + PRIME32_1 + PRIME32_2; - U32 v2 = seed + PRIME32_2; - U32 v3 = seed + 0; - U32 v4 = seed - PRIME32_1; - - do - { - v1 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); - } - else - { - h32 = seed + PRIME32_5; - } - - h32 += (U32) len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32_align((const U32*)p, endian, align) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32(const void* input, int len, U32 seed) -{ -#if 0 - // Simple version, good for code maintenance, but unfortunately slow for small inputs - void* state = XXH32_init(seed); - XXH32_update(state, input, len); - return XXH32_digest(state); -#else - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - -# if !defined(XXH_USE_UNALIGNED_ACCESS) - if ((((size_t)input) & 3)) // Input is aligned, let's leverage the speed advantage - { - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); - } -# endif - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); - else - return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); -#endif -} - - -//**************************** -// Advanced Hash Functions -//**************************** - -struct XXH_state32_t -{ - U64 total_len; - U32 seed; - U32 v1; - U32 v2; - U32 v3; - U32 v4; - int memsize; - char memory[16]; -}; - - -int XXH32_sizeofState() -{ - XXH_STATIC_ASSERT(XXH32_SIZEOFSTATE >= sizeof(struct XXH_state32_t)); // A compilation error here means XXH32_SIZEOFSTATE is not large enough - return sizeof(struct XXH_state32_t); -} - - -XXH_errorcode XXH32_resetState(void* state_in, U32 seed) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - state->seed = seed; - state->v1 = seed + PRIME32_1 + PRIME32_2; - state->v2 = seed + PRIME32_2; - state->v3 = seed + 0; - state->v4 = seed - PRIME32_1; - state->total_len = 0; - state->memsize = 0; - return XXH_OK; -} - - -void* XXH32_init (U32 seed) -{ - void* state = XXH_malloc (sizeof(struct XXH_state32_t)); - XXH32_resetState(state, seed); - return state; -} - - -FORCE_INLINE XXH_errorcode XXH32_update_endian (void* state_in, const void* input, int len, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE* p = (const BYTE*)input; - const BYTE* const bEnd = p + len; - -#ifdef XXH_ACCEPT_NULL_INPUT_POINTER - if (input==NULL) return XXH_ERROR; -#endif - - state->total_len += len; - - if (state->memsize + len < 16) // fill in tmp buffer - { - XXH_memcpy(state->memory + state->memsize, input, len); - state->memsize += len; - return XXH_OK; - } - - if (state->memsize) // some data left from previous update - { - XXH_memcpy(state->memory + state->memsize, input, 16-state->memsize); - { - const U32* p32 = (const U32*)state->memory; - state->v1 += XXH_readLE32(p32, endian) * PRIME32_2; state->v1 = XXH_rotl32(state->v1, 13); state->v1 *= PRIME32_1; p32++; - state->v2 += XXH_readLE32(p32, endian) * PRIME32_2; state->v2 = XXH_rotl32(state->v2, 13); state->v2 *= PRIME32_1; p32++; - state->v3 += XXH_readLE32(p32, endian) * PRIME32_2; state->v3 = XXH_rotl32(state->v3, 13); state->v3 *= PRIME32_1; p32++; - state->v4 += XXH_readLE32(p32, endian) * PRIME32_2; state->v4 = XXH_rotl32(state->v4, 13); state->v4 *= PRIME32_1; p32++; - } - p += 16-state->memsize; - state->memsize = 0; - } - - if (p <= bEnd-16) - { - const BYTE* const limit = bEnd - 16; - U32 v1 = state->v1; - U32 v2 = state->v2; - U32 v3 = state->v3; - U32 v4 = state->v4; - - do - { - v1 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v1 = XXH_rotl32(v1, 13); v1 *= PRIME32_1; p+=4; - v2 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v2 = XXH_rotl32(v2, 13); v2 *= PRIME32_1; p+=4; - v3 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v3 = XXH_rotl32(v3, 13); v3 *= PRIME32_1; p+=4; - v4 += XXH_readLE32((const U32*)p, endian) * PRIME32_2; v4 = XXH_rotl32(v4, 13); v4 *= PRIME32_1; p+=4; - } while (p<=limit); - - state->v1 = v1; - state->v2 = v2; - state->v3 = v3; - state->v4 = v4; - } - - if (p < bEnd) - { - XXH_memcpy(state->memory, p, bEnd-p); - state->memsize = (int)(bEnd-p); - } - - return XXH_OK; -} - -XXH_errorcode XXH32_update (void* state_in, const void* input, int len) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_update_endian(state_in, input, len, XXH_littleEndian); - else - return XXH32_update_endian(state_in, input, len, XXH_bigEndian); -} - - - -FORCE_INLINE U32 XXH32_intermediateDigest_endian (void* state_in, XXH_endianess endian) -{ - struct XXH_state32_t * state = (struct XXH_state32_t *) state_in; - const BYTE * p = (const BYTE*)state->memory; - BYTE* bEnd = (BYTE*)state->memory + state->memsize; - U32 h32; - - if (state->total_len >= 16) - { - h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); - } - else - { - h32 = state->seed + PRIME32_5; - } - - h32 += (U32) state->total_len; - - while (p<=bEnd-4) - { - h32 += XXH_readLE32((const U32*)p, endian) * PRIME32_3; - h32 = XXH_rotl32(h32, 17) * PRIME32_4; - p+=4; - } - - while (p> 15; - h32 *= PRIME32_2; - h32 ^= h32 >> 13; - h32 *= PRIME32_3; - h32 ^= h32 >> 16; - - return h32; -} - - -U32 XXH32_intermediateDigest (void* state_in) -{ - XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; - - if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) - return XXH32_intermediateDigest_endian(state_in, XXH_littleEndian); - else - return XXH32_intermediateDigest_endian(state_in, XXH_bigEndian); -} - - -U32 XXH32_digest (void* state_in) -{ - U32 h32 = XXH32_intermediateDigest(state_in); - - XXH_free(state_in); - - return h32; -} - diff --git a/src/xxhash.h b/src/xxhash.h deleted file mode 100644 index fa2d0f4..0000000 --- a/src/xxhash.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - xxHash - Fast Hash algorithm - Header File - Copyright (C) 2012-2014, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - You can contact the author at : - - xxHash source repository : http://code.google.com/p/xxhash/ -*/ - -/* Notice extracted from xxHash homepage : - -xxHash is an extremely fast Hash algorithm, running at RAM speed limits. -It also successfully passes all tests from the SMHasher suite. - -Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) - -Name Speed Q.Score Author -xxHash 5.4 GB/s 10 -CrapWow 3.2 GB/s 2 Andrew -MumurHash 3a 2.7 GB/s 10 Austin Appleby -SpookyHash 2.0 GB/s 10 Bob Jenkins -SBox 1.4 GB/s 9 Bret Mulvey -Lookup3 1.2 GB/s 9 Bob Jenkins -SuperFastHash 1.2 GB/s 1 Paul Hsieh -CityHash64 1.05 GB/s 10 Pike & Alakuijala -FNV 0.55 GB/s 5 Fowler, Noll, Vo -CRC32 0.43 GB/s 9 -MD5-32 0.33 GB/s 10 Ronald L. Rivest -SHA1-32 0.28 GB/s 10 - -Q.Score is a measure of quality of the hash function. -It depends on successfully passing SMHasher test set. -10 is a perfect score. -*/ - -#pragma once - -#if defined (__cplusplus) -extern "C" { -#endif - - -//**************************** -// Type -//**************************** -typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; - - - -//**************************** -// Simple Hash Functions -//**************************** - -unsigned int XXH32 (const void* input, int len, unsigned int seed); - -/* -XXH32() : - Calculate the 32-bits hash of sequence of length "len" stored at memory address "input". - The memory between input & input+len must be valid (allocated and read-accessible). - "seed" can be used to alter the result predictably. - This function successfully passes all SMHasher tests. - Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s - Note that "len" is type "int", which means it is limited to 2^31-1. - If your data is larger, use the advanced functions below. -*/ - - - -//**************************** -// Advanced Hash Functions -//**************************** - -void* XXH32_init (unsigned int seed); -XXH_errorcode XXH32_update (void* state, const void* input, int len); -unsigned int XXH32_digest (void* state); - -/* -These functions calculate the xxhash of an input provided in several small packets, -as opposed to an input provided as a single block. - -It must be started with : -void* XXH32_init() -The function returns a pointer which holds the state of calculation. - -This pointer must be provided as "void* state" parameter for XXH32_update(). -XXH32_update() can be called as many times as necessary. -The user must provide a valid (allocated) input. -The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. -Note that "len" is type "int", which means it is limited to 2^31-1. -If your data is larger, it is recommended to chunk your data into blocks -of size for example 2^30 (1GB) to avoid any "int" overflow issue. - -Finally, you can end the calculation anytime, by using XXH32_digest(). -This function returns the final 32-bits hash. -You must provide the same "void* state" parameter created by XXH32_init(). -Memory will be freed by XXH32_digest(). -*/ - - -int XXH32_sizeofState(void); -XXH_errorcode XXH32_resetState(void* state, unsigned int seed); - -#define XXH32_SIZEOFSTATE 48 -typedef struct { long long ll[(XXH32_SIZEOFSTATE+(sizeof(long long)-1))/sizeof(long long)]; } XXH32_stateSpace_t; -/* -These functions allow user application to make its own allocation for state. - -XXH32_sizeofState() is used to know how much space must be allocated for the xxHash 32-bits state. -Note that the state must be aligned to access 'long long' fields. Memory must be allocated and referenced by a pointer. -This pointer must then be provided as 'state' into XXH32_resetState(), which initializes the state. - -For static allocation purposes (such as allocation on stack, or freestanding systems without malloc()), -use the structure XXH32_stateSpace_t, which will ensure that memory space is large enough and correctly aligned to access 'long long' fields. -*/ - - -unsigned int XXH32_intermediateDigest (void* state); -/* -This function does the same as XXH32_digest(), generating a 32-bit hash, -but preserve memory context. -This way, it becomes possible to generate intermediate hashes, and then continue feeding data with XXH32_update(). -To free memory context, use XXH32_digest(), or free(). -*/ - - - -//**************************** -// Deprecated function names -//**************************** -// The following translations are provided to ease code transition -// You are encouraged to no longer this function names -#define XXH32_feed XXH32_update -#define XXH32_result XXH32_digest -#define XXH32_getIntermediateResult XXH32_intermediateDigest - - - -#if defined (__cplusplus) -} -#endif -