diff --git a/cmake/modules/FindLZ4.cmake b/cmake/modules/FindLZ4.cmake new file mode 100644 index 0000000000000..c9492039262fd --- /dev/null +++ b/cmake/modules/FindLZ4.cmake @@ -0,0 +1,41 @@ +# Find the LZ4 includes and library. +# +# This module defines +# LZ4_INCLUDE_DIR, where to locate LZ4 header files +# LZ4_LIBRARIES, the libraries to link against to use LZ4 +# LZ4_FOUND. If false, you cannot build anything that requires LZ4. + +if(LZ4_CONFIG_EXECUTABLE) + set(LZ4_FIND_QUIETLY 1) +endif() +set(LZ4_FOUND 0) + +find_path(LZ4_INCLUDE_DIR lz4.h + $ENV{LZ4_DIR}/include + /usr/local/include + /opt/lzma/include + DOC "Specify the directory containing lz4.h" +) + +find_library(LZ4_LIBRARY NAMES lz4 PATHS + $ENV{LZ4_DIR}/lib + /usr/local/lz4/lib + /usr/local/lib + /usr/lib/lz4 + /usr/local/lib/lz4 + /usr/lz4/lib /usr/lib + /usr/lz4 /usr/local/lz4 + /opt/lz4 /opt/lz4/lib + DOC "Specify the lz4 library here." +) + +if(LZ4_INCLUDE_DIR AND LZ4_LIBRARY) + set(LZ4_FOUND 1) + if(NOT LZ4_FIND_QUIETLY) + message(STATUS "Found LZ4 includes at ${LZ4_INCLUDE_DIR}") + message(STATUS "Found LZ4 library at ${LZ4_LIBRARY}") + endif() +endif() + +set(LZ4_LIBRARIES ${LZ4_LIBRARY}) +mark_as_advanced(LZ4_FOUND LZ4_LIBRARY LZ4_INCLUDE_DIR) diff --git a/cmake/modules/FindLZMA.cmake b/cmake/modules/FindLZMA.cmake index 8a06c5f0eb09d..c27540afe4026 100644 --- a/cmake/modules/FindLZMA.cmake +++ b/cmake/modules/FindLZMA.cmake @@ -2,8 +2,8 @@ # # This module defines # LZMA_INCLUDE_DIR, where to locate LZMA header files -# LZMA_LIBRARIES, the libraries to link against to use Pythia6 -# LZMA_FOUND. If false, you cannot build anything that requires Pythia6. +# LZMA_LIBRARIES, the libraries to link against to use LZMA +# LZMA_FOUND. If false, you cannot build anything that requires LZMA. if(LZMA_CONFIG_EXECUTABLE) set(LZMA_FIND_QUIETLY 1) diff --git a/cmake/modules/RootBuildOptions.cmake b/cmake/modules/RootBuildOptions.cmake index c2d2274d6bd66..13dfd4e25bddf 100644 --- a/cmake/modules/RootBuildOptions.cmake +++ b/cmake/modules/RootBuildOptions.cmake @@ -75,6 +75,7 @@ ROOT_BUILD_OPTION(builtin_openssl OFF "Build OpenSSL internally, or use system O ROOT_BUILD_OPTION(builtin_pcre OFF "Built included libpcre, or use system libpcre") ROOT_BUILD_OPTION(builtin_zlib OFF "Built included libz, or use system libz") ROOT_BUILD_OPTION(builtin_lzma OFF "Built included liblzma, or use system liblzma") +ROOT_BUILD_OPTION(builtin_lz4 OFF "Built included liblz4, or use system liblz4") ROOT_BUILD_OPTION(builtin_davix OFF "Built the Davix library internally (downloading tarfile from the Web)") ROOT_BUILD_OPTION(builtin_gsl OFF "Built the GSL library internally (downloading tarfile from the Web)") ROOT_BUILD_OPTION(builtin_cfitsio OFF "Built the FITSIO library internally (downloading tarfile from the Web)") diff --git a/cmake/modules/RootConfiguration.cmake b/cmake/modules/RootConfiguration.cmake index 72d49581b1e58..19e3a7b9f9d08 100644 --- a/cmake/modules/RootConfiguration.cmake +++ b/cmake/modules/RootConfiguration.cmake @@ -435,6 +435,11 @@ if(lzma) else() set(haslzmacompression undef) endif() +if(lz4) + set(haslz4compression define) +else() + set(haslz4compression undef) +endif() if(cocoa) set(hascocoa define) else() diff --git a/cmake/modules/SearchInstalledSoftware.cmake b/cmake/modules/SearchInstalledSoftware.cmake index 7f03472f5ada4..a7350f4cb095e 100644 --- a/cmake/modules/SearchInstalledSoftware.cmake +++ b/cmake/modules/SearchInstalledSoftware.cmake @@ -180,6 +180,38 @@ if(builtin_lzma) endif() +#---Check for LZ4-------------------------------------------------------------------- +if(NOT builtin_lz4) + message(STATUS "Looking for LZ4") + find_package(LZ4) + if(LZ4_FOUND) + else() + message(STATUS "LZ4 not found. Switching on builtin_lz4 option") + set(builtin_lz4 ON CACHE BOOL "" FORCE) + endif() +endif() +if(builtin_lz4) + set(lz4_version r127) + message(STATUS "Building LZ4 version ${lz4_version} included in ROOT itself") + if(CMAKE_CXX_COMPILER_ID STREQUAL Clang) + set(LZ4_CFLAGS "-Wno-format-nonliteral") + elseif( CMAKE_CXX_COMPILER_ID STREQUAL Intel) + set(LZ4_CFLAGS "-wd188 -wd181 -wd1292 -wd10006 -wd10156 -wd2259 -wd981 -wd128 -wd3179") + endif() + ExternalProject_Add( + LZ4 + URL ${CMAKE_SOURCE_DIR}/core/lz4/src/lz4-${lz4_version}.tar.gz + URL_MD5 0601f6b8477209d07db33d504feb6ac4 + INSTALL_DIR ${CMAKE_BINARY_DIR} + CONFIGURE_COMMAND PREFIX= make cmake + BUILD_COMMAND PREFIX= MOREFLAGS=-fPIC make + INSTALL_COMMAND PREFIX= make install + BUILD_IN_SOURCE 1) + set(LZ4_LIBRARIES ${CMAKE_BINARY_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(LZ4_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include) +endif() + + #---Check for X11 which is mandatory lib on Unix-------------------------------------- if(x11) message(STATUS "Looking for X11") diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 22dca7c86bc2a..9bbf742bb6eea 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -1,7 +1,6 @@ ############################################################################ # CMakeLists.txt file for building ROOT (global) core package ############################################################################ - add_subdirectory(rint) add_subdirectory(thread) add_subdirectory(multiproc) @@ -31,13 +30,15 @@ if(cocoa) endif() add_subdirectory(zip) add_subdirectory(lzma) +add_subdirectory(lz4) add_subdirectory(base) set(objectlibs $ $ $ - $ $ + $ + $ $ $ $ @@ -96,7 +97,7 @@ add_subdirectory(utils) ROOT_LINKER_LIBRARY(Core $ ${objectlibs} - LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY} + LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${LZ4_LIBRARIES} ${ZLIB_LIBRARY} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${corelinklibs} ) if(cling) @@ -108,5 +109,8 @@ endif() if(builtin_lzma) ROOT_ADD_BUILTIN_DEPENDENCIES(Core LZMA) endif() +if(builtin_lz4) + ROOT_ADD_BUILTIN_DEPENDENCIES(Core LZ4) +endif() #---------------------------------------------------------------------------------------- diff --git a/core/lz4/CMakeLists.txt b/core/lz4/CMakeLists.txt new file mode 100644 index 0000000000000..fb2529c414bc5 --- /dev/null +++ b/core/lz4/CMakeLists.txt @@ -0,0 +1,22 @@ +############################################################################ +# CMakeLists.txt file for building ROOT core/lz4 package +############################################################################ + + +#---The builtin LZ4 library is built using the CMake ExternalProject standard module +# in cmake/modules/SearchInstalledSoftare.cmake + +#---Declare ZipLZ4 sources as part of libCore------------------------------- +set(headers ${CMAKE_CURRENT_SOURCE_DIR}/inc/ZipLZ4.h) +set(sources ${CMAKE_CURRENT_SOURCE_DIR}/src/ZipLZ4.c) + + +include_directories(${LZ4_INCLUDE_DIR}) +ROOT_OBJECT_LIBRARY(Lz4 ${sources}) + +if(builtin_lz4) + add_dependencies(Lz4 LZ4) +endif() + +ROOT_INSTALL_HEADERS() +install(FILES ${LZ4_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/core/lz4/inc/ZipLZ4.h b/core/lz4/inc/ZipLZ4.h new file mode 100644 index 0000000000000..3a82767b26b5c --- /dev/null +++ b/core/lz4/inc/ZipLZ4.h @@ -0,0 +1,13 @@ +// Author: Brian Bockelman March 2015 + +/************************************************************************* + * Copyright (C) 1995-2015, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep); + +void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep); diff --git a/core/lz4/src/ZipLZ4.c b/core/lz4/src/ZipLZ4.c new file mode 100644 index 0000000000000..3798620b98a41 --- /dev/null +++ b/core/lz4/src/ZipLZ4.c @@ -0,0 +1,99 @@ +// Original Author: Brian Bockelman + +/************************************************************************* + * Copyright (C) 1995-2015, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#include "ZipLZ4.h" +#include "lz4.h" +#include "lz4hc.h" +#include +#include + +#if (__GNUC__ >= 3) || defined(__INTEL_COMPILER) +#if !defined(R__unlikely) + #define R__unlikely(expr) __builtin_expect(!!(expr), 0) +#endif +#if !defined(R__likely) + #define R__likely(expr) __builtin_expect(!!(expr), 1) +#endif +#else + #define R__unlikely(expr) expr + #define R__likely(expr) expr +#endif + +static const int kHeaderSize = 9; + +void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, int *irep) +{ + int LZ4_version = LZ4_versionNumber(); + uint64_t out_size; /* compressed size */ + uint64_t in_size = (unsigned) (*srcsize); + + *irep = 0; + + if (*tgtsize <= 0) { + return; + } + + if (*srcsize > 0xffffff || *srcsize < 0) { + return; + } + + int returnStatus; + if (cxlevel > 9) { + cxlevel = 9; + } + if (cxlevel >= 4) { + returnStatus = LZ4_compress_HC(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize, cxlevel); + } else { + returnStatus = LZ4_compress_default(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize); + } + + if (R__unlikely(returnStatus == 0)) { /* LZ4 compression failed */ + return; + } + + tgt[0] = 'L'; + tgt[1] = '4'; + tgt[2] = (LZ4_version / (100*100)); + + out_size = returnStatus; /* compressed size */ + + tgt[3] = (char)(out_size & 0xff); + tgt[4] = (char)((out_size >> 8) & 0xff); + tgt[5] = (char)((out_size >> 16) & 0xff); + + tgt[6] = (char)(in_size & 0xff); /* decompressed size */ + tgt[7] = (char)((in_size >> 8) & 0xff); + tgt[8] = (char)((in_size >> 16) & 0xff); + + *irep = (int)returnStatus + kHeaderSize; +} + +void R__unzipLZ4(int *srcsize, unsigned char *src, int *tgtsize, unsigned char *tgt, int *irep) +{ + int LZ4_version = LZ4_versionNumber()/(100*100); + *irep = 0; + if (R__unlikely(src[0] != 'L' || src[1] != '4')) { + fprintf(stderr, "R__unzipLZ4: algorithm run against buffer with incorrect header (got %d%d; expected %d%d).\n", src[0], src[1], 'L', '4'); + return; + } + if (R__unlikely(src[2] != LZ4_version)) { + fprintf(stderr, "R__unzipLZ4: This version of LZ4 is incompatible with the on-disk version (got %d; expected %d).\n", src[2], LZ4_version); + return; + } + + int returnStatus = LZ4_decompress_safe((char *)(&src[kHeaderSize]), (char *)(tgt), *srcsize-kHeaderSize, *tgtsize); + if (R__unlikely(returnStatus < 0)) { + fprintf(stderr, "R__unzipLZ4: error in decompression around byte %d out of maximum %d.\n", -returnStatus, *tgtsize); + return; + } + + *irep = returnStatus; +} + diff --git a/core/lz4/src/lz4-r127 b/core/lz4/src/lz4-r127 new file mode 160000 index 0000000000000..7ed257191b65d --- /dev/null +++ b/core/lz4/src/lz4-r127 @@ -0,0 +1 @@ +Subproject commit 7ed257191b65d7185a5847a21d14aaa7948e2104 diff --git a/core/lz4/src/lz4-r127.tar.gz b/core/lz4/src/lz4-r127.tar.gz new file mode 100644 index 0000000000000..50879e3305b55 Binary files /dev/null and b/core/lz4/src/lz4-r127.tar.gz differ diff --git a/core/unix/inc/TSignalHandler.h b/core/unix/inc/TSignalHandler.h new file mode 100644 index 0000000000000..d5f49c97100ff --- /dev/null +++ b/core/unix/inc/TSignalHandler.h @@ -0,0 +1,246 @@ +// @(#)root/unix:$Id$ +// Author: Fons Rademakers 15/09/95 + +/************************************************************************* + * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#ifndef ROOT_TUnixSystem +#define ROOT_TUnixSystem + + +////////////////////////////////////////////////////////////////////////// +// // +// TUnixSystem // +// // +// Class providing an interface to the UNIX Operating System. // +// // +////////////////////////////////////////////////////////////////////////// + +#ifndef ROOT_TSystem +#include "TSystem.h" +#endif +#ifndef ROOT_TSysEvtHandler +#include "TSysEvtHandler.h" +#endif +#ifndef ROOT_TTimer +#include "TTimer.h" +#endif + +#include +#include + +int StackTraceExec(void *); + +typedef void (*SigHandler_t)(ESignals); + + +class TUnixSystem : public TSystem { + + friend int StackTraceExec(void *); + +private: + struct StackTraceHelper_t { + static const int fStringLength = 255; + char fShellExec[fStringLength]; + char fPidString[fStringLength]; + char fPidNum[fStringLength]; + int fParentToChild[2]; + int fChildToParent[2]; + std::unique_ptr fHelperThread; + }; + + static StackTraceHelper_t fStackTraceHelper; + +protected: + const char *FindDynamicLibrary(TString &lib, Bool_t quiet = kFALSE); + const char *GetLinkedLibraries(); + + // static functions providing semi-low level interface to raw Unix + static int UnixMakedir(const char *name); + static void *UnixOpendir(const char *name); + static const char *UnixGetdirentry(void *dir); + static const char *UnixHomedirectory(const char *user = 0); + static Long64_t UnixNow(); + static int UnixWaitchild(); + static int UnixSetitimer(Long_t ms); + static int UnixSelect(Int_t nfds, TFdSet *readready, TFdSet *writeready, + Long_t timeout); + static void UnixSignal(ESignals sig, SigHandler_t h); + static const char *UnixSigname(ESignals sig); + static void UnixSigAlarmInterruptsSyscalls(Bool_t set); + static void UnixResetSignal(ESignals sig); + static void UnixResetSignals(); + static void UnixIgnoreSignal(ESignals sig, Bool_t ignore); + static int UnixFilestat(const char *path, FileStat_t &buf); + static int UnixFSstat(const char *path, Long_t *id, Long_t *bsize, + Long_t *blocks, Long_t *bfree); + static int UnixTcpConnect(const char *hostname, int port, int tcpwindowsize); + static int UnixUdpConnect(const char *hostname, int port); + static int UnixUnixConnect(int port); + static int UnixUnixConnect(const char *path); + static int UnixTcpService(int port, Bool_t reuse, int backlog, + int tcpwindowsize); + static int UnixUdpService(int port, int backlog); + static int UnixUnixService(int port, int backlog); + static int UnixUnixService(const char *sockpath, int backlog); + static int UnixRecv(int sock, void *buf, int len, int flag); + static int UnixSend(int sock, const void *buf, int len, int flag); + + // added helper static members for stacktrace + static char *const kStackArgv[]; + static char *const *GetStackArgv(); + static void StackTraceHelperThread(); + void CachePidInfo(); + +public: + TUnixSystem(); + virtual ~TUnixSystem(); + + //---- Misc ------------------------------------------------- + Bool_t Init(); + void SetProgname(const char *name); + void SetDisplay(); + const char *GetError(); + const char *HostName(); + + //---- EventLoop -------------------------------------------- + void DispatchOneEvent(Bool_t pendingOnly = kFALSE); + Int_t Select(TList *active, Long_t timeout); + Int_t Select(TFileHandler *fh, Long_t timeout); + + //---- Handling of system events ---------------------------- + void CheckChilds(); + Bool_t CheckSignals(Bool_t sync); + Bool_t CheckDescriptors(); + void DispatchSignals(ESignals sig); + void AddSignalHandler(TSignalHandler *sh); + TSignalHandler *RemoveSignalHandler(TSignalHandler *sh); + void ResetSignal(ESignals sig, Bool_t reset = kTRUE); + void ResetSignals(); + void IgnoreSignal(ESignals sig, Bool_t ignore = kTRUE); + void SigAlarmInterruptsSyscalls(Bool_t set); + void AddFileHandler(TFileHandler *fh); + TFileHandler *RemoveFileHandler(TFileHandler *fh); + + //---- Floating Point Exceptions Control -------------------- + Int_t GetFPEMask(); + Int_t SetFPEMask(Int_t mask = kDefaultMask); + + //---- Time & Date ------------------------------------------ + TTime Now(); + void AddTimer(TTimer *ti); + TTimer *RemoveTimer(TTimer *ti); + void ResetTimer(TTimer *ti); + Bool_t DispatchTimers(Bool_t mode); + void Sleep(UInt_t milliSec); + + //---- Processes -------------------------------------------- + Int_t Exec(const char *shellcmd); + FILE *OpenPipe(const char *shellcmd, const char *mode); + int ClosePipe(FILE *pipe); + void Exit(int code, Bool_t mode = kTRUE); + void Abort(int code = 0); + int GetPid(); + void StackTrace(); + static void StackTraceFromThread(); + + //---- Directories ------------------------------------------ + int MakeDirectory(const char *name); + void *OpenDirectory(const char *name); + void FreeDirectory(void *dirp); + const char *GetDirEntry(void *dirp); + Bool_t ChangeDirectory(const char *path); + const char *WorkingDirectory(); + const char *HomeDirectory(const char *userName = 0); + const char *TempDirectory() const; + FILE *TempFileName(TString &base, const char *dir = 0); + + //---- Paths & Files ---------------------------------------- + const char *PrependPathName(const char *dir, TString& name); + Bool_t ExpandPathName(TString &patbuf); + char *ExpandPathName(const char *path); + Bool_t AccessPathName(const char *path, EAccessMode mode = kFileExists); + Bool_t IsPathLocal(const char *path); + int CopyFile(const char *from, const char *to, Bool_t overwrite = kFALSE); + int Rename(const char *from, const char *to); + int Link(const char *from, const char *to); + int Symlink(const char *from, const char *to); + int Unlink(const char *name); + int GetPathInfo(const char *path, FileStat_t &buf); + int GetFsInfo(const char *path, Long_t *id, Long_t *bsize, + Long_t *blocks, Long_t *bfree); + int Chmod(const char *file, UInt_t mode); + int Umask(Int_t mask); + int Utime(const char *file, Long_t modtime, Long_t actime); + const char *FindFile(const char *search, TString& file, EAccessMode mode = kFileExists); + + //---- Users & Groups --------------------------------------- + Int_t GetUid(const char *user = 0); + Int_t GetGid(const char *group = 0); + Int_t GetEffectiveUid(); + Int_t GetEffectiveGid(); + UserGroup_t *GetUserInfo(Int_t uid); + UserGroup_t *GetUserInfo(const char *user = 0); + UserGroup_t *GetGroupInfo(Int_t gid); + UserGroup_t *GetGroupInfo(const char *group = 0); + + //---- Environment Manipulation ----------------------------- + const char *Getenv(const char *name); + void Setenv(const char *name, const char *value); + void Unsetenv(const char *name); + + //---- System Logging --------------------------------------- + void Openlog(const char *name, Int_t options, ELogFacility facility); + void Syslog(ELogLevel level, const char *mess); + void Closelog(); + + //---- Standard Output redirection -------------------------- + Int_t RedirectOutput(const char *name, const char *mode = "a", + RedirectHandle_t *h = 0); + + //---- Dynamic Loading -------------------------------------- + void AddDynamicPath(const char *lib); + const char *GetDynamicPath(); + void SetDynamicPath(const char *lib); + Func_t DynFindSymbol(const char *module, const char *entry); + int Load(const char *module, const char *entry = "", Bool_t system = kFALSE); + void Unload(const char *module); + void ListSymbols(const char *module, const char *re = ""); + void ListLibraries(const char *regexp = ""); + + //---- RPC -------------------------------------------------- + TInetAddress GetHostByName(const char *server); + TInetAddress GetPeerName(int sock); + TInetAddress GetSockName(int sock); + int GetServiceByName(const char *service); + char *GetServiceByPort(int port); + int ConnectService(const char *server, int port, int tcpwindowsize, const char *protocol = "tcp"); + int OpenConnection(const char *server, int port, int tcpwindowsize = -1, const char *protocol = "tcp"); + int AnnounceTcpService(int port, Bool_t reuse, int backlog, int tcpwindowsize = -1); + int AnnounceUdpService(int port, int backlog); + int AnnounceUnixService(int port, int backlog); + int AnnounceUnixService(const char *sockpath, int backlog); + int AcceptConnection(int sock); + void CloseConnection(int sock, Bool_t force = kFALSE); + int RecvRaw(int sock, void *buffer, int length, int flag); + int SendRaw(int sock, const void *buffer, int length, int flag); + int RecvBuf(int sock, void *buffer, int length); + int SendBuf(int sock, const void *buffer, int length); + int SetSockOpt(int sock, int option, int val); + int GetSockOpt(int sock, int option, int *val); + + //---- System, CPU and Memory info + int GetSysInfo(SysInfo_t *info) const; + int GetCpuInfo(CpuInfo_t *info, Int_t sampleTime = 1000) const; + int GetMemInfo(MemInfo_t *info) const; + int GetProcInfo(ProcInfo_t *info) const; + + ClassDef(TUnixSystem,0) //Interface to Unix OS services +}; + +#endif diff --git a/core/utils/CMakeLists.txt b/core/utils/CMakeLists.txt index 85b621c8a5235..17d24d483596b 100644 --- a/core/utils/CMakeLists.txt +++ b/core/utils/CMakeLists.txt @@ -26,13 +26,14 @@ ROOT_EXECUTABLE(rootcling src/LinkdefReader.cxx src/DictSelectionReader.cxx $ $ $ + $ $ $ $ ${macosx_objects} ${unix_objects} ${winnt_objects} - LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${ZLIB_LIBRARY} + LIBRARIES ${PCRE_LIBRARIES} ${LZMA_LIBRARIES} ${LZ4_LIBRARIES} ${ZLIB_LIBRARY} ${CLING_LIBRARIES} ${CMAKE_DL_LIBS} ${CMAKE_THREAD_LIBS_INIT} ${corelinklibs}) diff --git a/core/zip/inc/Compression.h b/core/zip/inc/Compression.h index 90f985efe5924..77501129231fe 100644 --- a/core/zip/inc/Compression.h +++ b/core/zip/inc/Compression.h @@ -26,6 +26,9 @@ namespace ROOT { // and memory when compressing. LZMA memory usage is particularly // high for compression levels 8 and 9. // + // Finally, the LZ4 package results in worse compression ratios + // than ZLIB but achieves much faster decompression rates. + // // The current algorithms support level 1 to 9. The higher // the level the greater the compression and more CPU time // and memory resources used during compression. Level 0 @@ -34,6 +37,7 @@ namespace ROOT { kZLIB, kLZMA, kOldCompressionAlgo, + kLZ4, // if adding new algorithm types, // keep this enum value last kUndefinedCompressionAlgorithm diff --git a/core/zip/src/Bits.h b/core/zip/src/Bits.h index 12543359eef5e..b4c5518506a14 100644 --- a/core/zip/src/Bits.h +++ b/core/zip/src/Bits.h @@ -15,6 +15,7 @@ #include "zlib.h" #include "RConfigure.h" #include "ZipLZMA.h" +#include "ZipLZ4.h" #include #include @@ -287,11 +288,16 @@ struct bits_internal_state { and when R__zipMultipleAlgorithm is called with its last argument set to 0. R__ZipMode = 1 : ZLIB compression algorithm is used (default) R__ZipMode = 2 : LZMA compression algorithm is used + R__ZipMode = 4 : LZ4 compression algorithm is used R__ZipMode = 0 or 3 : a very old compression algorithm is used (the very old algorithm is supported for backward compatibility) The LZMA algorithm requires the external XZ package be installed when linking is done. LZMA typically has significantly higher compression factors, but takes more CPU time and memory resources while compressing. + + The LZ4 algorithm requires the external LZ4 package to be installed when linking + is done. LZ4 typically has the worst compression ratios, but much faster decompression + speeds - sometimes by an order of magnitude. */ int R__ZipMode = 1; @@ -583,6 +589,9 @@ void R__zipMultipleAlgorithm(int cxlevel, int *srcsize, char *src, int *tgtsize, if (compressionAlgorithm == 2) { R__zipLZMA(cxlevel, srcsize, src, tgtsize, tgt, irep); return; + } else if (compressionAlgorithm == 4) { + R__zipLZ4(cxlevel, srcsize, src, tgtsize, tgt, irep); + return; } // The very old algorithm for backward compatibility diff --git a/core/zip/src/ZInflate.c b/core/zip/src/ZInflate.c index 47e0078d821e0..b6a9365c8edfe 100644 --- a/core/zip/src/ZInflate.c +++ b/core/zip/src/ZInflate.c @@ -20,6 +20,7 @@ static const int qflag = 0; #include "zlib.h" #include "RConfigure.h" #include "ZipLZMA.h" +#include "ZipLZ4.h" /* inflate.c -- put in the public domain by Mark Adler @@ -1130,8 +1131,9 @@ int R__unzip_header(int *srcsize, uch *src, int *tgtsize) /* C H E C K H E A D E R */ if (!(src[0] == 'Z' && src[1] == 'L' && src[2] == Z_DEFLATED) && !(src[0] == 'C' && src[1] == 'S' && src[2] == Z_DEFLATED) && - !(src[0] == 'X' && src[1] == 'Z' && src[2] == 0)) { - fprintf(stderr, "Error R__unzip_header: error in header\n"); + !(src[0] == 'X' && src[1] == 'Z' && src[2] == 0) && + !(src[0] == 'L' && src[1] == '4')) { + fprintf(stderr, "Error R__unzip_header: error in header. Values: %c%c\n", src[0], src[1]); return 1; } @@ -1159,7 +1161,8 @@ void R__unzip(int *srcsize, uch *src, int *tgtsize, uch *tgt, int *irep) /* C H E C K H E A D E R */ if (!(src[0] == 'Z' && src[1] == 'L' && src[2] == Z_DEFLATED) && !(src[0] == 'C' && src[1] == 'S' && src[2] == Z_DEFLATED) && - !(src[0] == 'X' && src[1] == 'Z' && src[2] == 0)) { + !(src[0] == 'X' && src[1] == 'Z' && src[2] == 0) && + !(src[0] == 'L' && src[1] == '4')) { fprintf(stderr,"Error R__unzip: error in header\n"); return; } @@ -1217,6 +1220,10 @@ void R__unzip(int *srcsize, uch *src, int *tgtsize, uch *tgt, int *irep) R__unzipLZMA(srcsize, src, tgtsize, tgt, irep); return; } + else if (src[0] == 'L' && src[1] == '4') { + R__unzipLZ4(srcsize, src, tgtsize, tgt, irep); + return; + } /* Old zlib format */ if (R__Inflate(&ibufptr, &ibufcnt, &obufptr, &obufcnt)) { diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3ad0a248eebec..fc93b34452139 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -38,9 +38,21 @@ ROOT_EXECUTABLE(hworld hworld.cxx LIBRARIES Gpad) ROOT_GENERATE_DICTIONARY(EventDict ${CMAKE_CURRENT_SOURCE_DIR}/Event.h MODULE Event LINKDEF EventLinkDef.h) ROOT_LINKER_LIBRARY(Event Event.cxx EventDict.cxx LIBRARIES Hist MathCore) -ROOT_EXECUTABLE(eventexe MainEvent.cxx LIBRARIES Event RIO Tree Hist Net) +ROOT_EXECUTABLE(eventexe MainEvent.cxx LIBRARIES Event RIO Tree TreePlayer Hist Net) ROOT_ADD_TEST(test-event COMMAND eventexe) +#---localcompression------------------------------------------------------------------------------------- +ROOT_GENERATE_DICTIONARY(LocalcompressionDict ${CMAKE_CURRENT_SOURCE_DIR}/Localcompression.h MODULE Localcompression LINKDEF LocalcompressionLinkDef.h) +ROOT_LINKER_LIBRARY(Localcompression Localcompression.cxx LocalcompressionDict.cxx LIBRARIES Hist MathCore) + +ROOT_EXECUTABLE(dummyexe MainDummy.cxx LIBRARIES Localcompression RIO Tree TreePlayer Hist Net) +ROOT_ADD_TEST(test-dummy COMMAND dummyexe) + +ROOT_EXECUTABLE(localcompressionexe MainLocalcompression.cxx LIBRARIES Localcompression RIO Tree Hist Net) +ROOT_ADD_TEST(test-localcompression COMMAND localcompressionexe) +ROOT_EXECUTABLE(localcombineexe MainLocalcombine.cxx LIBRARIES Localcompression RIO Tree Hist Net) +ROOT_ADD_TEST(test-localcombine COMMAND localcombineexe) + #---hsimple------------------------------------------------------------------------------------ #ROOT_EXECUTABLE(hsimple hsimple.cxx LIBRARIES RIO Tree Hist) #ROOT_ADD_TEST(test-hsimple COMMAND hsimple) diff --git a/test/Localcompression.cxx b/test/Localcompression.cxx new file mode 100644 index 0000000000000..e4c4bb60708fc --- /dev/null +++ b/test/Localcompression.cxx @@ -0,0 +1,359 @@ +// @(#)root/test:$Id$ +// Author: Rene Brun 19/08/96 + +#include "RVersion.h" +#include "TRandom.h" +#include "TDirectory.h" +#include "TProcessID.h" + +#include "Localcompression.h" +#include + +ClassImp(TDummy) +ClassImp(TLarge) +ClassImp(TSmall) +ClassImp(TFloat) + +//////////////////////////////////////////////////////////////////////////////// +/// Create an TDummy. +TDummy::TDummy(Int_t size) +{ + fSize = size; + fDummy = new Float_t[fSize]; + for(int i=0;iRndm(1)); + if(tmp < 0.4) { + dummy.ch[offset] = 'a'; + } else if(tmp < 0.65) { + dummy.ch[offset] = 'b'; + } else if(tmp < 0.9) { + dummy.ch[offset] = 'c'; + } else { + dummy.ch[offset] = 'd'; + } +/* + if(tmp < 0.08167) { + dummy.ch[offset] = 'a'; + } else if(tmp < 0.09659) { + dummy.ch[offset] = 'b'; + } else if(tmp < 0.12441) { + dummy.ch[offset] = 'c'; + } else if(tmp < 0.16694) { + dummy.ch[offset] = 'd'; + } else if(tmp < 0.29396) { + dummy.ch[offset] = 'e'; + } else if(tmp < 0.31624) { + dummy.ch[offset] = 'f'; + } else if(tmp < 0.33639) { + dummy.ch[offset] = 'g'; + } else if(tmp < 0.39733) { + dummy.ch[offset] = 'h'; + } else if(tmp < 0.46699) { + dummy.ch[offset] = 'i'; + } else if(tmp < 0.46852) { + dummy.ch[offset] = 'j'; + } else if(tmp < 0.47624) { + dummy.ch[offset] = 'k'; + } else if(tmp < 0.51649) { + dummy.ch[offset] = 'l'; + } else if(tmp < 0.54055) { + dummy.ch[offset] = 'm'; + } else if(tmp < 0.60804) { + dummy.ch[offset] = 'n'; + } else if(tmp < 0.68311) { + dummy.ch[offset] = 'o'; + } else if(tmp < 0.7024) { + dummy.ch[offset] = 'p'; + } else if(tmp < 0.70335) { + dummy.ch[offset] = 'q'; + } else if(tmp < 0.76322) { + dummy.ch[offset] = 'r'; + } else if(tmp < 0.82649) { + dummy.ch[offset] = 's'; + } else if(tmp < 0.91705) { + dummy.ch[offset] = 't'; + } else if(tmp < 0.94463) { + dummy.ch[offset] = 'u'; + } else if(tmp < 0.95441) { + dummy.ch[offset] = 'v'; + } else if(tmp < 0.97801) { + dummy.ch[offset] = 'w'; + } else if(tmp < 0.97951) { + dummy.ch[offset] = 'x'; + } else if(tmp < 0.99925) { + dummy.ch[offset] = 'y'; + } else { + dummy.ch[offset] = 'z'; + } +*/ +// std::cout << "dummy.ch[" << offset << "]=" << dummy.ch[offset] << ", ";//## + } + fDummy[i] = dummy.fp; +// std::cout << "fDummy[" << i << "]=" << fDummy[i] << std::endl;//## + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Create an TDummy. +TDummy::TDummy(const TDummy& dummy) : TObject(dummy) +{ + Float_t *intermediate = dummy.GetDummy(); + Int_t size = dummy.GetSize(); + fDummy = new Float_t[size]; + for(int i=0;iRndm(1)); + if(tmp < 0.4) { + dummy.ch[offset] = 'a'; + } else if(tmp < 0.65) { + dummy.ch[offset] = 'b'; + } else if(tmp < 0.9) { + dummy.ch[offset] = 'c'; + } else { + dummy.ch[offset] = 'd'; + } +/* + Float_t tmp = Float_t(gRandom->Rndm(1)); + if(tmp < 0.08167) { + dummy.ch[offset] = 'a'; + } else if(tmp < 0.09659) { + dummy.ch[offset] = 'b'; + } else if(tmp < 0.12441) { + dummy.ch[offset] = 'c'; + } else if(tmp < 0.16694) { + dummy.ch[offset] = 'd'; + } else if(tmp < 0.29396) { + dummy.ch[offset] = 'e'; + } else if(tmp < 0.31624) { + dummy.ch[offset] = 'f'; + } else if(tmp < 0.33639) { + dummy.ch[offset] = 'g'; + } else if(tmp < 0.39733) { + dummy.ch[offset] = 'h'; + } else if(tmp < 0.46699) { + dummy.ch[offset] = 'i'; + } else if(tmp < 0.46852) { + dummy.ch[offset] = 'j'; + } else if(tmp < 0.47624) { + dummy.ch[offset] = 'k'; + } else if(tmp < 0.51649) { + dummy.ch[offset] = 'l'; + } else if(tmp < 0.54055) { + dummy.ch[offset] = 'm'; + } else if(tmp < 0.60804) { + dummy.ch[offset] = 'n'; + } else if(tmp < 0.68311) { + dummy.ch[offset] = 'o'; + } else if(tmp < 0.7024) { + dummy.ch[offset] = 'p'; + } else if(tmp < 0.70335) { + dummy.ch[offset] = 'q'; + } else if(tmp < 0.76322) { + dummy.ch[offset] = 'r'; + } else if(tmp < 0.82649) { + dummy.ch[offset] = 's'; + } else if(tmp < 0.91705) { + dummy.ch[offset] = 't'; + } else if(tmp < 0.94463) { + dummy.ch[offset] = 'u'; + } else if(tmp < 0.95441) { + dummy.ch[offset] = 'v'; + } else if(tmp < 0.97801) { + dummy.ch[offset] = 'w'; + } else if(tmp < 0.97951) { + dummy.ch[offset] = 'x'; + } else if(tmp < 0.99925) { + dummy.ch[offset] = 'y'; + } else { + dummy.ch[offset] = 'z'; + } +*/ +// std::cout << "dummy.ch[" << offset << "]=" << dummy.ch[offset] << ", ";//## + } + fDummy[i] = dummy.fp; +// std::cout << "fDummy[" << i << "]=" << fDummy[i] << std::endl;//## + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Create an TLarge. +TLarge::TLarge(Int_t size) +{ + fSize = size; + fLarge = new Float_t[fSize]; + for(int i=0;iRndm(1); + else fLarge[i] = fLarge[i-1]; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Create an TLarge. +TLarge::TLarge(const TLarge& large) : TObject(large) +{ + Float_t *intermediate = large.GetLarge(); + Int_t size = large.GetSize(); + fLarge = new Float_t[size]; + for(int i=0;iRndm(1); + else fLarge[i] = fLarge[i-1]; + } +} + +/////////////////////////////////////////////////////////////////////////////// +/// Create an TSmall. +TSmall::TSmall(Int_t size) +{ + fSize = size; + fSmall = new Float_t[fSize]; + for(int i=0;iRndm(1); + fSmall[i] = fSmall[i-1]; + } +} + +//////////////////////////////////////////////////////////////////////////////// +/// Create an TSmall. +TSmall::TSmall(const TSmall& small) : TObject(small) +{ + Float_t *intermediate = small.GetSmall(); + Int_t size = small.GetSize(); + fSmall = new Float_t[size]; + for(int i=0;iRndm(1); + else fSmall[i] = fSmall[i-1]; + } +} + +/////////////////////////////////////////////////////////////////////////////// + +void TSmall::Clear(Option_t * /*option*/) +{ + TObject::Clear(); + for(int i=0;iRndm(1)); + for(int i=1;iRndm(1)); + for(int i=1;i + +#include "Riostream.h" +#include "TROOT.h" +#include "TFile.h" +#include "TNetFile.h" +#include "TRandom.h" +#include "TTree.h" +#include "TTreePerfStats.h" +#include "TBranch.h" +#include "TClonesArray.h" +#include "TStopwatch.h" + +#include "Localcompression.h" + +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char **argv) +{ + Int_t nevent = 400; // by default create 400 events + Int_t comp = 1; // by default file is compressed + Int_t split = 1; // by default, split Event in sub branches + Int_t write = 1; // by default the tree is filled + Int_t read = 0; + Int_t arg4 = 1; + Int_t arg5 = 600; //default number of tracks per event + Int_t compAlg = 1; + Int_t netf = 0; + Int_t punzip = 0; + + if (argc > 1) nevent = atoi(argv[1]); + if (argc > 2) comp = atoi(argv[2]); + if (argc > 3) split = atoi(argv[3]); + if (argc > 4) arg4 = atoi(argv[4]); + if (argc > 5) arg5 = atoi(argv[5]); + if (argc > 6) compAlg= atoi(argv[6]); + if (arg4 == 0) { write = 0; read = 1;} + if (arg4 == 1) { write = 1;} + if (arg4 == 2) { write = 0;} + if (arg4 == 10) { write = 0;} + if (arg4 == 11) { write = 1;} + if (arg4 == 20) { write = 0; read = 1;} //read sequential + if (arg4 == 21) { write = 0; read = 1; punzip = 1;} //read sequential + parallel unzipping + if (arg4 == 25) { write = 0; read = 2;} //read random + if (arg4 >= 30) { netf = 1; } //use TNetFile + if (arg4 == 30) { write = 0; read = 1;} //netfile + read sequential + if (arg4 == 35) { write = 0; read = 2;} //netfile + read random + if (arg4 == 36) { write = 1; } //netfile + write sequential + Int_t branchStyle = 1; //new style by default + if (split < 0) {branchStyle = 0; split = -1-split;} + + TFile *hfile; + TTree *tree; + TTreePerfStats *ioperf = NULL; + TDummy *dummy = 0; + + // Fill dummy, header and tracks with some random numbers + // Create a timer object to benchmark this loop + TStopwatch timer; + timer.Start(); + Long64_t nb = 0; + Int_t ev; + Int_t bufsize; + Double_t told = 0; + Double_t tnew = 0; + Int_t printev = 100; + if (arg5 < 100) printev = 1000; + if (arg5 < 10) printev = 10000; + +// Read case + if (read) { + if (netf) { + hfile = new TNetFile("root://localhost/root/test/DummyNet.root"); + } else + hfile = new TFile("Dummy.root"); + tree = (TTree*)hfile->Get("T"); + TBranch *branch = tree->GetBranch("dummy"); + branch->SetAddress(&dummy); + Int_t nentries = (Int_t)tree->GetEntries(); + nevent = TMath::Min(nevent,nentries); + if (read == 1) { //read sequential + //by setting the read cache to -1 we set it to the AutoFlush value when writing + ioperf = new TTreePerfStats("Perf Stats", tree); + Int_t cachesize = -1; + if (punzip) tree->SetParallelUnzip(); + tree->SetCacheSize(cachesize); + tree->SetCacheLearnEntries(1); //one entry is sufficient to learn + tree->SetCacheEntryRange(0,nevent); + for (ev = 0; ev < nevent; ev++) { + tree->LoadTree(ev); //this call is required when using the cache + if (ev%printev == 0) { + tnew = timer.RealTime(); + printf("dummy:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + nb += tree->GetEntry(ev); //read complete event in memory + } + ioperf->Finish(); + } else { //read random + Int_t evrandom; + for (ev = 0; ev < nevent; ev++) { + if (ev%printev == 0) std::cout<<"dummy="<Rndm(1)); + nb += tree->GetEntry(evrandom); //read complete event in memory + } + } + } else { +// Write case + // Create a new ROOT binary machine independent file. + // Note that this file may contain any kind of ROOT objects, histograms, + // pictures, graphics objects, detector geometries, tracks, events, etc.. + // This file is now becoming the current directory. + if (netf) { + hfile = new TNetFile("root://localhost/root/test/DummyNet.root","RECREATE","TTree benchmark ROOT file"); + } else + hfile = new TFile("Dummy.root","RECREATE","TTree benchmark ROOT file"); + hfile->SetCompressionLevel(comp); + hfile->SetCompressionAlgorithm(compAlg); + + // Create a ROOT Tree and one superbranch + tree = new TTree("T","An example of a ROOT tree"); + tree->SetAutoSave(1000000000); // autosave when 1 Gbyte written + tree->SetCacheSize(10000000); // set a 10 MBytes cache (useless when writing local files) + bufsize = 64000; + if (split) bufsize /= 4; + dummy = new TDummy(); // By setting the value, we own the pointer and must delete it. + TTree::SetBranchStyle(branchStyle); + TBranch *branch = tree->Branch("dummy", &dummy, bufsize,split); + branch->SetAutoDelete(kFALSE); + if(split >= 0 && branchStyle) tree->BranchRef(); + + for (ev = 0; ev < nevent; ev++) { + if (ev%printev == 0) { + tnew = timer.RealTime(); + printf("dummy:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + + dummy->Build(); + + if (write) nb += tree->Fill(); //fill the tree + } + if (write) { + hfile = tree->GetCurrentFile(); //just in case we switched to a new file + hfile->Write(); + tree->Print(); + } + } + // We own the dummy (since we set the branch address explicitly), we need to delete it. + delete dummy; dummy = 0; + + // Stop timer and print results + timer.Stop(); + Float_t mbytes = 0.000001*nb; + Double_t rtime = timer.RealTime(); + Double_t ctime = timer.CpuTime(); + + + printf("\n%d dummies and %lld bytes processed.\n",nevent,nb); + printf("RealTime=%f seconds, CpuTime=%f seconds\n",rtime,ctime); + if (read) { + tree->PrintCacheStats(); + if (ioperf) {ioperf->Print();} + printf("You read %f Mbytes/Realtime seconds\n",mbytes/rtime); + printf("You read %f Mbytes/Cputime seconds\n",mbytes/ctime); + } else { + printf("compression level=%d, split=%d, arg4=%d, compression algorithm=%d\n",comp,split,arg4,compAlg); + printf("You write %f Mbytes/Realtime seconds\n",mbytes/rtime); + printf("You write %f Mbytes/Cputime seconds\n",mbytes/ctime); + //printf("file compression factor = %f\n",hfile.GetCompressionFactor()); + } + hfile->Close(); + return 0; +} diff --git a/test/MainEvent.cxx b/test/MainEvent.cxx index b102e1098e774..96c3c81238d43 100644 --- a/test/MainEvent.cxx +++ b/test/MainEvent.cxx @@ -88,6 +88,7 @@ #include "TNetFile.h" #include "TRandom.h" #include "TTree.h" +#include "TTreePerfStats.h" #include "TBranch.h" #include "TClonesArray.h" #include "TStopwatch.h" @@ -108,6 +109,7 @@ int main(int argc, char **argv) Int_t read = 0; Int_t arg4 = 1; Int_t arg5 = 600; //default number of tracks per event + Int_t compAlg = 1; Int_t netf = 0; Int_t punzip = 0; @@ -116,6 +118,7 @@ int main(int argc, char **argv) if (argc > 3) split = atoi(argv[3]); if (argc > 4) arg4 = atoi(argv[4]); if (argc > 5) arg5 = atoi(argv[5]); + if (argc > 6) compAlg= atoi(argv[6]); if (arg4 == 0) { write = 0; hfill = 0; read = 1;} if (arg4 == 1) { write = 1; hfill = 0;} if (arg4 == 2) { write = 0; hfill = 0;} @@ -133,6 +136,7 @@ int main(int argc, char **argv) TFile *hfile; TTree *tree; + TTreePerfStats *ioperf = NULL; Event *event = 0; // Fill event, header and tracks with some random numbers @@ -161,6 +165,7 @@ int main(int argc, char **argv) nevent = TMath::Min(nevent,nentries); if (read == 1) { //read sequential //by setting the read cache to -1 we set it to the AutoFlush value when writing + ioperf = new TTreePerfStats("Perf Stats", tree); Int_t cachesize = -1; if (punzip) tree->SetParallelUnzip(); tree->SetCacheSize(cachesize); @@ -176,6 +181,7 @@ int main(int argc, char **argv) } nb += tree->GetEntry(ev); //read complete event in memory } + ioperf->Finish(); } else { //read random Int_t evrandom; for (ev = 0; ev < nevent; ev++) { @@ -195,6 +201,7 @@ int main(int argc, char **argv) } else hfile = new TFile("Event.root","RECREATE","TTree benchmark ROOT file"); hfile->SetCompressionLevel(comp); + hfile->SetCompressionAlgorithm(compAlg); // Create histogram to show write_time in function of time Float_t curtime = -0.5; @@ -255,10 +262,11 @@ int main(int argc, char **argv) printf("RealTime=%f seconds, CpuTime=%f seconds\n",rtime,ctime); if (read) { tree->PrintCacheStats(); + if (ioperf) {ioperf->Print();} printf("You read %f Mbytes/Realtime seconds\n",mbytes/rtime); printf("You read %f Mbytes/Cputime seconds\n",mbytes/ctime); } else { - printf("compression level=%d, split=%d, arg4=%d\n",comp,split,arg4); + printf("compression level=%d, split=%d, arg4=%d, compression algorithm=%d\n",comp,split,arg4,compAlg); printf("You write %f Mbytes/Realtime seconds\n",mbytes/rtime); printf("You write %f Mbytes/Cputime seconds\n",mbytes/ctime); //printf("file compression factor = %f\n",hfile.GetCompressionFactor()); diff --git a/test/MainLocalcombine.cxx b/test/MainLocalcombine.cxx new file mode 100644 index 0000000000000..a98d0dc789644 --- /dev/null +++ b/test/MainLocalcombine.cxx @@ -0,0 +1,198 @@ +// @(#)root/test:$Id$ +// Author: Rene Brun 19/01/97 + +#include + +#include "Riostream.h" +#include "TROOT.h" +#include "TFile.h" +#include "TNetFile.h" +#include "TRandom.h" +#include "TTree.h" +#include "TBranch.h" +#include "TClonesArray.h" +#include "TStopwatch.h" + +#include "Localcompression.h" + +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char **argv) +{ + Int_t nevent = 1000; // by default create 400 events + Int_t comp = 1; // by default file is compressed + Int_t read = 0; + Int_t object = 0; + Int_t algorithm = 1; + + if (argc > 1) nevent = atoi(argv[1]); + if (argc > 2) comp = atoi(argv[2]); + if (argc > 3) read = atoi(argv[3]); + if (argc > 4) object = atoi(argv[4]); + if (argc > 5) algorithm = atoi(argv[5]); + + Int_t branchStyle = 1; //new style by default + + TFile *hfile; + TTree *tree; + TLarge *eventlarge = 0; + TSmall *eventsmall = 0; + TFloat *eventfloat = 0; + Int_t nsmall = 1000; + Int_t nfloat = nsmall*100; + + TStopwatch timer; + timer.Start(); + Long64_t nb = 0; + Long64_t nbl = 0; + Long64_t nbs = 0; + Long64_t nbi = 0; + + Int_t ev; + Int_t bufsize; + Double_t told = 0; + Double_t tnew = 0; + Int_t printev = 100; + + TBranch *branch; + TBranch *branchsmall; + TBranch *branchlarge; + TBranch *branchfloat; + +// Read case + if (read) { + hfile = new TFile("TCombine.root"); + tree = (TTree*)hfile->Get("T"); + if (object == 1) { + branch = tree->GetBranch("EventSmall"); + branch->SetAddress(&eventsmall); + } else if (object == 0) { + branch = tree->GetBranch("EventLarge"); + branch->SetAddress(&eventlarge); + } else if (object == 2) { + branch = tree->GetBranch("EventFloat"); + branch->SetAddress(&eventfloat); + } else { + branch = 0; + } + + Int_t nentries = (Int_t)branch->GetEntries(); + nevent = TMath::Min(nevent,nentries); + printf("nentries=%d,nevent=%d\n",nentries,nevent); + if (read == 1) { //read sequential + //by setting the read cache to -1 we set it to the AutoFlush value when writing + Int_t cachesize = -1; + tree->SetCacheSize(cachesize); + tree->SetCacheLearnEntries(1); //one entry is sufficient to learn + tree->SetCacheEntryRange(0,nevent); + for (ev = 0; ev < nevent; ev++) { + tree->LoadTree(ev); //this call is required when using the cache + if (ev%printev == 0) { + tnew = timer.RealTime(); + printf("event:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + nb += branch->GetEntry(ev); //read branch in memory + } + } else { //read random + Int_t distance = nentries/nevent; + Int_t cnt = 0; + for (ev = 0; ev < nentries; ev+=distance) { + tree->LoadTree(ev); + cnt++; + if (cnt%printev == 0) { + tnew = timer.RealTime(); + printf("event:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + nb += branch->GetEntry(ev); + } + } + } else { +// Write case + hfile = new TFile("TCombine.root","RECREATE","TTree benchmark ROOT file"); + + eventsmall = new TSmall(); + eventlarge = new TLarge(); + eventfloat = new TFloat(); + + hfile->SetCompressionLevel(comp); + hfile->SetCompressionAlgorithm(algorithm); + + // Create a ROOT Tree and one superbranch + tree = new TTree("T","An example of a ROOT tree"); + tree->SetAutoSave(1000000000); // autosave when 1 Gbyte written + tree->SetCacheSize(10000000); // set a 10 MBytes cache (useless when writing local files) + bufsize = 64000; + + TTree::SetBranchStyle(branchStyle); + branchsmall = tree->Branch("EventSmall", &eventsmall, bufsize,0); + branchlarge = tree->Branch("EventLarge", &eventlarge, bufsize,0); + branchfloat = tree->Branch("EventFloat", &eventfloat, bufsize, 0); + + branchsmall->SetAutoDelete(kFALSE); + branchlarge->SetAutoDelete(kFALSE); + branchfloat->SetAutoDelete(kFALSE); + + if(branchStyle) tree->BranchRef(); + + for (ev = 0; ev < nevent; ev++) { + if (ev%printev == 0) { + tnew = timer.RealTime(); + printf("event:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + eventlarge->Build(); + nbl += branchlarge->Fill(); +// printf("event%d, nb large = %lld\n", ev, nbl); + for (Int_t i = 0; i < nsmall; ++i) { + eventsmall->Build(); + nbs += branchsmall->Fill(); +// printf("event%d, iner%d, nb small = %lld\n", ev, i, nbs); + } + for (Int_t i = 0; i < nfloat; ++i) { + eventfloat->Build(); + nbi += branchfloat->Fill(); +// printf("event%d, iner%d, nb int = %lld\n", ev, i, nbi); + } +// nb += tree->Fill(); //fill the tree +// printf("nb total = %lld\n", nb); + } + nb = nbl + nbs + nbi; + hfile = tree->GetCurrentFile(); //just in case we switched to a new file + hfile->Write(); + } + + // We own the event (since we set the branch address explicitly), we need to delete it. + delete eventsmall; eventsmall = 0; + delete eventlarge; eventlarge = 0; + delete eventfloat; eventfloat = 0; + + // Stop timer and print results + timer.Stop(); + Float_t mbytes = 0.000001*nb; + Double_t rtime = timer.RealTime(); + Double_t ctime = timer.CpuTime(); + + printf("\n%d events and %lld bytes processed.\n",nevent,nb); + printf("RealTime=%f seconds, CpuTime=%f seconds\n",rtime,ctime); + + if (read) { + tree->PrintCacheStats(); + printf("You read %f Mbytes/Realtime seconds\n",mbytes/rtime); + printf("You read %f Mbytes/Cputime seconds\n",mbytes/ctime); + } else { + printf("compression level=%d\n", comp); + printf("You write %f Mbytes/Realtime seconds\n",mbytes/rtime); + printf("You write %f Mbytes/Cputime seconds\n",mbytes/ctime); + //printf("file compression factor = %f\n",hfile.GetCompressionFactor()); + } + tree->Print(); + hfile->Close(); + return 0; +} diff --git a/test/MainLocalcompression.cxx b/test/MainLocalcompression.cxx new file mode 100644 index 0000000000000..0e70b45f1a5fa --- /dev/null +++ b/test/MainLocalcompression.cxx @@ -0,0 +1,188 @@ +// @(#)root/test:$Id$ +// Author: Rene Brun 19/01/97 + +#include + +#include "Riostream.h" +#include "TROOT.h" +#include "TFile.h" +#include "TNetFile.h" +#include "TRandom.h" +#include "TTree.h" +#include "TBranch.h" +#include "TClonesArray.h" +#include "TStopwatch.h" + +#include "Localcompression.h" + +using namespace std; + +//////////////////////////////////////////////////////////////////////////////// + +int main(int argc, char **argv) +{ + Int_t nevent = 40; // by default create 400 events + Int_t comp = 1; // by default file is compressed + Int_t read = 0; + Int_t object = 0; + Int_t algorithm = 1; + + if (argc > 1) nevent = atoi(argv[1]); + if (argc > 2) comp = atoi(argv[2]); + if (argc > 3) read = atoi(argv[3]); + if (argc > 4) object = atoi(argv[4]); + if (argc > 5) algorithm = atoi(argv[5]); + + Int_t branchStyle = 1; //new style by default + + TFile *hfile = 0; + TTree *tree = 0; + TBranch *branch = 0; + TLarge *eventlarge = 0; + TSmall *eventsmall = 0; + TFloat *eventfloat = 0; + + // Fill event, header and tracks with some random numbers + // Create a timer object to benchmark this loop + TStopwatch timer; + timer.Start(); + Long64_t nb = 0; + Int_t ev; + Int_t bufsize; + Double_t told = 0; + Double_t tnew = 0; + Int_t printev = 100; + +// Read case + if (read) { + if (object == 1) { + hfile = new TFile("TSmall.root"); + tree = (TTree*)hfile->Get("T"); + branch = tree->GetBranch("event"); + branch->SetAddress(&eventsmall); + Int_t nentries = (Int_t)tree->GetEntries(); + nevent = TMath::Min(nevent,nentries); + } else if (object == 0) { + hfile = new TFile("TLarge.root"); + tree = (TTree*)hfile->Get("T"); + branch = tree->GetBranch("event"); + branch->SetAddress(&eventlarge); + Int_t nentries = (Int_t)tree->GetEntries(); + nevent = TMath::Min(nevent,nentries); + } else if (object == 2) { + hfile = new TFile("TFloat.root"); + tree = (TTree*)hfile->Get("T"); + branch = tree->GetBranch("event"); + branch->SetAddress(&eventfloat); + Int_t nentries = (Int_t)tree->GetEntries(); + nevent = TMath::Min(nevent,nentries); + } + + if (read == 1) { //read sequential + //by setting the read cache to -1 we set it to the AutoFlush value when writing + Int_t cachesize = -1; + tree->SetCacheSize(cachesize); + tree->SetCacheLearnEntries(1); //one entry is sufficient to learn + tree->SetCacheEntryRange(0,nevent); + for (ev = 0; ev < nevent; ev++) { + tree->LoadTree(ev); //this call is required when using the cache + if (ev%printev == 0) { + tnew = timer.RealTime(); + printf("event:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + nb += tree->GetEntry(ev); //read complete event in memory + } + } else { //read random + Int_t evrandom = Int_t(nevent*gRandom->Rndm(1)); + nb += tree->GetEntry(evrandom); + } + } else { +// Write case + // Create a new ROOT binary machine independent file. + // Note that this file may contain any kind of ROOT objects, histograms, + // pictures, graphics objects, detector geometries, tracks, events, etc.. + // This file is now becoming the current directory. + if (object == 1) { + hfile = new TFile("TSmall.root","RECREATE","TTree benchmark ROOT file"); + eventsmall = new TSmall(); + } else if (object == 0) { + hfile = new TFile("TLarge.root","RECREATE","TTree benchmark ROOT file"); + eventlarge = new TLarge(); + } else if (object == 2) { + hfile = new TFile("TFloat.root","RECREATE","TTree benchmark ROOT file"); + eventfloat = new TFloat(); + } + + hfile->SetCompressionLevel(comp); + hfile->SetCompressionAlgorithm(algorithm); + + // Create a ROOT Tree and one superbranch + tree = new TTree("T","An example of a ROOT tree"); + tree->SetAutoSave(1000000000); // autosave when 1 Gbyte written + tree->SetCacheSize(10000000); // set a 10 MBytes cache (useless when writing local files) + bufsize = 64000; + + TTree::SetBranchStyle(branchStyle); + if (object == 1) { + branch = tree->Branch("event", &eventsmall, bufsize,0); + } else if (object == 0) { + branch = tree->Branch("event", &eventlarge, bufsize,0); + } else if (object == 2) { + branch = tree->Branch("event", &eventfloat, bufsize, 0); + } + + branch->SetAutoDelete(kFALSE); + if(branchStyle) tree->BranchRef(); + + for (ev = 0; ev < nevent; ev++) { + if (ev%printev == 0) { + tnew = timer.RealTime(); + printf("event:%d, rtime=%f s\n",ev,tnew-told); + told=tnew; + timer.Continue(); + } + if (object == 1) { + eventsmall->Build(); + } else if (object == 0) { + eventlarge->Build(); + } else if (object == 2) { + eventfloat->Build(); + } + nb += tree->Fill(); //fill the tree + } + hfile = tree->GetCurrentFile(); //just in case we switched to a new file + hfile->Write(); + } + // We own the event (since we set the branch address explicitly), we need to delete it. + if (object == 1) { + delete eventsmall; eventsmall = 0; + } else if (object == 0) { + delete eventlarge; eventlarge = 0; + } else if (object == 2) { + delete eventfloat; eventfloat = 0; + } + + // Stop timer and print results + timer.Stop(); + Float_t mbytes = 0.000001*nb; + Double_t rtime = timer.RealTime(); + Double_t ctime = timer.CpuTime(); + + + printf("\n%d events and %lld bytes processed.\n",nevent,nb); + printf("RealTime=%f seconds, CpuTime=%f seconds\n",rtime,ctime); + if (read) { + tree->PrintCacheStats(); + printf("You read %f Mbytes/Realtime seconds\n",mbytes/rtime); + printf("You read %f Mbytes/Cputime seconds\n",mbytes/ctime); + } else { + printf("compression level=%d\n", comp); + printf("You write %f Mbytes/Realtime seconds\n",mbytes/rtime); + printf("You write %f Mbytes/Cputime seconds\n",mbytes/ctime); + //printf("file compression factor = %f\n",hfile.GetCompressionFactor()); + } + hfile->Close(); + return 0; +} diff --git a/test/Makefile b/test/Makefile index 101fe22f239b0..03933b5e6fe17 100644 --- a/test/Makefile +++ b/test/Makefile @@ -38,6 +38,7 @@ endif EVENTO = Event.$(ObjSuf) EventDict.$(ObjSuf) EVENTS = Event.$(SrcSuf) EventDict.$(SrcSuf) EVENTSO = libEvent.$(DllSuf) +EVENTLIBS = -lTreePlayer EVENT = Event$(ExeSuf) ifeq ($(PLATFORM),win32) EVENTLIB = libEvent.lib @@ -393,7 +394,7 @@ endif @echo "$@ done" $(EVENT): $(EVENTSO) $(MAINEVENTO) - $(LD) $(LDFLAGS) $(MAINEVENTO) $(EVENTO) $(LIBS) $(OutPutOpt)$@ + $(LD) $(LDFLAGS) $(MAINEVENTO) $(EVENTO) $(LIBS) $(EVENTLIBS) $(OutPutOpt)$@ $(MT_EXE) @echo "$@ done" diff --git a/tree/tree/src/TBasket.cxx b/tree/tree/src/TBasket.cxx index 00450a1f2c8cc..10d50e13f13bd 100644 --- a/tree/tree/src/TBasket.cxx +++ b/tree/tree/src/TBasket.cxx @@ -23,6 +23,9 @@ #include "TTimeStamp.h" #include "RZip.h" +#include +#include + // TODO: Copied from TBranch.cxx #if (__GNUC__ >= 3) || defined(__INTEL_COMPILER) #if !defined(R__unlikely) @@ -47,6 +50,9 @@ Manages buffers for branches of a Tree. See picture in TTree. */ +static std::chrono::steady_clock::duration globalDuration = std::chrono::steady_clock::duration::zero();//## +static std::chrono::steady_clock::time_point startDuration;//## +static std::chrono::steady_clock::time_point endDuration;//## //////////////////////////////////////////////////////////////////////////////// /// Default contructor. @@ -563,6 +569,8 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) fBuffer = rawUncompressedBuffer; oldCase = OLD_CASE_EXPRESSION; + +// startDuration = std::chrono::steady_clock::now();//## // Case where ROOT thinks the buffer is compressed. Copy over the key and uncompress the object if (fObjlen > fNbytes-fKeylen || oldCase) { if (R__unlikely(TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1))) { @@ -591,10 +599,17 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) if (R__unlikely(oldCase && (nin > fObjlen || nbuf > fObjlen))) { //buffer was very likely not compressed in an old version memcpy(rawUncompressedBuffer+fKeylen, rawCompressedObjectBuffer+fKeylen, fObjlen); +// endDuration = std::chrono::steady_clock::now();//## +// globalDuration += endDuration - startDuration;//## goto AfterBuffer; } + startDuration = std::chrono::steady_clock::now();//## R__unzip(&nin, rawCompressedObjectBuffer, &nbuf, (unsigned char*) rawUncompressedObjectBuffer, &nout); + endDuration = std::chrono::steady_clock::now();//## + globalDuration += endDuration - startDuration;//## + std::cout << globalDuration.count() << std::endl;//## +// printf("fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d", fNbytes,fKeylen,fObjlen, noutot,nout,nin,nbuf);//## if (!nout) break; noutot += nout; nintot += nin; @@ -620,6 +635,8 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) // Nothing is compressed - copy over wholesale. memcpy(rawUncompressedBuffer, rawCompressedBuffer, len); } +// endDuration = std::chrono::steady_clock::now();//## +// globalDuration += endDuration - startDuration;//## AfterBuffer: @@ -1009,8 +1026,29 @@ Int_t TBasket::WriteBuffer() if (i == nbuffers - 1) bufmax = fObjlen - nzip; else bufmax = kMAXZIPBUF; //compress the buffer +/* + TString tmpname(fBranch->GetName());//## + if(tmpname == "fDummy"){ + for(int index = 0; index < bufmax; ++index) { + printf("0x%x ", objbuf[index]);//## + } + printf("\n");//## + } +*/ + startDuration = std::chrono::steady_clock::now();//## R__zipMultipleAlgorithm(cxlevel, &bufmax, objbuf, &bufmax, bufcur, &nout, cxAlgorithm); - + endDuration = std::chrono::steady_clock::now();//## + globalDuration += endDuration - startDuration;//## + std::cout << globalDuration.count() << std::endl;//## +/* + if(tmpname == "fDummy"){ + printf("fKeylen = %d, fObjlen = %d, cxlevel = %d, bufmax = %d, nout = %d, cxAlgorithm = %d\n", fKeylen, fObjlen, cxlevel, bufmax, nout, cxAlgorithm);//## + for(int index = 0; index < nout; ++index) { + printf("0x%x ", bufcur[index]);//## + } + printf("\n");//## + } +*/ // test if buffer has really been compressed. In case of small buffers // when the buffer contains random data, it may happen that the compressed // buffer is larger than the input. In this case, we write the original uncompressed buffer