Change cmake file of lz4. Now lz4 has faster read speed.

cmake/modules/SearchInstalledSoftware.cmake

@@ -201,11 +201,11 @@ if(builtin_lz4)
   ExternalProject_Add(
     LZ4
     URL ${CMAKE_SOURCE_DIR}/core/lz4/src/lz4-${lz4_version}.tar.gz
-    URL_MD5 5ec580285a5c9bf1e5db3df0fce980b1
+    URL_MD5 0601f6b8477209d07db33d504feb6ac4
     INSTALL_DIR ${CMAKE_BINARY_DIR}
-    CONFIGURE_COMMAND cd <SOURCE_DIR>/cmake_unofficial && cmake . -DCMAKE_INSTALL_PREFIX=<INSTALL_DIR> -DBUILD_LIBS=1 -DCMAKE_C_FLAGS:STRING=-fPIC -O3
-    BUILD_COMMAND cd cmake_unofficial && make
-    INSTALL_COMMAND cd cmake_unofficial && make install
+    CONFIGURE_COMMAND  PREFIX=<INSTALL_DIR> make cmake
+    BUILD_COMMAND PREFIX=<INSTALL_DIR> MOREFLAGS=-fPIC make
+    INSTALL_COMMAND PREFIX=<INSTALL_DIR> make install
     BUILD_IN_SOURCE 1)
   set(LZ4_LIBRARIES ${CMAKE_BINARY_DIR}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lz4${CMAKE_STATIC_LIBRARY_SUFFIX})
   set(LZ4_INCLUDE_DIR ${CMAKE_BINARY_DIR}/include)

core/lz4/src/ZipLZ4.c

@@ -45,11 +45,13 @@ void R__zipLZ4(int cxlevel, int *srcsize, char *src, int *tgtsize, char *tgt, in
    }
 
    int returnStatus;
-   unsigned char cxByte = cxlevel >= 4;
-   if (cxByte) {
-      returnStatus = LZ4_compressHC_limitedOutput(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
+   if (cxlevel > 9) {
+      cxlevel = 9;
+   }
+   if (cxlevel >= 4) {
+      returnStatus = LZ4_compress_HC(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize, cxlevel);
    } else {
-      returnStatus = LZ4_compress_limitedOutput(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
+      returnStatus = LZ4_compress_default(src, &tgt[kHeaderSize], *srcsize, *tgtsize - kHeaderSize);
    }
 
    if (R__unlikely(returnStatus == 0)) { /* LZ4 compression failed */

io/io/src/TFile.cxx

@@ -2122,7 +2122,6 @@ void TFile::SetCompressionAlgorithm(Int_t algorithm)
       int level = fCompress % 100;
       fCompress = 100 * algorithm + level;
    }
-   printf("algorithm = %d\n", algorithm);//##
 }
 
 ////////////////////////////////////////////////////////////////////////////////

test/CMakeLists.txt

@@ -45,12 +45,12 @@ ROOT_ADD_TEST(test-event COMMAND eventexe)
 ROOT_GENERATE_DICTIONARY(LocalcompressionDict ${CMAKE_CURRENT_SOURCE_DIR}/Localcompression.h MODULE Localcompression LINKDEF LocalcompressionLinkDef.h)
 ROOT_LINKER_LIBRARY(Localcompression Localcompression.cxx LocalcompressionDict.cxx LIBRARIES Hist MathCore)
 
-ROOT_EXECUTABLE(dummyexe MainDummy.cxx LIBRARIES Localcompression RIO Tree TreePlayer Hist MathCore Net)
+ROOT_EXECUTABLE(dummyexe MainDummy.cxx LIBRARIES Localcompression RIO Tree TreePlayer Hist Net)
 ROOT_ADD_TEST(test-dummy COMMAND dummyexe)
 
-ROOT_EXECUTABLE(localcompressionexe MainLocalcompression.cxx LIBRARIES Localcompression RIO Tree Hist MathCore Net)
+ROOT_EXECUTABLE(localcompressionexe MainLocalcompression.cxx LIBRARIES Localcompression RIO Tree Hist Net)
 ROOT_ADD_TEST(test-localcompression COMMAND localcompressionexe)
-ROOT_EXECUTABLE(localcombineexe MainLocalcombine.cxx LIBRARIES Localcompression RIO Tree Hist MathCore Net)
+ROOT_EXECUTABLE(localcombineexe MainLocalcombine.cxx LIBRARIES Localcompression RIO Tree Hist Net)
 ROOT_ADD_TEST(test-localcombine COMMAND localcombineexe)
 
 #---hsimple------------------------------------------------------------------------------------

test/Localcompression.cxx

@@ -24,6 +24,16 @@ TDummy::TDummy(Int_t size)
       Dummy_t dummy;
       for(int offset = 0; offset < 4; ++offset) {
          Float_t tmp = Float_t(gRandom->Rndm(1));
+         if(tmp < 0.4) {
+            dummy.ch[offset] = 'a';
+         } else if(tmp < 0.65) {
+            dummy.ch[offset] = 'b';
+         } else if(tmp < 0.9) {
+            dummy.ch[offset] = 'c';
+         } else {
+            dummy.ch[offset] = 'd';
+         }
+/*
          if(tmp < 0.08167) {
             dummy.ch[offset] = 'a';
          } else if(tmp < 0.09659) {
@@ -77,6 +87,7 @@ TDummy::TDummy(Int_t size)
          } else {
             dummy.ch[offset] = 'z';
          }
+*/
 //         std::cout << "dummy.ch[" << offset << "]=" << dummy.ch[offset] << ", ";//##
       }
       fDummy[i] = dummy.fp;
@@ -120,6 +131,17 @@ void TDummy::Build()
    for(int i=0;i<fSize;++i) {
       Dummy_t dummy;
       for(int offset = 0; offset < 4; ++offset) {
+         Float_t tmp = Float_t(gRandom->Rndm(1));
+         if(tmp < 0.4) {
+            dummy.ch[offset] = 'a';
+         } else if(tmp < 0.65) {
+            dummy.ch[offset] = 'b';
+         } else if(tmp < 0.9) {
+            dummy.ch[offset] = 'c';
+         } else {
+            dummy.ch[offset] = 'd';
+         }
+/*
          Float_t tmp = Float_t(gRandom->Rndm(1));
          if(tmp < 0.08167) {
             dummy.ch[offset] = 'a';
@@ -174,6 +196,7 @@ void TDummy::Build()
          } else {
             dummy.ch[offset] = 'z';
          }
+*/
 //         std::cout << "dummy.ch[" << offset << "]=" << dummy.ch[offset] << ", ";//##
       }
       fDummy[i] = dummy.fp;

test/Localcompression.h

@@ -17,7 +17,7 @@
 #include "TBits.h"
 #include "TMath.h"
 
-#define DUMMYSIZE 10
+#define DUMMYSIZE 1000000
 #define LARGESIZE 1000000
 #define SMALLSIZE 1000
 #define FLOATSIZE 6

test/MainDummy.cxx

@@ -0,0 +1,178 @@
+#include <stdlib.h>
+
+#include "Riostream.h"
+#include "TROOT.h"
+#include "TFile.h"
+#include "TNetFile.h"
+#include "TRandom.h"
+#include "TTree.h"
+#include "TTreePerfStats.h"
+#include "TBranch.h"
+#include "TClonesArray.h"
+#include "TStopwatch.h"
+
+#include "Localcompression.h"
+
+using namespace std;
+
+////////////////////////////////////////////////////////////////////////////////
+
+int main(int argc, char **argv)
+{
+   Int_t nevent = 400;     // by default create 400 events
+   Int_t comp   = 1;       // by default file is compressed
+   Int_t split  = 1;       // by default, split Event in sub branches
+   Int_t write  = 1;       // by default the tree is filled
+   Int_t read   = 0;
+   Int_t arg4   = 1;
+   Int_t arg5   = 600;     //default number of tracks per event
+   Int_t compAlg = 1;
+   Int_t netf   = 0;
+   Int_t punzip = 0;
+
+   if (argc > 1)  nevent = atoi(argv[1]);
+   if (argc > 2)  comp   = atoi(argv[2]);
+   if (argc > 3)  split  = atoi(argv[3]);
+   if (argc > 4)  arg4   = atoi(argv[4]);
+   if (argc > 5)  arg5   = atoi(argv[5]);
+   if (argc > 6)  compAlg= atoi(argv[6]);
+   if (arg4 ==  0) { write = 0; read = 1;}
+   if (arg4 ==  1) { write = 1;}
+   if (arg4 ==  2) { write = 0;}
+   if (arg4 == 10) { write = 0;}
+   if (arg4 == 11) { write = 1;}
+   if (arg4 == 20) { write = 0; read  = 1;}  //read sequential
+   if (arg4 == 21) { write = 0; read  = 1;  punzip = 1;}  //read sequential + parallel unzipping
+   if (arg4 == 25) { write = 0; read  = 2;}  //read random
+   if (arg4 >= 30) { netf  = 1; }            //use TNetFile
+   if (arg4 == 30) { write = 0; read  = 1;}  //netfile + read sequential
+   if (arg4 == 35) { write = 0; read  = 2;}  //netfile + read random
+   if (arg4 == 36) { write = 1; }            //netfile + write sequential
+   Int_t branchStyle = 1; //new style by default
+   if (split < 0) {branchStyle = 0; split = -1-split;}
+
+   TFile *hfile;
+   TTree *tree;
+   TTreePerfStats *ioperf = NULL;
+   TDummy *dummy = 0;
+
+   // Fill dummy, header and tracks with some random numbers
+   //   Create a timer object to benchmark this loop
+   TStopwatch timer;
+   timer.Start();
+   Long64_t nb = 0;
+   Int_t ev;
+   Int_t bufsize;
+   Double_t told = 0;
+   Double_t tnew = 0;
+   Int_t printev = 100;
+   if (arg5 < 100) printev = 1000;
+   if (arg5 < 10)  printev = 10000;
+
+//         Read case
+   if (read) {
+      if (netf) {
+         hfile = new TNetFile("root://localhost/root/test/DummyNet.root");
+      } else
+         hfile = new TFile("Dummy.root");
+      tree = (TTree*)hfile->Get("T");
+      TBranch *branch = tree->GetBranch("dummy");
+      branch->SetAddress(&dummy);
+      Int_t nentries = (Int_t)tree->GetEntries();
+      nevent = TMath::Min(nevent,nentries);
+      if (read == 1) {  //read sequential
+         //by setting the read cache to -1 we set it to the AutoFlush value when writing
+         ioperf = new TTreePerfStats("Perf Stats", tree);
+         Int_t cachesize = -1;
+         if (punzip) tree->SetParallelUnzip();
+         tree->SetCacheSize(cachesize);
+         tree->SetCacheLearnEntries(1); //one entry is sufficient to learn
+         tree->SetCacheEntryRange(0,nevent);
+         for (ev = 0; ev < nevent; ev++) {
+            tree->LoadTree(ev);  //this call is required when using the cache
+            if (ev%printev == 0) {
+               tnew = timer.RealTime();
+               printf("dummy:%d, rtime=%f s\n",ev,tnew-told);
+               told=tnew;
+               timer.Continue();
+            }
+            nb += tree->GetEntry(ev);        //read complete event in memory
+         }
+         ioperf->Finish();
+      } else {    //read random
+         Int_t evrandom;
+         for (ev = 0; ev < nevent; ev++) {
+            if (ev%printev == 0) std::cout<<"dummy="<<ev<<std::endl;
+            evrandom = Int_t(nevent*gRandom->Rndm(1));
+            nb += tree->GetEntry(evrandom);  //read complete event in memory
+         }
+      }
+   } else {
+//         Write case
+      // Create a new ROOT binary machine independent file.
+      // Note that this file may contain any kind of ROOT objects, histograms,
+      // pictures, graphics objects, detector geometries, tracks, events, etc..
+      // This file is now becoming the current directory.
+      if (netf) {
+         hfile = new TNetFile("root://localhost/root/test/DummyNet.root","RECREATE","TTree benchmark ROOT file");
+      } else
+         hfile = new TFile("Dummy.root","RECREATE","TTree benchmark ROOT file");
+      hfile->SetCompressionLevel(comp);
+      hfile->SetCompressionAlgorithm(compAlg);
+
+      // Create a ROOT Tree and one superbranch
+      tree = new TTree("T","An example of a ROOT tree");
+      tree->SetAutoSave(1000000000); // autosave when 1 Gbyte written
+      tree->SetCacheSize(10000000);  // set a 10 MBytes cache (useless when writing local files)
+      bufsize = 64000;
+      if (split)  bufsize /= 4;
+      dummy = new TDummy();           // By setting the value, we own the pointer and must delete it.
+      TTree::SetBranchStyle(branchStyle);
+      TBranch *branch = tree->Branch("dummy", &dummy, bufsize,split);
+      branch->SetAutoDelete(kFALSE);
+      if(split >= 0 && branchStyle) tree->BranchRef();
+
+      for (ev = 0; ev < nevent; ev++) {
+         if (ev%printev == 0) {
+            tnew = timer.RealTime();
+            printf("dummy:%d, rtime=%f s\n",ev,tnew-told);
+            told=tnew;
+            timer.Continue();
+         }
+
+         dummy->Build();
+
+         if (write) nb += tree->Fill();  //fill the tree
+      }
+      if (write) {
+         hfile = tree->GetCurrentFile(); //just in case we switched to a new file
+         hfile->Write();
+         tree->Print();
+      }
+   }
+   // We own the dummy (since we set the branch address explicitly), we need to delete it.
+   delete dummy;  dummy = 0;
+
+   //  Stop timer and print results
+   timer.Stop();
+   Float_t mbytes = 0.000001*nb;
+   Double_t rtime = timer.RealTime();
+   Double_t ctime = timer.CpuTime();
+
+
+   printf("\n%d dummies and %lld bytes processed.\n",nevent,nb);
+   printf("RealTime=%f seconds, CpuTime=%f seconds\n",rtime,ctime);
+   if (read) {
+      tree->PrintCacheStats();
+      if (ioperf) {ioperf->Print();}
+      printf("You read %f Mbytes/Realtime seconds\n",mbytes/rtime);
+      printf("You read %f Mbytes/Cputime seconds\n",mbytes/ctime);
+   } else {
+      printf("compression level=%d, split=%d, arg4=%d, compression algorithm=%d\n",comp,split,arg4,compAlg);
+      printf("You write %f Mbytes/Realtime seconds\n",mbytes/rtime);
+      printf("You write %f Mbytes/Cputime seconds\n",mbytes/ctime);
+      //printf("file compression factor = %f\n",hfile.GetCompressionFactor());
+   }
+   hfile->Close();
+   return 0;
+}

tree/tree/src/TBasket.cxx

@@ -570,7 +570,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
 
    oldCase = OLD_CASE_EXPRESSION;
 
-   startDuration = std::chrono::steady_clock::now();//##
+//   startDuration = std::chrono::steady_clock::now();//##
    // Case where ROOT thinks the buffer is compressed.  Copy over the key and uncompress the object
    if (fObjlen > fNbytes-fKeylen || oldCase) {
       if (R__unlikely(TestBit(TBufferFile::kNotDecompressed) && (fNevBuf==1))) {
@@ -599,12 +599,17 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
          if (R__unlikely(oldCase && (nin > fObjlen || nbuf > fObjlen))) {
             //buffer was very likely not compressed in an old version
             memcpy(rawUncompressedBuffer+fKeylen, rawCompressedObjectBuffer+fKeylen, fObjlen);
-            endDuration = std::chrono::steady_clock::now();//##
-            globalDuration += endDuration - startDuration;//##
+//            endDuration = std::chrono::steady_clock::now();//##
+//            globalDuration += endDuration - startDuration;//##
             goto AfterBuffer;
          }
 
+         startDuration = std::chrono::steady_clock::now();//##
          R__unzip(&nin, rawCompressedObjectBuffer, &nbuf, (unsigned char*) rawUncompressedObjectBuffer, &nout);
+         endDuration = std::chrono::steady_clock::now();//##
+         globalDuration += endDuration - startDuration;//##
+         std::cout << globalDuration.count() << std::endl;//##
+//         printf("fNbytes = %d, fKeylen = %d, fObjlen = %d, noutot = %d, nout=%d, nin=%d, nbuf=%d", fNbytes,fKeylen,fObjlen, noutot,nout,nin,nbuf);//##
          if (!nout) break;
          noutot += nout;
          nintot += nin;
@@ -630,12 +635,11 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file)
       // Nothing is compressed - copy over wholesale.
       memcpy(rawUncompressedBuffer, rawCompressedBuffer, len);
    }
-   endDuration = std::chrono::steady_clock::now();//##
-   globalDuration += endDuration - startDuration;//##
+//   endDuration = std::chrono::steady_clock::now();//##
+//   globalDuration += endDuration - startDuration;//##
 
 AfterBuffer:
 
-   std::cout << globalDuration.count() << std::endl;//##
    fBranch->GetTree()->IncrementTotalBuffers(fBufferSize);
 
    // Read offsets table if needed.
@@ -1022,10 +1026,29 @@ Int_t TBasket::WriteBuffer()
          if (i == nbuffers - 1) bufmax = fObjlen - nzip;
          else bufmax = kMAXZIPBUF;
          //compress the buffer
+/*
+         TString tmpname(fBranch->GetName());//##
+         if(tmpname == "fDummy"){
+            for(int index = 0; index < bufmax; ++index) {
+               printf("0x%x ", objbuf[index]);//##
+            }
+            printf("\n");//##
+         }
+*/
+         startDuration = std::chrono::steady_clock::now();//##
          R__zipMultipleAlgorithm(cxlevel, &bufmax, objbuf, &bufmax, bufcur, &nout, cxAlgorithm);
-//         std::cout << lbuf << ", " << fObjlen << ", " << fKeylen << ", " << nbuffers << ", " << bufmax << ", " << nout << std::endl;//##
-//         std::cout << fObjlen << std::endl;//##
-
+         endDuration = std::chrono::steady_clock::now();//##
+         globalDuration += endDuration - startDuration;//##
+         std::cout << globalDuration.count() << std::endl;//##
+/*
+         if(tmpname == "fDummy"){
+            printf("fKeylen = %d, fObjlen = %d, cxlevel = %d, bufmax = %d, nout = %d, cxAlgorithm = %d\n", fKeylen, fObjlen, cxlevel, bufmax, nout, cxAlgorithm);//##
+            for(int index = 0; index < nout; ++index) {
+               printf("0x%x ", bufcur[index]);//##
+            }
+            printf("\n");//##
+         }
+*/
          // test if buffer has really been compressed. In case of small buffers
          // when the buffer contains random data, it may happen that the compressed
          // buffer is larger than the input. In this case, we write the original uncompressed buffer