Skip to content

Add support for LZ4 as a compression format #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zzxuanyuan wants to merge 7 commits into from
Closed

Add support for LZ4 as a compression format #81

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
be733dd
Add support for LZ4 as a compression format.
bbockelm Mar 11, 2015
4366feb
Resolve two conflicts
zzxuanyuan Jan 12, 2016
2b00926
Add compiling for MainLocalCombine.cxx
zzxuanyuan Jan 30, 2017
fbe481a
Add test files
zzxuanyuan Jan 30, 2017
2144790
Add chrono to test decompression time
zzxuanyuan Feb 1, 2017
fb09b25
Add Dummy object to test and add debug code in TFile.cxx and TBasket.cxx
zzxuanyuan Feb 7, 2017
12c579e
Change cmake file of lz4. Now lz4 has faster read speed.
zzxuanyuan Feb 19, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add Dummy object to test and add debug code in TFile.cxx and TBasket.cxx
  • Loading branch information
@zzxuanyuan
zzxuanyuan committed Feb 7, 2017
commit fb09b25f24c5d0abb53cd25dcd37f4c6bb9494c0
1 change: 1 addition & 0 deletions io/io/src/TFile.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2122,6 +2122,7 @@ void TFile::SetCompressionAlgorithm(Int_t algorithm)
int level = fCompress % 100;
fCompress = 100 * algorithm + level;
}
printf("algorithm = %d\n", algorithm);//##
}

////////////////////////////////////////////////////////////////////////////////
Expand Down
7 changes: 5 additions & 2 deletions test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,12 @@ ROOT_ADD_TEST(test-event COMMAND eventexe)
ROOT_GENERATE_DICTIONARY(LocalcompressionDict ${CMAKE_CURRENT_SOURCE_DIR}/Localcompression.h MODULE Localcompression LINKDEF LocalcompressionLinkDef.h)
ROOT_LINKER_LIBRARY(Localcompression Localcompression.cxx LocalcompressionDict.cxx LIBRARIES Hist MathCore)

ROOT_EXECUTABLE(localcompressionexe MainLocalcompression.cxx LIBRARIES Localcompression RIO Tree Hist Net)
ROOT_EXECUTABLE(dummyexe MainDummy.cxx LIBRARIES Localcompression RIO Tree TreePlayer Hist MathCore Net)
ROOT_ADD_TEST(test-dummy COMMAND dummyexe)

ROOT_EXECUTABLE(localcompressionexe MainLocalcompression.cxx LIBRARIES Localcompression RIO Tree Hist MathCore Net)
ROOT_ADD_TEST(test-localcompression COMMAND localcompressionexe)
ROOT_EXECUTABLE(localcombineexe MainLocalcombine.cxx LIBRARIES Localcompression RIO Tree Hist Net)
ROOT_EXECUTABLE(localcombineexe MainLocalcombine.cxx LIBRARIES Localcompression RIO Tree Hist MathCore Net)
ROOT_ADD_TEST(test-localcombine COMMAND localcombineexe)

#---hsimple------------------------------------------------------------------------------------
Expand Down
170 changes: 169 additions & 1 deletion test/Localcompression.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,180 @@
#include "TProcessID.h"

#include "Localcompression.h"
#include <iostream>


ClassImp(TDummy)
ClassImp(TLarge)
ClassImp(TSmall)
ClassImp(TFloat)

////////////////////////////////////////////////////////////////////////////////
/// Create an TDummy.
TDummy::TDummy(Int_t size)
{
fSize = size;
fDummy = new Float_t[fSize];
for(int i=0;i<fSize;++i) {
Dummy_t dummy;
for(int offset = 0; offset < 4; ++offset) {
Float_t tmp = Float_t(gRandom->Rndm(1));
if(tmp < 0.08167) {
dummy.ch[offset] = 'a';
} else if(tmp < 0.09659) {
dummy.ch[offset] = 'b';
} else if(tmp < 0.12441) {
dummy.ch[offset] = 'c';
} else if(tmp < 0.16694) {
dummy.ch[offset] = 'd';
} else if(tmp < 0.29396) {
dummy.ch[offset] = 'e';
} else if(tmp < 0.31624) {
dummy.ch[offset] = 'f';
} else if(tmp < 0.33639) {
dummy.ch[offset] = 'g';
} else if(tmp < 0.39733) {
dummy.ch[offset] = 'h';
} else if(tmp < 0.46699) {
dummy.ch[offset] = 'i';
} else if(tmp < 0.46852) {
dummy.ch[offset] = 'j';
} else if(tmp < 0.47624) {
dummy.ch[offset] = 'k';
} else if(tmp < 0.51649) {
dummy.ch[offset] = 'l';
} else if(tmp < 0.54055) {
dummy.ch[offset] = 'm';
} else if(tmp < 0.60804) {
dummy.ch[offset] = 'n';
} else if(tmp < 0.68311) {
dummy.ch[offset] = 'o';
} else if(tmp < 0.7024) {
dummy.ch[offset] = 'p';
} else if(tmp < 0.70335) {
dummy.ch[offset] = 'q';
} else if(tmp < 0.76322) {
dummy.ch[offset] = 'r';
} else if(tmp < 0.82649) {
dummy.ch[offset] = 's';
} else if(tmp < 0.91705) {
dummy.ch[offset] = 't';
} else if(tmp < 0.94463) {
dummy.ch[offset] = 'u';
} else if(tmp < 0.95441) {
dummy.ch[offset] = 'v';
} else if(tmp < 0.97801) {
dummy.ch[offset] = 'w';
} else if(tmp < 0.97951) {
dummy.ch[offset] = 'x';
} else if(tmp < 0.99925) {
dummy.ch[offset] = 'y';
} else {
dummy.ch[offset] = 'z';
}
// std::cout << "dummy.ch[" << offset << "]=" << dummy.ch[offset] << ", ";//##
}
fDummy[i] = dummy.fp;
// std::cout << "fDummy[" << i << "]=" << fDummy[i] << std::endl;//##
}
}

////////////////////////////////////////////////////////////////////////////////
/// Create an TDummy.
TDummy::TDummy(const TDummy& dummy) : TObject(dummy)
{
Float_t *intermediate = dummy.GetDummy();
Int_t size = dummy.GetSize();
fDummy = new Float_t[size];
for(int i=0;i<size;++i)
fDummy[i] = intermediate[i];
}

////////////////////////////////////////////////////////////////////////////////

TDummy::~TDummy()
{
Clear();
delete fDummy;
fSize = 0;
}

////////////////////////////////////////////////////////////////////////////////

void TDummy::Clear(Option_t * /*option*/)
{
TObject::Clear();
for(int i=0;i<fSize;++i)
fDummy[i] = 0;
}

/////////////////////////////////////////////////////////////////////////////////

void TDummy::Build()
{
for(int i=0;i<fSize;++i) {
Dummy_t dummy;
for(int offset = 0; offset < 4; ++offset) {
Float_t tmp = Float_t(gRandom->Rndm(1));
if(tmp < 0.08167) {
dummy.ch[offset] = 'a';
} else if(tmp < 0.09659) {
dummy.ch[offset] = 'b';
} else if(tmp < 0.12441) {
dummy.ch[offset] = 'c';
} else if(tmp < 0.16694) {
dummy.ch[offset] = 'd';
} else if(tmp < 0.29396) {
dummy.ch[offset] = 'e';
} else if(tmp < 0.31624) {
dummy.ch[offset] = 'f';
} else if(tmp < 0.33639) {
dummy.ch[offset] = 'g';
} else if(tmp < 0.39733) {
dummy.ch[offset] = 'h';
} else if(tmp < 0.46699) {
dummy.ch[offset] = 'i';
} else if(tmp < 0.46852) {
dummy.ch[offset] = 'j';
} else if(tmp < 0.47624) {
dummy.ch[offset] = 'k';
} else if(tmp < 0.51649) {
dummy.ch[offset] = 'l';
} else if(tmp < 0.54055) {
dummy.ch[offset] = 'm';
} else if(tmp < 0.60804) {
dummy.ch[offset] = 'n';
} else if(tmp < 0.68311) {
dummy.ch[offset] = 'o';
} else if(tmp < 0.7024) {
dummy.ch[offset] = 'p';
} else if(tmp < 0.70335) {
dummy.ch[offset] = 'q';
} else if(tmp < 0.76322) {
dummy.ch[offset] = 'r';
} else if(tmp < 0.82649) {
dummy.ch[offset] = 's';
} else if(tmp < 0.91705) {
dummy.ch[offset] = 't';
} else if(tmp < 0.94463) {
dummy.ch[offset] = 'u';
} else if(tmp < 0.95441) {
dummy.ch[offset] = 'v';
} else if(tmp < 0.97801) {
dummy.ch[offset] = 'w';
} else if(tmp < 0.97951) {
dummy.ch[offset] = 'x';
} else if(tmp < 0.99925) {
dummy.ch[offset] = 'y';
} else {
dummy.ch[offset] = 'z';
}
// std::cout << "dummy.ch[" << offset << "]=" << dummy.ch[offset] << ", ";//##
}
fDummy[i] = dummy.fp;
// std::cout << "fDummy[" << i << "]=" << fDummy[i] << std::endl;//##
}
}

////////////////////////////////////////////////////////////////////////////////
/// Create an TLarge.
TLarge::TLarge(Int_t size)
Expand Down
27 changes: 27 additions & 0 deletions test/Localcompression.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,37 @@
#include "TBits.h"
#include "TMath.h"

#define DUMMYSIZE 10
#define LARGESIZE 1000000
#define SMALLSIZE 1000
#define FLOATSIZE 6

union Dummy_t {
char ch[4];
Float_t fp;
};

class TDummy : public TObject {

private:
Int_t fSize;
Float_t *fDummy; //[fSize]

public:
TDummy(Int_t size = DUMMYSIZE);
TDummy(const TDummy& dummy);
virtual ~TDummy();
TDummy &operator=(const TDummy &dummy);

void Clear(Option_t *option ="");
void Build();
Int_t GetSize() const { return fSize; }
Float_t *GetDummy() const { return fDummy; }

ClassDef(TDummy,1)
};


class TLarge : public TObject {

private:
Expand Down
1 change: 1 addition & 0 deletions test/LocalcompressionLinkDef.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#pragma link off all classes;
#pragma link off all functions;

#pragma link C++ class TDummy+;
#pragma link C++ class TLarge+;
#pragma link C++ class TSmall+;
#pragma link C++ class TFloat+;
Expand Down
6 changes: 4 additions & 2 deletions tree/tree/src/TBasket.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ See picture in TTree.
*/

static std::chrono::steady_clock::duration globalDuration = std::chrono::steady_clock::duration::zero();//##
static std::chrono::steady_clock::time_point startDuration;
static std::chrono::steady_clock::time_point endDuration;
static std::chrono::steady_clock::time_point startDuration;//##
static std::chrono::steady_clock::time_point endDuration;//##
////////////////////////////////////////////////////////////////////////////////
/// Default contructor.

Expand Down Expand Up @@ -1023,6 +1023,8 @@ Int_t TBasket::WriteBuffer()
else bufmax = kMAXZIPBUF;
//compress the buffer
R__zipMultipleAlgorithm(cxlevel, &bufmax, objbuf, &bufmax, bufcur, &nout, cxAlgorithm);
// std::cout << lbuf << ", " << fObjlen << ", " << fKeylen << ", " << nbuffers << ", " << bufmax << ", " << nout << std::endl;//##
// std::cout << fObjlen << std::endl;//##

// test if buffer has really been compressed. In case of small buffers
// when the buffer contains random data, it may happen that the compressed
Expand Down