Skip to content
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
[hadd] add option to specify what should be merged
  • Loading branch information
marianstahl committed Jul 8, 2020
commit 08c4057be95e6a6d88f101cdd05e3527b913f24d
2 changes: 2 additions & 0 deletions main/src/hadd-argparse.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ def get_argparse():
parser.add_argument("-dbg", help="Parallelize the execution in multiple processes in debug mode (Does not delete partial files stored inside working directory)")
parser.add_argument("-d", help="Carry out the partial multiprocess execution in the specified directory")
parser.add_argument("-n", help="Open at most 'maxopenedfiles' at once (use 0 to request to use the system maximum)")
parser.add_argument("-l", help="Comma-separated list of objects (not) to merge.")
parser.add_argument("-b", help="Use objects passed to -l as blacklist.")
parser.add_argument("-cachesize", help="Resize the prefetching cache use to speed up I/O operations(use 0 to disable)")
parser.add_argument("-experimental-io-features", help="Used with an argument provided, enables the corresponding experimental feature for output trees")
parser.add_argument("-f", help="Gives the ability to specify the compression level of the target file(by default 4) ")
Expand Down
42 changes: 39 additions & 3 deletions main/src/hadd.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
#include "TKey.h"
#include "TClass.h"
#include "TSystem.h"
#include "TStopwatch.h"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
#include "TStopwatch.h"
#include "TStopwatch.h"

Indeed we ought to skip this.

#include "TUUID.h"
#include "ROOT/StringConv.hxx"
#include "snprintf.h"
Expand Down Expand Up @@ -116,6 +117,9 @@ int main( int argc, char **argv )
Int_t maxopenedfiles = 0;
Int_t verbosity = 99;
TString cacheSize;
std::vector<std::string> AddObjectNamesList;
Bool_t blacklist = kFALSE;
TStopwatch clock;
SysInfo_t s;
gSystem->GetSysInfo(&s);
auto nProcesses = s.fCpus;
Expand Down Expand Up @@ -143,7 +147,10 @@ int main( int argc, char **argv )
debug = kTRUE;
verbosity = kTRUE;
++ffirst;
} else if (strcmp(argv[a], "-d") == 0) {
} else if ( strcmp(argv[a],"-b") == 0 ) {
blacklist = kTRUE;
++ffirst;
} else if (strcmp(argv[a], "-d") == 0) {
if (a + 1 != argc && argv[a + 1][0] != '-') {
if (gSystem->AccessPathName(argv[a + 1])) {
std::cerr << "Error: could not access the directory specified: " << argv[a + 1]
Expand Down Expand Up @@ -288,6 +295,18 @@ int main( int argc, char **argv )
}
}
++ffirst;
} else if (!strcmp(argv[a], "-l")) {
if (a+1 >= argc) {
std::cerr << "Error: no AddObjectNamesList items were specified after -l; ignoring\n";
} else {
std::stringstream ss;
ss.str(argv[++a]);
++ffirst;
std::string item;
while (std::getline(ss, item, ','))
AddObjectNamesList.push_back(item);
}
++ffirst;
} else if ( argv[a][0] == '-' ) {
bool farg = false;
if (force && argv[a][1] == 'f') {
Expand Down Expand Up @@ -344,12 +363,15 @@ int main( int argc, char **argv )
}

if (verbosity > 1) {
clock.Start();
std::cout << "hadd Target file: " << targetname << std::endl;
}

TFileMerger fileMerger(kFALSE, kFALSE);
fileMerger.SetMsgPrefix("hadd");
fileMerger.SetPrintLevel(verbosity - 1);
for(const auto& objn : AddObjectNamesList)
fileMerger.AddObjectNames(objn.data());
if (maxopenedfiles > 0) {
fileMerger.SetMaxOpenedFiles(maxopenedfiles);
}
Expand Down Expand Up @@ -398,6 +420,12 @@ int main( int argc, char **argv )
}

auto filesToProcess = argc - ffirst;
for (auto i = argc - ffirst; i < argc; i++)
if (argv[i] && argv[i][0] == '@'){
auto inFile = std::ifstream(argv[i]+1);
filesToProcess = std::count(std::istreambuf_iterator<char>(inFile), std::istreambuf_iterator<char>(), '\n');
}

auto step = (filesToProcess + nProcesses - 1) / nProcesses;
if (multiproc && step < 3) {
// At least 3 files per process
Expand Down Expand Up @@ -437,8 +465,12 @@ int main( int argc, char **argv )
Bool_t status;
if (append)
status = merger.PartialMerge(TFileMerger::kIncremental | TFileMerger::kAll);
else
status = merger.Merge();
else {
Int_t merge_type = TFileMerger::kAll | TFileMerger::kRegular;
if (!AddObjectNamesList.empty())
blacklist ? merge_type |= TFileMerger::kSkipListed : merge_type |= TFileMerger::kOnlyListed;
status = merger.PartialMerge(merge_type);
}
return status;
};

Expand Down Expand Up @@ -515,6 +547,10 @@ int main( int argc, char **argv )
#endif

if (status) {
if (verbosity > 1) {
clock.Stop();
clock.Print();
}
if (verbosity == 1) {
std::cout << "hadd merged " << fileMerger.GetMergeList()->GetEntries() << " input files in " << targetname
<< ".\n";
Expand Down