Skip to content

TMVA MultiProcessing #858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mammadhajili wants to merge 4 commits into from
Closed

TMVA MultiProcessing #858

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b191f1b
TMVA MultiProcessing
Aug 15, 2017
6b66171
Fixed clang-format issues
mammadhajili Aug 16, 2017
691f6cd
Loading files fixes and clang format
mammadhajili Aug 18, 2017
d34f639
Fixing clang-format issues
mammadhajili Aug 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Loading files fixes and clang format
  • Loading branch information
@mammadhajili
mammadhajili committed Aug 18, 2017
commit 691f6cdb47a14dab5a05a3d5d1301654778245d7
5 changes: 2 additions & 3 deletions tmva/tmva/inc/TMVA/Config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ namespace TMVA {
void SetDrawProgressBar( Bool_t d ) { fDrawProgressBar = d; }

UInt_t NWorkers() const { return fNWorkers; }
void SetNWorkers ( UInt_t n ) { fNWorkers = n; }
void SetNWorkers (UInt_t n) { fNWorkers = n; }

public:

Expand Down Expand Up @@ -101,7 +101,6 @@ namespace TMVA {
TString fOptionsReferenceFileDir;
} fIONames; // Customisable weight file properties


private:

// private constructor
Expand All @@ -121,7 +120,7 @@ namespace TMVA {
std::atomic<Bool_t> fSilent; // no output at all
std::atomic<Bool_t> fWriteOptionsReference; // if set true: Configurable objects write file with option reference
std::atomic<Bool_t> fDrawProgressBar; // draw progress bar to indicate training evolution
std::atomic<UInt_t> fNWorkers; // number of workers in multiprocessing parallelization
std::atomic<UInt_t> fNWorkers;
#else
Bool_t fUseColoredConsole; // coloured standard output
Bool_t fSilent; // no output at all
Expand Down
11 changes: 5 additions & 6 deletions tmva/tmva/inc/TMVA/DataLoader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,16 +85,15 @@ namespace TMVA {
// special case: signal/background

// Data input related
void SetInputTrees( const TString& signalFileName, const TString& backgroundFileName,
void SetInputTrees(const TString& signalFileName, const TString& backgroundFileName,
Double_t signalWeight=1.0, Double_t backgroundWeight=1.0 );
void SetInputTrees( TTree* inputTree, const TCut& SigCut, const TCut& BgCut );
void SetInputTrees( TTree *inputTree, const TCut& SigCut, const TCut& BgCut );
// Set input trees at once
void SetInputTrees( TTree* signal, TTree* background,
Double_t signalWeight=1.0, Double_t backgroundWeight=1.0) ;
void SetInputTrees( TTree *signal, TTree* background, Double_t signalWeight=1.0, Double_t backgroundWeight=1.0) ;

void AddSignalTree( TTree* signal, Double_t weight=1.0, Types::ETreeType treetype = Types::kMaxTreeType );
void AddSignalTree( TTree *signal, Double_t weight=1.0, Types::ETreeType treetype = Types::kMaxTreeType );
void AddSignalTree( TString datFileS, Double_t weight=1.0, Types::ETreeType treetype = Types::kMaxTreeType );
void AddSignalTree( TTree* signal, Double_t weight, const TString& treetype );
void AddSignalTree( TTree *signal, Double_t weight, const TString& treetype );

// ... depreciated, kept for backwards compatibility
void SetSignalTree( TTree* signal, Double_t weight=1.0);
Expand Down
43 changes: 38 additions & 5 deletions tmva/tmva/src/DataLoader.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -868,9 +868,43 @@ std::vector<std::shared_ptr<TFile>> TMVA::DataLoaderCopyMP(TMVA::DataLoader *des
{

std::vector<std::shared_ptr<TFile>> vec_files;
for(std::vector<TreeInfo>::const_iterator treeinfo_signal=src->DataInput().Sbegin(),
std::map<TString, std::shared_ptr<TFile>> map_files;
for(std::vector<TreeInfo>::const_iterator treeinfo = src->DataInput().Sbegin(); treeinfo != src->DataInput().Send(); ++ treeinfo) {
TTree *stree = treeinfo -> GetTree();
TString sfileName = stree->GetCurrentFile()->GetName();
std::shared_ptr<TFile> sfile;
if(map_files.find(sfileName) == map_files.end()) {
sfile = std::shared_ptr<TFile>(TFile::Open(sfileName)) ;
map_files[sfileName] = sfile;
vec_files.push_back(sfile);
}
else {
sfile = map_files[sfileName];
}

TTree* signalTree = (TTree*)sfile->Get(stree->GetName());
des->AddSignalTree(signalTree);
}
for(std::vector<TreeInfo>::const_iterator treeinfo = src->DataInput().Bbegin(); treeinfo != src->DataInput().Bend(); ++ treeinfo) {
TTree *btree = treeinfo -> GetTree();
TString bfileName = btree->GetCurrentFile()->GetName();
std::shared_ptr<TFile> bfile;
if(map_files.find(bfileName) == map_files.end()) {
bfile = std::shared_ptr<TFile>(TFile::Open(bfileName)) ;
map_files[bfileName] = bfile;
vec_files.push_back(bfile);
}
else {
bfile = map_files[bfileName];
}

TTree* backgroundTree = (TTree*)bfile->Get(btree->GetName());
des->AddBackgroundTree(backgroundTree);
}

/*for(std::vector<TreeInfo>::const_iterator treeinfo_signal=src->DataInput().Sbegin(),
treeinfo_back=src->DataInput().Bbegin();
treeinfo_signal!=src->DataInput().Send(), treeinfo_back!=src->DataInput().Bend();
treeinfo_signal!=src->DataInput().Send() | treeinfo_back!=src->DataInput().Bend();
treeinfo_signal++, treeinfo_back++) {
TTree *stree = treeinfo_signal -> GetTree();
TTree *btree = treeinfo_back -> GetTree();
Expand All @@ -892,11 +926,10 @@ std::vector<std::shared_ptr<TFile>> TMVA::DataLoaderCopyMP(TMVA::DataLoader *des

vec_files.push_back(sfile);
vec_files.push_back(bfile);
Copy link
Contributor

@ashlaban ashlaban Aug 23, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can these commented out lines be removed?

}
}*/
return vec_files;
}
void TMVA::DataLoaderCopyMPCloseFiles(std::vector<std::shared_ptr<TFile>> files)
{
void TMVA::DataLoaderCopyMPCloseFiles(std::vector<std::shared_ptr<TFile> > files) {
for(auto file: files) {
file->Close();
}
Expand Down
12 changes: 10 additions & 2 deletions tmva/tmva/src/HyperParameterOptimisation.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ void TMVA::HyperParameterOptimisationResult::Print() const
TMVA::HyperParameterOptimisation::HyperParameterOptimisation(TMVA::DataLoader *dataloader):Envelope("HyperParameterOptimisation",dataloader),
fFomType("Separation"),
fFitType("Minuit"),
fNumFolds(5),
fNumFolds(4),
Copy link
Contributor

@ashlaban ashlaban Aug 23, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Prefer to change as little as possible (makes it easier to merge with other people's work!). I'd keep this as the old value unless there is a good motivation for this change. (This principle applies to more places)

fResults(),
fClassifier(new TMVA::Factory("HyperParameterOptimisation","!V:!ROC:Silent:!ModelPersistence:!Color:!DrawProgressBar:AnalysisType=Classification"))
{
Expand All @@ -90,6 +90,7 @@ void TMVA::HyperParameterOptimisation::SetNumFolds(UInt_t i)

void TMVA::HyperParameterOptimisation::Evaluate()
{
cout << "Number of Workers : " << TMVA::gConfig().NWorkers() << endl;
TString methodName = fMethod.GetValue<TString>("MethodName");
TString methodTitle = fMethod.GetValue<TString>("MethodTitle");
TString methodOptions = fMethod.GetValue<TString>("MethodOptions");
Expand All @@ -102,7 +103,8 @@ void TMVA::HyperParameterOptimisation::Evaluate()
fResults.fMethodName = methodName;
auto workItem = [&](UInt_t workerID) {

TString foldTitle = methodTitle;
TString foldTitle = methodTitle;

foldTitle += "_opt";
foldTitle += workerID+1;

Expand All @@ -124,16 +126,22 @@ void TMVA::HyperParameterOptimisation::Evaluate()
};
vector<map<TString,Double_t>> res;
auto nWorkers = TMVA::gConfig().NWorkers();
cout << "Number of Workers : " << TMVA::gConfig().NWorkers() << endl;
Copy link
Contributor

@ashlaban ashlaban Aug 23, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These seem like debug print outs, please use Log() << kDEBUG << ... << Endl; for these kind of messages.

if(nWorkers > 1) {
cout << "I am here" << endl;
ROOT::TProcessExecutor workers(nWorkers);
cout << "Number of Workers : " << TMVA::gConfig().NWorkers() << endl;
res = workers.Map(workItem, ROOT::TSeqI(fNumFolds));
cout << "Number of Workers : " << TMVA::gConfig().NWorkers() << endl;
} else {
for(UInt_t i = 0; i < fNumFolds; ++ i) {
res.push_back(workItem(i));
}
}
cout << "Number of Workers : " << TMVA::gConfig().NWorkers() << endl;
for(auto results : res) {
fResults.fFoldParameters.push_back(results);
}
cout << "Number of Workers : " << TMVA::gConfig().NWorkers() << endl;

}