Skip to content

TMVA various fixes #610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lmoneta merged 8 commits into from
Jun 7, 2017
Merged

TMVA various fixes #610

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
4d9ba3b
Take weights into consideration in text output.
ashlaban Jun 4, 2017
284f1b5
Add doc and explicit multiclass support for Factory:GetROCIntegral
ashlaban Jun 6, 2017
d2714c2
TMVA Multiclass: Disable ranking by effS*purS
ashlaban Jun 6, 2017
485e044
Add output when using event weights with DNN
ashlaban Jun 6, 2017
b6804be
Deprecate DNN arch=STANDARD, early exist check DNN arch=CPU/GPU
ashlaban Jun 6, 2017
2b271c7
clang-format
ashlaban Jun 6, 2017
4540b9f
clang-format
ashlaban Jun 6, 2017
dc66ecc
more clang-format
ashlaban Jun 7, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
clang-format
  • Loading branch information
@ashlaban
ashlaban committed Jun 6, 2017
commit 2b271c74c7fcdea442ec7d30151106c12d405d07
12 changes: 6 additions & 6 deletions tmva/tmva/inc/TMVA/Factory.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,9 @@ namespace TMVA {

Bool_t IsSilentFile();
Bool_t IsModelPersistence();
Double_t GetROCIntegral(DataLoader *loader,TString theMethodName, UInt_t iClass=0);
Double_t GetROCIntegral(TString datasetname,TString theMethodName, UInt_t iClass=0);

Double_t GetROCIntegral(DataLoader *loader, TString theMethodName, UInt_t iClass = 0);
Double_t GetROCIntegral(TString datasetname, TString theMethodName, UInt_t iClass = 0);

// Methods to get a TGraph for an indicated method in dataset.
// Optional title and axis added with fLegend=kTRUE.
Expand Down Expand Up @@ -182,9 +182,9 @@ namespace TMVA {
TH1F* GetImportance(const int nbits,std::vector<Double_t> importances,std::vector<TString> varNames);

// Helpers for public facing ROC methods
ROCCurve * GetROC(DataLoader *loader, TString theMethodName, UInt_t iClass=0);
ROCCurve * GetROC(TString datasetname, TString theMethodName, UInt_t iClass=0);
ROCCurve *GetROC(DataLoader *loader, TString theMethodName, UInt_t iClass = 0);
ROCCurve *GetROC(TString datasetname, TString theMethodName, UInt_t iClass = 0);

void WriteDataInformation(DataSetInfo& fDataSetInfo);

void SetInputTreesFromEventAssignTrees();
Expand Down
85 changes: 49 additions & 36 deletions tmva/tmva/src/Factory.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -661,51 +661,58 @@ std::map<TString,Double_t> TMVA::Factory::OptimizeAllMethods(TString fomType, TS
}

////////////////////////////////////////////////////////////////////////////////
/// Private method to generate an instance of a ROCCurve regardless of
/// Private method to generate an instance of a ROCCurve regardless of
/// analysis type.
///
/// \note You own the retured pointer.
///
///

TMVA::ROCCurve * TMVA::Factory::GetROC(TMVA::DataLoader *loader, TString theMethodName, UInt_t iClass) {
TMVA::ROCCurve *TMVA::Factory::GetROC(TMVA::DataLoader *loader, TString theMethodName, UInt_t iClass)
{
return GetROC((TString)loader->GetName(), theMethodName, iClass);
}

////////////////////////////////////////////////////////////////////////////////
/// Private method to generate an instance of a ROCCurve regardless of
/// Private method to generate an instance of a ROCCurve regardless of
/// analysis type.
///
/// \note You own the retured pointer.
///

TMVA::ROCCurve * TMVA::Factory::GetROC(TString datasetname, TString theMethodName, UInt_t iClass) {
///

TMVA::ROCCurve *TMVA::Factory::GetROC(TString datasetname, TString theMethodName, UInt_t iClass)
{
if (fMethodsMap.find(datasetname) == fMethodsMap.end()) {
Log() << kERROR << Form("DataSet = %s not found in methods map.", datasetname.Data()) << Endl;
return nullptr;
}

if ( ! this->HasMethod(datasetname, theMethodName) ) {
Log() << kERROR << Form("Method = %s not found with Dataset = %s ", theMethodName.Data(), datasetname.Data()) << Endl;

if (!this->HasMethod(datasetname, theMethodName)) {
Log() << kERROR << Form("Method = %s not found with Dataset = %s ", theMethodName.Data(), datasetname.Data())
<< Endl;
return nullptr;
}

std::set<Types::EAnalysisType> allowedAnalysisTypes = {Types::kClassification, Types::kMulticlass};
if ( allowedAnalysisTypes.count(this->fAnalysisType) == 0 ) {
Log() << kERROR << Form("Can only generate ROC curves for analysis type kClassification and kMulticlass.") << Endl;
if (allowedAnalysisTypes.count(this->fAnalysisType) == 0) {
Log() << kERROR << Form("Can only generate ROC curves for analysis type kClassification and kMulticlass.")
<< Endl;
return nullptr;
}

TMVA::MethodBase *method = dynamic_cast<TMVA::MethodBase *>( this->GetMethod(datasetname, theMethodName) );
TMVA::DataSet *dataset = method->Data();
TMVA::Results *results = dataset->GetResults(theMethodName, Types::kTesting, this->fAnalysisType);
TMVA::MethodBase *method = dynamic_cast<TMVA::MethodBase *>(this->GetMethod(datasetname, theMethodName));
TMVA::DataSet *dataset = method->Data();
TMVA::Results *results = dataset->GetResults(theMethodName, Types::kTesting, this->fAnalysisType);

UInt_t nClasses = method->DataInfo().GetNClasses();
if ( this->fAnalysisType == Types::kMulticlass && iClass >= nClasses ) {
Log() << kERROR << Form("Given class number (iClass = %i) does not exist. There are %i classes in dataset.", iClass, nClasses) << Endl;
if (this->fAnalysisType == Types::kMulticlass && iClass >= nClasses) {
Log() << kERROR
<< Form("Given class number (iClass = %i) does not exist. There are %i classes in dataset.", iClass,
nClasses)
<< Endl;
return nullptr;
}

TMVA::ROCCurve * rocCurve = nullptr;
TMVA::ROCCurve *rocCurve = nullptr;
if (this->fAnalysisType == Types::kClassification) {

std::vector<Float_t> *mvaRes = dynamic_cast<ResultsClassification *>(results)->GetValueVector();
Expand All @@ -725,9 +732,9 @@ TMVA::ROCCurve * TMVA::Factory::GetROC(TString datasetname, TString theMethodNam
std::vector<Bool_t> mvaResTypes;
std::vector<Float_t> mvaResWeights;

std::vector<std::vector<Float_t>> * rawMvaRes = dynamic_cast<ResultsMulticlass *>(results)->GetValueVector();
// Vector transpose due to values being stored as
std::vector<std::vector<Float_t>> *rawMvaRes = dynamic_cast<ResultsMulticlass *>(results)->GetValueVector();

// Vector transpose due to values being stored as
// [ [0, 1, 2], [0, 1, 2], ... ]
// in ResultsMulticlass::GetValueVector.
mvaRes.reserve(rawMvaRes->size());
Expand All @@ -752,23 +759,23 @@ TMVA::ROCCurve * TMVA::Factory::GetROC(TString datasetname, TString theMethodNam
////////////////////////////////////////////////////////////////////////////////
/// Calculate the integral of the ROC curve, also known as the area under curve
/// (AUC), for a given method.
///
/// Argument iClass specifies the class to generate the ROC curve in a
///
/// Argument iClass specifies the class to generate the ROC curve in a
/// multiclass setting. It is ignored for binary classification.
///
///

Double_t TMVA::Factory::GetROCIntegral(TMVA::DataLoader *loader, TString theMethodName, UInt_t iClass)
{
return GetROCIntegral((TString)loader->GetName(), theMethodName, iClass);
return GetROCIntegral((TString)loader->GetName(), theMethodName, iClass);
}

////////////////////////////////////////////////////////////////////////////////
/// Calculate the integral of the ROC curve, also known as the area under curve
/// (AUC), for a given method.
///
/// Argument iClass specifies the class to generate the ROC curve in a
///
/// Argument iClass specifies the class to generate the ROC curve in a
/// multiclass setting. It is ignored for binary classification.
///
///

Double_t TMVA::Factory::GetROCIntegral(TString datasetname, TString theMethodName, UInt_t iClass)
{
Expand All @@ -784,13 +791,17 @@ Double_t TMVA::Factory::GetROCIntegral(TString datasetname, TString theMethodNam

std::set<Types::EAnalysisType> allowedAnalysisTypes = {Types::kClassification, Types::kMulticlass};
if ( allowedAnalysisTypes.count(this->fAnalysisType) == 0 ) {
Log() << kERROR << Form("Can only generate ROC integral for analysis type kClassification. and kMulticlass.") << Endl;
Log() << kERROR << Form("Can only generate ROC integral for analysis type kClassification. and kMulticlass.")
<< Endl;
return 0;
}

TMVA::ROCCurve *rocCurve = GetROC(datasetname, theMethodName, iClass);
if (!rocCurve) {
Log() << kFATAL << Form("ROCCurve object was not created in Method = %s not found with Dataset = %s ", theMethodName.Data(), datasetname.Data()) << Endl;
Log() << kFATAL
<< Form("ROCCurve object was not created in Method = %s not found with Dataset = %s ", theMethodName.Data(),
datasetname.Data())
<< Endl;
}

Int_t npoints = TMVA::gConfig().fVariablePlotting.fNbinsXOfROCCurve + 1;
Expand Down Expand Up @@ -1364,7 +1375,6 @@ void TMVA::Factory::EvaluateAllMethods( void )
doMulticlass = kTRUE;
Log() << kINFO << "Evaluate multiclass classification method: " << theMethod->GetMethodName() << Endl;


// This part uses a genetic alg. to evaluate the optimal sig eff * sig pur.
// This is why it is disabled for now.
// Find approximate optimal working point w.r.t. signalEfficiency * signalPurity.
Expand Down Expand Up @@ -1734,7 +1744,7 @@ void TMVA::Factory::EvaluateAllMethods( void )

// This part uses a genetic alg. to evaluate the optimal sig eff * sig pur.
// This is why it is disabled for now.
//
//
// // --- Acheivable signal efficiency * signal purity
// // --------------------------------------------------------------------
// Log() << kINFO << Endl;
Expand Down Expand Up @@ -1946,7 +1956,8 @@ void TMVA::Factory::EvaluateAllMethods( void )

TMVA::DataSet *dataset = theMethod->Data();
TMVA::Results *results = dataset->GetResults(methodName, Types::kTesting, this->fAnalysisType);
std::vector<Bool_t> *mvaResType = dynamic_cast<ResultsClassification *>(results)->GetValueVectorTypes();
std::vector<Bool_t> *mvaResType =
dynamic_cast<ResultsClassification *>(results)->GetValueVectorTypes();

Double_t rocIntegral = 0.0;
if (mvaResType->size() != 0) {
Expand All @@ -1955,7 +1966,8 @@ void TMVA::Factory::EvaluateAllMethods( void )

if (sep[k][i] < 0 || sig[k][i] < 0) {
// cannot compute separation/significance -> no MVA (usually for Cuts)
Log() << kINFO << Form("%-13s %-15s: %#1.3f", datasetName.Data(), methodName.Data(), effArea[k][i]) << Endl;
Log() << kINFO << Form("%-13s %-15s: %#1.3f", datasetName.Data(), methodName.Data(), effArea[k][i])
<< Endl;

// Log() << kDEBUG << Form("%-20s %-15s: %#1.3f(%02i) %#1.3f(%02i) %#1.3f(%02i)
// %#1.3f %#1.3f | -- --",
Expand All @@ -1966,7 +1978,8 @@ void TMVA::Factory::EvaluateAllMethods( void )
// eff30[k][i], Int_t(1000*eff30err[k][i]),
// effArea[k][i],rocIntegral) << Endl;
} else {
Log() << kINFO << Form("%-13s %-15s: %#1.3f", datasetName.Data(), methodName.Data(), rocIntegral) << Endl;
Log() << kINFO << Form("%-13s %-15s: %#1.3f", datasetName.Data(), methodName.Data(), rocIntegral)
<< Endl;
// Log() << kDEBUG << Form("%-20s %-15s: %#1.3f(%02i) %#1.3f(%02i) %#1.3f(%02i)
// %#1.3f %#1.3f | %#1.3f %#1.3f",
// datasetName.Data(),
Expand Down
85 changes: 50 additions & 35 deletions tmva/tmva/src/MethodDNN.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,12 +122,11 @@ Bool_t TMVA::MethodDNN::HasAnalysisType(Types::EAnalysisType type,
void TMVA::MethodDNN::Init()
{
// TODO: Remove once weights are considered by the method.
auto & dsi = this->DataInfo();
auto &dsi = this->DataInfo();
auto numClasses = dsi.GetNClasses();
for (UInt_t i = 0; i < numClasses; ++i) {
if (dsi.GetWeightExpression(i) != TString("")) {
Log() << kERROR
<< "Currently event weights are not considered properly by this method." << Endl;
Log() << kERROR << "Currently event weights are not considered properly by this method." << Endl;
Log() << kFATAL << "See above." << Endl;
}
}
Expand Down Expand Up @@ -418,47 +417,63 @@ void TMVA::MethodDNN::ProcessOptions()
}

if (fArchitectureString == "STANDARD") {
Log() << kERROR << "The STANDARD architecture has been deprecated. "
"Please use Architecture=CPU or Architecture=CPU."
"See the TMVA Users' Guide for instructions if you "
"encounter problems." << Endl;
Log() << kFATAL << "The STANDARD architecture has been deprecated. "
"Please use Architecture=CPU or Architecture=CPU."
"See the TMVA Users' Guide for instructions if you "
"encounter problems." << Endl;
Log() << kERROR
<< "The STANDARD architecture has been deprecated. "
"Please use Architecture=CPU or Architecture=CPU."
"See the TMVA Users' Guide for instructions if you "
"encounter problems."
<< Endl;
Log() << kFATAL
<< "The STANDARD architecture has been deprecated. "
"Please use Architecture=CPU or Architecture=CPU."
"See the TMVA Users' Guide for instructions if you "
"encounter problems."
<< Endl;
}

if (fArchitectureString == "OPENCL") {
Log() << kERROR << "The OPENCL architecture has not been implemented yet. "
"Please use Architecture=CPU or Architecture=CPU for the "
"time being. See the TMVA Users' Guide for instructions "
"if you encounter problems." << Endl;
Log() << kFATAL << "The OPENCL architecture has not been implemented yet. "
"Please use Architecture=CPU or Architecture=CPU for the "
"time being. See the TMVA Users' Guide for instructions "
"if you encounter problems." << Endl;
Log() << kERROR
<< "The OPENCL architecture has not been implemented yet. "
"Please use Architecture=CPU or Architecture=CPU for the "
"time being. See the TMVA Users' Guide for instructions "
"if you encounter problems."
<< Endl;
Log() << kFATAL
<< "The OPENCL architecture has not been implemented yet. "
"Please use Architecture=CPU or Architecture=CPU for the "
"time being. See the TMVA Users' Guide for instructions "
"if you encounter problems."
<< Endl;
}

if (fArchitectureString == "GPU") {
#ifndef DNNCUDA // Included only if DNNCUDA flag is _not_ set.
Log() << kERROR << "CUDA backend not enabled. Please make sure "
"you have CUDA installed and it was successfully "
"detected by CMAKE." << Endl;
Log() << kFATAL << "CUDA backend not enabled. Please make sure "
"you have CUDA installed and it was successfully "
"detected by CMAKE." << Endl;
#endif // DNNCUDA
#ifndef DNNCUDA // Included only if DNNCUDA flag is _not_ set.
Log() << kERROR
<< "CUDA backend not enabled. Please make sure "
"you have CUDA installed and it was successfully "
"detected by CMAKE."
<< Endl;
Log() << kFATAL
<< "CUDA backend not enabled. Please make sure "
"you have CUDA installed and it was successfully "
"detected by CMAKE."
<< Endl;
#endif // DNNCUDA
}

if (fArchitectureString == "CPU") {
#ifndef DNNCPU // Included only if DNNCPU flag is _not_ set.
Log() << kERROR << "Multi-core CPU backend not enabled. Please make sure "
"you have a BLAS implementation and it was successfully "
"detected by CMake as well that the imt CMake flag is set." << Endl;
Log() << kFATAL << "Multi-core CPU backend not enabled. Please make sure "
"you have a BLAS implementation and it was successfully "
"detected by CMake as well that the imt CMake flag is set." << Endl;
#endif // DNNCPU
#ifndef DNNCPU // Included only if DNNCPU flag is _not_ set.
Log() << kERROR
<< "Multi-core CPU backend not enabled. Please make sure "
"you have a BLAS implementation and it was successfully "
"detected by CMake as well that the imt CMake flag is set."
<< Endl;
Log() << kFATAL
<< "Multi-core CPU backend not enabled. Please make sure "
"you have a BLAS implementation and it was successfully "
"detected by CMake as well that the imt CMake flag is set."
<< Endl;
#endif // DNNCPU
}

//
Expand Down
1 change: 0 additions & 1 deletion tmva/tmva/src/ROCCurve.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,6 @@ std::vector<Double_t> TMVA::ROCCurve::ComputeSpecificity(const UInt_t num_points
specificity_vector.push_back(specificity);
}


specificity_vector.push_back(1.0);
return specificity_vector;
}
Expand Down