Skip to content

First CPU implementation of HashedNets compression algorithm for DNNs #220

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adi-sharma wants to merge 42 commits into from
Closed

First CPU implementation of HashedNets compression algorithm for DNNs #220

Changes from 1 commit
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
818afb9
included xxhash.h
adi-sharma Jun 23, 2016
db6990f
added hashing initializations
adi-sharma Jun 24, 2016
efa2efb
added weightBucket
adi-sharma Jun 24, 2016
c821a57
edited applyWeights() without drop-out as an example
adi-sharma Jun 28, 2016
b6328a7
Updated all applyWeights() with HashedNets
adi-sharma Jul 1, 2016
a58ac1d
Update NeuralNet.h
adi-sharma Jul 1, 2016
c81f248
updated update() and Steepest Gradient Descent
adi-sharma Jul 1, 2016
96c8f42
Updated SGD operator() with HashedNets
adi-sharma Jul 1, 2016
0250775
updated weightDecay()
adi-sharma Jul 1, 2016
7c62f3b
updated weightDecay() with HashedNets
adi-sharma Jul 1, 2016
fe93bb2
Edited dropOutWeightFactor()
adi-sharma Jul 13, 2016
13d151a
Edited dropOutWeightFactor()
adi-sharma Jul 13, 2016
725bba3
Updated with HashedNets
adi-sharma Jul 18, 2016
4ec8917
Updated train() with HashedNets
adi-sharma Jul 18, 2016
b1caf73
Updated with complete HashedNets
adi-sharma Aug 23, 2016
c696958
Updated with complete HashedNets
adi-sharma Aug 23, 2016
8d6b2d4
Updated with Complete HashedNets
adi-sharma Aug 23, 2016
e277c73
Update MethodDNN.h
adi-sharma Aug 23, 2016
a820db6
Update NeuralNet.h
adi-sharma Aug 23, 2016
919a319
Updated with HashedNets
adi-sharma Aug 23, 2016
4d9e46c
Error corrections
adi-sharma Aug 24, 2016
3d7eec3
Error corrections NeuralNet.icc
adi-sharma Aug 24, 2016
d202ae1
Error corrections MethodDNN.cxx
adi-sharma Aug 24, 2016
52dc851
Successful compile
adi-sharma Aug 25, 2016
70484aa
Successful compile
adi-sharma Aug 25, 2016
6a0932f
Successful compile
adi-sharma Aug 25, 2016
4f5b919
Successful compile
adi-sharma Aug 25, 2016
96b29df
Successful build NeuralNet.h
adi-sharma Aug 25, 2016
f301784
Successful build NeuralNet.icc
adi-sharma Aug 25, 2016
9f1dce5
Successful build NeuralNet.cxx
adi-sharma Aug 25, 2016
f5e6943
Update NeuralNet.icc
adi-sharma Aug 26, 2016
c4f8749
Update NeuralNet.h
adi-sharma Aug 26, 2016
9637a69
Update MethodDNN.cxx
adi-sharma Aug 26, 2016
f62fcf2
Update NeuralNet.cxx
adi-sharma Aug 26, 2016
e48d24f
Made some logical changes in HashedNets
adi-sharma Aug 27, 2016
2bf3295
Made some logical changes in HashedNets
adi-sharma Aug 27, 2016
e048347
Update NeuralNet.icc
adi-sharma Aug 28, 2016
24c19c2
Production version v1.0
adi-sharma Aug 28, 2016
3c7f7ac
Production Version v1.0
adi-sharma Aug 28, 2016
e6332d5
Production Version v1.0
adi-sharma Aug 28, 2016
5c33612
Production Version v1.0
adi-sharma Aug 28, 2016
6eed664
Production Version v1.0
adi-sharma Aug 28, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Production Version v1.0
  • Loading branch information
@adi-sharma
adi-sharma authored Aug 28, 2016
commit e6332d5de059bb6431bd8ae31d4b1f6a75b08a8f
70 changes: 36 additions & 34 deletions tmva/tmva/inc/TMVA/NeuralNet.icc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ namespace TMVA
*/
template <bool HasDropOut, typename ItSource, typename ItTarget, typename ItDrop>
void applyWeights (ItSource itSourceBegin, ItSource itSourceEnd,
int itWeight, std::vector<double>& weightBucket, size_t layerNumber,
int itWeight, std::vector<double>& weightBucket, size_t layerNumber, int BUCKET_SIZE,
ItTarget itTargetBegin, ItTarget itTargetEnd,
ItDrop itDrop)
{
Expand All @@ -105,7 +105,7 @@ template <bool HasDropOut, typename ItSource, typename ItTarget, typename ItDrop
* itDrop correlates with itPrev (to be in agreement with "applyWeights" where it correlates with itSources (same node as itTarget here in applyBackwards)
*/
template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
void applyWeightsBackwards (ItSource itCurrBegin, ItSource itCurrEnd, int itWeight, std::vector<double>& weightBucket, size_t layerNumber, ItPrev itPrevBegin, ItPrev itPrevEnd, ItDrop itDrop)
void applyWeightsBackwards (ItSource itCurrBegin, ItSource itCurrEnd, int itWeight, std::vector<double>& weightBucket, size_t layerNumber, int BUCKET_SIZE, ItPrev itPrevBegin, ItPrev itPrevEnd, ItDrop itDrop)
{
for (auto itPrev = itPrevBegin; itPrev != itPrevEnd; ++itPrev)
{
Expand Down Expand Up @@ -170,7 +170,7 @@ template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
void update (ItSource itSource, ItSource itSourceEnd,
ItDelta itTargetDeltaBegin, ItDelta itTargetDeltaEnd,
ItTargetGradient itTargetGradientBegin,
int itGradient, std::vector<double>& gradientBucket, size_t layerNumber)
int itGradient, std::vector<double>& gradientBucket, size_t layerNumber, int BUCKET_SIZE)
{
while (itSource != itSourceEnd)
{
Expand Down Expand Up @@ -225,7 +225,7 @@ template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
ItDelta itTargetDeltaBegin, ItDelta itTargetDeltaEnd,
ItTargetGradient itTargetGradientBegin,
int itGradient, std::vector<double>& gradientBucket,
int itWeight, std::vector<double>& weightBucket, double& factorWeightDecay, size_t layerNumber)
int itWeight, std::vector<double>& weightBucket, double& factorWeightDecay, size_t layerNumber, int BUCKET_SIZE)
{
// ! the factor weightDecay has to be already scaled by 1/n where n is the number of weights
while (itSource != itSourceEnd)
Expand Down Expand Up @@ -254,7 +254,7 @@ template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
* Can be used with multithreading (i.e. "HogWild!" style); see call in trainCycle
*/
template <typename Function, typename PassThrough>
double Steepest::operator() (Function& fitnessFunction, std::vector<double>& weightBucket, PassThrough& passThrough, const size_t& numWeights, std::vector<int>& layerWeightNumber)
double Steepest::operator() (Function& fitnessFunction, std::vector<double>& weightBucket, PassThrough& passThrough, const size_t& numWeights, std::vector<int>& layerWeightNumber, const int& BUCKET_SIZE)
{
// std::vector<double> gradients (numWeights, 0.0);
std::vector<double> gradientBucket (weightBucket.size (), 0.0);
Expand Down Expand Up @@ -490,7 +490,7 @@ template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
*
*/
template <typename EnumRegularization>
double weightDecay (double error, int currLayerWeightIndex, int nextLayerWeightIndex, std::vector<double>& weightBucket, double factorWeightDecay, EnumRegularization eRegularization, size_t layerNumber)
double weightDecay (double error, int currLayerWeightIndex, int nextLayerWeightIndex, std::vector<double>& weightBucket, double factorWeightDecay, EnumRegularization eRegularization, size_t layerNumber, int BUCKET_SIZE)
{
if (eRegularization == EnumRegularization::L1)
{
Expand Down Expand Up @@ -540,20 +540,20 @@ template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
*
*/
template <typename LAYERDATA>
void forward (const LAYERDATA& prevLayerData, LAYERDATA& currLayerData, std::vector<double>& weightBucket, size_t layerNumber)
void forward (const LAYERDATA& prevLayerData, LAYERDATA& currLayerData, std::vector<double>& weightBucket, size_t layerNumber, int BUCKET_SIZE)
{
if (prevLayerData.hasDropOut ())
{
applyWeights<true> (prevLayerData.valuesBegin (), prevLayerData.valuesEnd (),
currLayerData.weightsBegin (), weightBucket, layerNumber,
currLayerData.weightsBegin (), weightBucket, layerNumber, BUCKET_SIZE,
currLayerData.valuesBegin (), currLayerData.valuesEnd (),
prevLayerData.dropOut ());
}
else
{
bool dummy = true;
applyWeights<false> (prevLayerData.valuesBegin (), prevLayerData.valuesEnd (),
currLayerData.weightsBegin (), weightBucket, layerNumber,
currLayerData.weightsBegin (), weightBucket, layerNumber, BUCKET_SIZE,
currLayerData.valuesBegin (), currLayerData.valuesEnd (),
&dummy); // dummy to turn on all nodes (no drop out)
}
Expand All @@ -566,20 +566,20 @@ template <bool HasDropOut, typename ItSource, typename ItPrev, typename ItDrop>
*
*/
template <typename LAYERDATA>
void backward (LAYERDATA& prevLayerData, LAYERDATA& currLayerData, std::vector<double>& weightBucket, size_t layerNumber)
void backward (LAYERDATA& prevLayerData, LAYERDATA& currLayerData, std::vector<double>& weightBucket, size_t layerNumber, int BUCKET_SIZE)
{
if (prevLayerData.hasDropOut ())
{
applyWeightsBackwards<true> (currLayerData.deltasBegin (), currLayerData.deltasEnd (),
currLayerData.weightsBegin (), weightBucket, layerNumber,
currLayerData.weightsBegin (), weightBucket, layerNumber, BUCKET_SIZE,
prevLayerData.deltasBegin (), prevLayerData.deltasEnd (),
prevLayerData.dropOut ());
}
else
{
bool dummy = true;
applyWeightsBackwards<false> (currLayerData.deltasBegin (), currLayerData.deltasEnd (),
currLayerData.weightsBegin (), weightBucket, layerNumber,
currLayerData.weightsBegin (), weightBucket, layerNumber, BUCKET_SIZE,
prevLayerData.deltasBegin (), prevLayerData.deltasEnd (),
&dummy); // dummy to use all nodes (no drop out)
}
Expand All @@ -594,7 +594,7 @@ template <typename LAYERDATA>
*
*/
template <typename LAYERDATA>
void update (const LAYERDATA& prevLayerData, LAYERDATA& currLayerData, double factorWeightDecay, EnumRegularization regularization, std::vector<double>& weightBucket, std::vector<double>& gradientBucket, size_t layerNumber)
void update (const LAYERDATA& prevLayerData, LAYERDATA& currLayerData, double factorWeightDecay, EnumRegularization regularization, std::vector<double>& weightBucket, std::vector<double>& gradientBucket, size_t layerNumber, int BUCKET_SIZE)
{
// ! the "factorWeightDecay" has already to be scaled by 1/n where n is the number of weights
if (factorWeightDecay != 0.0) // has weight regularization
Expand All @@ -605,7 +605,7 @@ template <typename LAYERDATA>
currLayerData.deltasEnd (),
currLayerData.valueGradientsBegin (),
currLayerData.gradientsBegin (), gradientBucket,
currLayerData.weightsBegin (), weightBucket, factorWeightDecay, layerNumber);
currLayerData.weightsBegin (), weightBucket, factorWeightDecay, layerNumber, BUCKET_SIZE);
}
else if (regularization == EnumRegularization::L2) // L2 regularization ( sum(w^2) )
{
Expand All @@ -614,22 +614,22 @@ template <typename LAYERDATA>
currLayerData.deltasEnd (),
currLayerData.valueGradientsBegin (),
currLayerData.gradientsBegin (), gradientBucket,
currLayerData.weightsBegin (), weightBucket, factorWeightDecay, layerNumber);
currLayerData.weightsBegin (), weightBucket, factorWeightDecay, layerNumber, BUCKET_SIZE);
}
else
{
update (prevLayerData.valuesBegin (), prevLayerData.valuesEnd (),
currLayerData.deltasBegin (), currLayerData.deltasEnd (),
currLayerData.valueGradientsBegin (),
currLayerData.gradientsBegin (), gradientBucket, layerNumber);
currLayerData.gradientsBegin (), gradientBucket, layerNumber, BUCKET_SIZE);
}

else
{ // no weight regularization
update (prevLayerData.valuesBegin (), prevLayerData.valuesEnd (),
currLayerData.deltasBegin (), currLayerData.deltasEnd (),
currLayerData.valueGradientsBegin (),
currLayerData.gradientsBegin (), gradientBucket, layerNumber);
currLayerData.gradientsBegin (), gradientBucket, layerNumber, BUCKET_SIZE);
}
}

Expand Down Expand Up @@ -686,7 +686,7 @@ template <typename LAYERDATA>
p = 1.0/p;
}
// size_t _numWeights = layer.numWeights (numNodesPrev);
for (size_t iWeightBucket = 0; iWeightBucket < BUCKET_SIZE; ++iWeightBucket)
for (size_t iWeightBucket = 0; iWeightBucket < m_bucketSize; ++iWeightBucket)
{
if (itWeightBucket == itWeightBucketEnd)
break;
Expand Down Expand Up @@ -728,6 +728,8 @@ template <typename LAYERDATA>
settings.create ("trainErrors", 100, 0, 100, 100, 0,1);
settings.create ("testErrors", 100, 0, 100, 100, 0,1);

m_bucketSize = settings.bucketSize ();

size_t cycleCount = 0;
size_t testCycleCount = 0;
double testError = 1e20;
Expand Down Expand Up @@ -935,6 +937,7 @@ template <typename LAYERDATA>
size_t numPattern = std::distance (itPatternBegin, itPatternEnd);
size_t numBatches = numPattern/settings.batchSize ();
size_t numBatches_stored = numBatches;
const int const_m_bucketSize = m_bucketSize;

std::random_shuffle (itPatternBegin, itPatternEnd);
Iterator itPatternBatchBegin = itPatternBegin;
Expand Down Expand Up @@ -989,7 +992,7 @@ template <typename LAYERDATA>
{
Batch& batch = *it;
pass_through_type settingsAndBatch (settings, batch, dropContainer);
localError += minimizer ((*this), weightBucket, settingsAndBatch, numWeights (), layerWeightNumber); /// call the minimizer
localError += minimizer ((*this), weightBucket, settingsAndBatch, numWeights (), layerWeightNumber, const_m_bucketSize); /// call the minimizer
}
return localError;
})
Expand All @@ -1004,7 +1007,7 @@ template <typename LAYERDATA>
for (auto& batch : batches)
{
pass_through_type settingsAndBatch (settings, batch, dropContainer);
error += minimizer ((*this), weightBucket, settingsAndBatch, numWeights (), layerWeightNumber);
error += minimizer ((*this), weightBucket, settingsAndBatch, numWeights (), layerWeightNumber, const_m_bucketSize);
}
}

Expand All @@ -1025,7 +1028,7 @@ template <typename LAYERDATA>
* \param weights the weight data
*/
template <typename Weights>
std::vector<double> Net::compute (const std::vector<double>& input, Weights& weightBucket) const
std::vector<double> Net::compute (const std::vector<double>& input, Weights& weightBucket, int BUCKET_SIZE) const
{
std::vector<LayerData> layerData;
layerData.reserve (m_layers.size ()+1);
Expand All @@ -1048,7 +1051,7 @@ template <typename LAYERDATA>


// --------- forward -------------
forwardPattern (m_layers, layerData, weightBucket);
forwardPattern (m_layers, layerData, weightBucket, BUCKET_SIZE);

// ------------- fetch output ------------------
std::vector<double> output;
Expand All @@ -1061,7 +1064,7 @@ template <typename LAYERDATA>
double Net::operator() (PassThrough& settingsAndBatch, std::vector<double>& weightBucket) const
{
std::vector<double> nothing; // empty gradients; no backpropagation is done, just forward
// assert (numWeights () == (weightBucket.size() * BUCKET_SIZE));

double error = forward_backward(m_layers, settingsAndBatch, 0, numWeights () - 1, 0, 0, 10000, nothing, false, weightBucket, nothing);
return error;
}
Expand All @@ -1070,7 +1073,7 @@ template <typename LAYERDATA>
double Net::operator() (PassThrough& settingsAndBatch, std::vector<double>& weightBucket, ModeOutput /*eFetch*/, OutContainer& outputContainer) const
{
std::vector<double> nothing; // empty gradients; no backpropagation is done, just forward
// assert (numWeights () == (weightBucket.size () * BUCKET_SIZE));

double error = forward_backward(m_layers, settingsAndBatch, 0, numWeights () - 1, 0, 0, 10000, outputContainer, true, weightBucket, nothing);
return error;
}
Expand All @@ -1081,8 +1084,7 @@ template <typename LAYERDATA>
{
std::vector<double> nothing;
// std::cout<<"\nnumWeights = "<<numWeights ()<<"\n";
// std::cout<<"((weightBucket.size () / BUCKET_SIZE) * numWeights ()) = "<<((weightBucket.size () / BUCKET_SIZE) * numWeights ());
// assert (numWeights () == (weightBucket.size () * BUCKET_SIZE));

assert ((weightBucket.size ()) == (gradientBucket.size ()));
double error = forward_backward(m_layers, settingsAndBatch, 0, numWeights () - 1, 0, numWeights () - 1, 0, nothing, false, weightBucket, gradientBucket);
return error;
Expand All @@ -1092,7 +1094,7 @@ template <typename LAYERDATA>
double Net::operator() (PassThrough& settingsAndBatch, std::vector<double>& weightBucket, std::vector<double>& gradientBucket, ModeOutput eFetch, OutContainer& outputContainer) const
{
MATH_UNUSED(eFetch);
// assert (numWeights () == (weightBucket.size () * BUCKET_SIZE));

assert (weightBucket.size () == gradientBucket.size ());
double error = forward_backward(m_layers, settingsAndBatch, 0, numWeights () - 1, 0, numWeights () - 1, 0, outputContainer, true, weightBucket, gradientBucket);
return error;
Expand Down Expand Up @@ -1215,7 +1217,7 @@ template <typename LAYERDATA>

template <typename LayerContainer>
void Net::forwardPattern (const LayerContainer& _layers,
std::vector<LayerData>& layerData, std::vector<double>& weightBucket) const
std::vector<LayerData>& layerData, std::vector<double>& weightBucket, int BUCKET_SIZE) const
{
size_t idxLayer = 0, idxLayerEnd = _layers.size ();
size_t cumulativeNodeCount = 0;
Expand All @@ -1224,7 +1226,7 @@ template <typename LAYERDATA>
LayerData& prevLayerData = layerData.at (idxLayer);
LayerData& currLayerData = layerData.at (idxLayer+1);

forward (prevLayerData, currLayerData, weightBucket, idxLayer);
forward (prevLayerData, currLayerData, weightBucket, idxLayer, BUCKET_SIZE);

applyFunctions (currLayerData.valuesBegin (), currLayerData.valuesEnd (), currLayerData.activationFunction ());
}
Expand Down Expand Up @@ -1264,7 +1266,7 @@ template <typename LAYERDATA>
LayerData& currLayerData = currLayerPatternData.at (idxPattern);


forward (prevLayerData, currLayerData, weightBucket, idxLayer); // feed forward
forward (prevLayerData, currLayerData, weightBucket, idxLayer, m_bucketSize); // feed forward
}

// ---------------- loop over layerDatas of pattern apply non-linearities ----------------------------
Expand Down Expand Up @@ -1384,7 +1386,7 @@ template <typename LAYERDATA>
LayerData& currLayerData = (*itCurrLayerData);
LayerData& prevLayerData = *(itPrevLayerData);

backward (prevLayerData, currLayerData, weightBucket, idxLayer-1);
backward (prevLayerData, currLayerData, weightBucket, idxLayer-1, m_bucketSize);

// the factorWeightDecay has to be scaled by 1/n where n is the number of weights (synapses)
// because L1 and L2 regularization
Expand All @@ -1393,7 +1395,7 @@ template <typename LAYERDATA>
//
// L1 : -factorWeightDecay*sgn(w)/numWeights
// L2 : -factorWeightDecay/numWeights
update (prevLayerData, currLayerData, settings.factorWeightDecay ()/totalNumWeights, settings.regularization (), weightBucket, gradientBucket, idxLayer-1);
update (prevLayerData, currLayerData, settings.factorWeightDecay ()/totalNumWeights, settings.regularization (), weightBucket, gradientBucket, idxLayer-1, m_bucketSize);
}
}
}
Expand Down Expand Up @@ -1471,7 +1473,7 @@ template <typename LAYERDATA>
*
*/
template <typename OutIterator>
void Net::initializeWeights (WeightInitializationStrategy eInitStrategy, OutIterator itWeight, std::vector<int>& layerWeightNumber)
void Net::initializeWeights (WeightInitializationStrategy eInitStrategy, OutIterator itWeight, std::vector<int>& layerWeightNumber, int BUCKET_SIZE)
{
if (eInitStrategy == WeightInitializationStrategy::XAVIER)
{
Expand Down Expand Up @@ -1632,7 +1634,7 @@ template <typename LAYERDATA>
}
if (factorWeightDecay != 0 && eRegularization != EnumRegularization::NONE)
{
error = weightDecay (error, layerData.weightsBegin (), nextLayerData.weightsBegin (), weightBucket, factorWeightDecay, eRegularization, layerNumber);
error = weightDecay (error, layerData.weightsBegin (), nextLayerData.weightsBegin (), weightBucket, factorWeightDecay, eRegularization, layerNumber, m_bucketSize);
}
return error;
}
Expand Down