27 #ifndef TMVA_DNN_GENERALLAYER
28 #define TMVA_DNN_GENERALLAYER
50 template <
typename Architecture_t>
53 using Tensor_t =
typename Architecture_t::Tensor_t;
54 using Matrix_t =
typename Architecture_t::Matrix_t;
55 using Scalar_t =
typename Architecture_t::Scalar_t;
84 VGeneralLayer(
size_t BatchSize,
size_t InputDepth,
size_t InputHeight,
size_t InputWidth,
size_t Depth,
85 size_t Height,
size_t Width,
size_t WeightsNSlices,
size_t WeightsNRows,
size_t WeightsNCols,
86 size_t BiasesNSlices,
size_t BiasesNRows,
size_t BiasesNCols,
size_t OutputNSlices,
size_t OutputNRows,
90 VGeneralLayer(
size_t BatchSize,
size_t InputDepth,
size_t InputHeight,
size_t InputWidth,
size_t Depth,
91 size_t Height,
size_t Width,
size_t WeightsNSlices, std::vector<size_t> WeightsNRows,
92 std::vector<size_t> WeightsNCols,
size_t BiasesNSlices, std::vector<size_t> BiasesNRows,
93 std::vector<size_t> BiasesNCols,
size_t OutputNSlices,
size_t OutputNRows,
size_t OutputNCols,
139 void CopyWeights(
const std::vector<Matrix_t> &otherWeights);
142 void CopyBiases(
const std::vector<Matrix_t> &otherBiases);
147 template <
typename Arch>
238 template <
typename Architecture_t>
240 size_t depth,
size_t height,
size_t width,
size_t weightsNSlices,
241 size_t weightsNRows,
size_t weightsNCols,
size_t biasesNSlices,
242 size_t biasesNRows,
size_t biasesNCols,
size_t outputNSlices,
244 : fBatchSize(batchSize), fInputDepth(inputDepth), fInputHeight(inputHeight), fInputWidth(inputWidth), fDepth(depth),
245 fHeight(height), fWidth(
width), fIsTraining(true), fWeights(), fBiases(), fWeightGradients(), fBiasGradients(),
246 fOutput( outputNSlices, outputNRows, outputNCols ),
247 fActivationGradients( outputNSlices, outputNRows, outputNCols ),
251 for (
size_t i = 0; i < weightsNSlices; i++) {
252 fWeights.emplace_back(weightsNRows, weightsNCols);
256 for (
size_t i = 0; i < biasesNSlices; i++) {
257 fBiases.emplace_back(biasesNRows, biasesNCols);
263 template <
typename Architecture_t>
265 size_t depth,
size_t height,
size_t width,
size_t weightsNSlices,
266 std::vector<size_t> weightsNRows, std::vector<size_t> weightsNCols,
267 size_t biasesNSlices, std::vector<size_t> biasesNRows,
268 std::vector<size_t> biasesNCols,
size_t outputNSlices,
size_t outputNRows,
270 : fBatchSize(batchSize), fInputDepth(inputDepth), fInputHeight(inputHeight), fInputWidth(inputWidth), fDepth(depth),
271 fHeight(height), fWidth(
width), fIsTraining(true), fWeights(), fBiases(), fWeightGradients(), fBiasGradients(),
272 fOutput( outputNSlices, outputNRows, outputNCols ),
273 fActivationGradients( outputNSlices, outputNRows, outputNCols ),
277 for (
size_t i = 0; i < weightsNSlices; i++) {
278 fWeights.emplace_back(weightsNRows[i], weightsNCols[i]);
282 for (
size_t i = 0; i < biasesNSlices; i++) {
283 fBiases.emplace_back(biasesNRows[i], biasesNCols[i]);
294 template <
typename Architecture_t>
296 : fBatchSize(layer->GetBatchSize()), fInputDepth(layer->GetInputDepth()), fInputHeight(layer->GetInputHeight()),
297 fInputWidth(layer->GetInputWidth()), fDepth(layer->GetDepth()), fHeight(layer->GetHeight()),
298 fWidth(layer->GetWidth()), fIsTraining(layer->IsTraining()), fWeights(), fBiases(), fWeightGradients(),
300 fOutput( layer->GetOutput().GetShape() ),
301 fActivationGradients( layer->GetActivationGradients().GetShape() ),
302 fInit(layer->GetInitialization() )
305 size_t weightsNSlices = (layer->
GetWeights()).size();
306 size_t weightsNRows = 0;
307 size_t weightsNCols = 0;
309 for (
size_t i = 0; i < weightsNSlices; i++) {
313 fWeights.emplace_back(weightsNRows, weightsNCols);
319 size_t biasesNSlices = (layer->
GetBiases()).size();
320 size_t biasesNRows = 0;
321 size_t biasesNCols = 0;
323 for (
size_t i = 0; i < biasesNSlices; i++) {
327 fBiases.emplace_back(biasesNRows, biasesNCols);
335 template <
typename Architecture_t>
337 : fBatchSize(layer.fBatchSize), fInputDepth(layer.fInputDepth), fInputHeight(layer.fInputHeight),
338 fInputWidth(layer.fInputWidth), fDepth(layer.fDepth), fHeight(layer.fHeight), fWidth(layer.fWidth),
339 fIsTraining(layer.fIsTraining), fWeights(), fBiases(), fWeightGradients(), fBiasGradients(),
340 fOutput( layer.GetOutput() ),
341 fActivationGradients( layer.GetActivationGradients() ),
342 fInit( layer.GetInitialization())
345 size_t weightsNSlices = layer.
fWeights.size();
346 size_t weightsNRows = 0;
347 size_t weightsNCols = 0;
349 for (
size_t i = 0; i < weightsNSlices; i++) {
350 weightsNRows = (layer.
fWeights[i]).GetNrows();
351 weightsNCols = (layer.
fWeights[i]).GetNcols();
353 fWeights.emplace_back(weightsNRows, weightsNCols);
359 size_t biasesNSlices = layer.
fBiases.size();
360 size_t biasesNRows = 0;
361 size_t biasesNCols = 0;
363 for (
size_t i = 0; i < biasesNSlices; i++) {
364 biasesNRows = (layer.
fBiases[i]).GetNrows();
365 biasesNCols = (layer.
fBiases[i]).GetNcols();
367 fBiases.emplace_back(biasesNRows, biasesNCols);
373 size_t outputNSlices = layer.
fOutput.size();
374 size_t outputNRows = 0;
375 size_t outputNCols = 0;
377 for (
size_t i = 0; i < outputNSlices; i++) {
378 outputNRows = (layer.
fOutput[i]).GetNrows();
379 outputNCols = (layer.
fOutput[i]).GetNcols();
381 fOutput.emplace_back(outputNRows, outputNCols);
387 template <
typename Architecture_t>
394 template <
typename Architecture_t>
397 for (
size_t i = 0; i < fWeights.size(); i++) {
398 initialize<Architecture_t>(fWeights[i], this->GetInitialization());
402 for (
size_t i = 0; i < fBiases.size(); i++) {
409 template <
typename Architecture_t>
412 this->UpdateWeights(fWeightGradients, learningRate);
413 this->UpdateBiases(fBiasGradients, learningRate);
417 template <
typename Architecture_t>
419 const Scalar_t learningRate) ->
void
421 for (
size_t i = 0; i < fWeights.size(); i++) {
422 Architecture_t::ScaleAdd(fWeights[i], weightGradients[i], -learningRate);
427 template <
typename Architecture_t>
429 const Scalar_t learningRate) ->
void
431 for (
size_t i = 0; i < fBiases.size(); i++) {
432 Architecture_t::ScaleAdd(fBiases[i], biasGradients[i], -learningRate);
437 template <
typename Architecture_t>
439 const Scalar_t learningRate) ->
void
441 for (
size_t i = 0; i < fWeightGradients.size(); i++) {
442 Architecture_t::ScaleAdd(fWeightGradients[i], weightGradients[i], -learningRate);
447 template <
typename Architecture_t>
449 const Scalar_t learningRate) ->
void
451 for (
size_t i = 0; i < fBiasGradients.size(); i++) {
452 Architecture_t::ScaleAdd(fBiasGradients[i], biasGradients[i], -learningRate);
457 template <
typename Architecture_t>
461 for (
size_t i = 0; i < fWeights.size(); i++) {
467 template <
typename Architecture_t>
470 for (
size_t i = 0; i < fBiases.size(); i++) {
476 template <
typename Architecture_t>
477 template <
typename Arch>
482 Architecture_t::CopyDiffArch(this->GetWeights(), layer.
GetWeights());
483 Architecture_t::CopyDiffArch(this->GetBiases(), layer.
GetBiases());
487 if (params.size() > 0) {
488 auto paramsToCopy = GetExtraLayerParameters();
489 Architecture_t::CopyDiffArch(paramsToCopy, params );
490 SetExtraLayerParameters(paramsToCopy);
495 template <
typename Architecture_t>
500 if (tensor.size() == 0)
return;
501 xmlengine.NewAttr(matnode,0,
"Depth",
gTools().StringFromInt(tensor.size()) );
503 xmlengine.NewAttr(matnode,0,
"Rows",
gTools().StringFromInt(tensor[0].GetNrows()) );
504 xmlengine.NewAttr(matnode,0,
"Columns",
gTools().StringFromInt(tensor[0].GetNcols()) );
506 for (
size_t i = 0; i < tensor.size(); ++i) {
507 auto & mat = tensor[i];
508 for (
Int_t row = 0; row < mat.GetNrows(); row++) {
509 for (
Int_t col = 0; col < mat.GetNcols(); col++) {
512 s << std::scientific << mat(row, col) <<
" ";
516 xmlengine.AddRawLine( matnode,
s.str().c_str() );
520 template <
typename Architecture_t>
526 xmlengine.NewAttr(matnode,0,
"Rows",
gTools().StringFromInt(matrix.GetNrows()) );
527 xmlengine.NewAttr(matnode,0,
"Columns",
gTools().StringFromInt(matrix.GetNcols()) );
529 s.precision( std::numeric_limits<Scalar_t>::digits10 );
530 size_t nrows = matrix.GetNrows();
531 size_t ncols = matrix.GetNcols();
532 for (
size_t row = 0; row < nrows; row++) {
533 for (
size_t col = 0; col < ncols; col++) {
535 s << std::scientific << matrix(row,col) <<
" ";
539 xmlengine.AddRawLine( matnode,
s.str().c_str() );
543 template <
typename Architecture_t>
551 R__ASSERT((
size_t) matrix.GetNrows() == rows);
552 R__ASSERT((
size_t) matrix.GetNcols() == cols);
557 std::stringstream matrixStringStream(matrixString);
559 for (
size_t i = 0; i < rows; i++)
561 for (
size_t j = 0; j < cols; j++)
563 #ifndef R__HAS_TMVAGPU
564 matrixStringStream >> tmatrix(i,j);
567 matrixStringStream >> value;
568 tmatrix(i,j) = value;
581 template <
typename Architecture>
582 auto debugTensor(
const typename Architecture::Tensor_t &
A,
const std::string
name =
"tensor") ->
void
584 Architecture::PrintTensor(
A,
name);