{ "cells": [ { "cell_type": "markdown", "id": "98d1e66e", "metadata": {}, "source": [ "# TMVA_RNN_Classification\n", " TMVA Classification Example Using a Recurrent Neural Network\n", "\n", "This is an example of using a RNN in TMVA. We do classification using a toy time dependent data set\n", "that is generated when running this example macro\n", "\n", "\n", "\n", "\n", "**Author:** Lorenzo Moneta \n", "This notebook tutorial was automatically generated with ROOTBOOK-izer from the macro found in the ROOT repository on Tuesday, May 19, 2026 at 08:23 PM." ] }, { "cell_type": "code", "execution_count": null, "id": "96d33342", "metadata": { "collapsed": false }, "outputs": [], "source": [ "/***\n", "\n", " # TMVA Classification Example Using a Recurrent Neural Network\n", "\n", " This is an example of using a RNN in TMVA.\n", " We do the classification using a toy data set containing a time series of data sample ntimes\n", " and with dimension ndim that is generated when running the provided function `MakeTimeData (nevents, ntime, ndim)`\n", "\n", "\n", "**/\n", "\n", "#include\n", "\n", "#include \"TMVA/Factory.h\"\n", "#include \"TMVA/DataLoader.h\"\n", "#include \"TMVA/DataSetInfo.h\"\n", "#include \"TMVA/Config.h\"\n", "#include \"TMVA/MethodDL.h\"\n", "\n", "\n", "#include \"TFile.h\"\n", "#include \"TTree.h\"" ] }, { "cell_type": "markdown", "id": "a42daa80", "metadata": {}, "source": [ " Helper function to generate the time data set\n", "make some time data but not of fixed length.\n", "use a poisson with mu = 5 and truncated at 10\n", "\n", "\n", " " ] }, { "cell_type": "code", "execution_count": null, "id": "c5b3792b", "metadata": { "collapsed": false }, "outputs": [], "source": [ "%%cpp -d\n", "void MakeTimeData(int n, int ntime, int ndim )\n", "{\n", "\n", " // const int ntime = 10;\n", " // const int ndim = 30; // number of dim/time\n", " TString fname = TString::Format(\"time_data_t%d_d%d.root\", ntime, ndim);\n", " std::vector v1(ntime);\n", " std::vector v2(ntime);\n", " int i = 0;\n", " for (int i = 0; i < ntime; ++i) {\n", " v1[i] = new TH1D(TString::Format(\"h1_%d\", i), \"h1\", ndim, 0, 10);\n", " v2[i] = new TH1D(TString::Format(\"h2_%d\", i), \"h2\", ndim, 0, 10);\n", " }\n", "\n", " auto f1 = new TF1(\"f1\", \"gaus\");\n", " auto f2 = new TF1(\"f2\", \"gaus\");\n", "\n", " TFile f(fname, \"RECREATE\");\n", " TTree sgn(\"sgn\", \"sgn\");\n", " TTree bkg(\"bkg\", \"bkg\");\n", "\n", " std::vector> x1(ntime);\n", " std::vector> x2(ntime);\n", "\n", " for (int i = 0; i < ntime; ++i) {\n", " x1[i] = std::vector(ndim);\n", " x2[i] = std::vector(ndim);\n", " }\n", "\n", " for (auto i = 0; i < ntime; i++) {\n", " bkg.Branch(Form(\"vars_time%d\", i), \"std::vector\", &x1[i]);\n", " sgn.Branch(Form(\"vars_time%d\", i), \"std::vector\", &x2[i]);\n", " }\n", "\n", " sgn.SetDirectory(&f);\n", " bkg.SetDirectory(&f);\n", " gRandom->SetSeed(0);\n", "\n", " std::vector mean1(ntime);\n", " std::vector mean2(ntime);\n", " std::vector sigma1(ntime);\n", " std::vector sigma2(ntime);\n", " for (int j = 0; j < ntime; ++j) {\n", " mean1[j] = 5. + 0.2 * sin(TMath::Pi() * j / double(ntime));\n", " mean2[j] = 5. + 0.2 * cos(TMath::Pi() * j / double(ntime));\n", " sigma1[j] = 4 + 0.3 * sin(TMath::Pi() * j / double(ntime));\n", " sigma2[j] = 4 + 0.3 * cos(TMath::Pi() * j / double(ntime));\n", " }\n", " for (int i = 0; i < n; ++i) {\n", "\n", " if (i % 1000 == 0)\n", " std::cout << \"Generating event ... \" << i << std::endl;\n", "\n", " for (int j = 0; j < ntime; ++j) {\n", " auto h1 = v1[j];\n", " auto h2 = v2[j];\n", " h1->Reset();\n", " h2->Reset();\n", "\n", " f1->SetParameters(1, mean1[j], sigma1[j]);\n", " f2->SetParameters(1, mean2[j], sigma2[j]);\n", "\n", " h1->FillRandom(\"f1\", 1000);\n", " h2->FillRandom(\"f2\", 1000);\n", "\n", " for (int k = 0; k < ndim; ++k) {\n", " // std::cout << j*10+k << \" \";\n", " x1[j][k] = h1->GetBinContent(k + 1) + gRandom->Gaus(0, 10);\n", " x2[j][k] = h2->GetBinContent(k + 1) + gRandom->Gaus(0, 10);\n", " }\n", " }\n", " // std::cout << std::endl;\n", " sgn.Fill();\n", " bkg.Fill();\n", "\n", " if (n == 1) {\n", " auto c1 = new TCanvas();\n", " c1->Divide(ntime, 2);\n", " for (int j = 0; j < ntime; ++j) {\n", " c1->cd(j + 1);\n", " v1[j]->Draw();\n", " }\n", " for (int j = 0; j < ntime; ++j) {\n", " c1->cd(ntime + j + 1);\n", " v2[j]->Draw();\n", " }\n", " gPad->Update();\n", " }\n", " }\n", " if (n > 1) {\n", " sgn.Write();\n", " bkg.Write();\n", " sgn.Print();\n", " bkg.Print();\n", " f.Close();\n", " }\n", "}" ] }, { "cell_type": "markdown", "id": "336d2ef3", "metadata": {}, "source": [ " Arguments are defined. " ] }, { "cell_type": "code", "execution_count": null, "id": "3f55e48b", "metadata": { "collapsed": false }, "outputs": [], "source": [ "int nevts = 2000;\n", "int use_type = 1;" ] }, { "cell_type": "code", "execution_count": null, "id": "5bb67c2d", "metadata": { "collapsed": false }, "outputs": [], "source": [ " const int ninput = 30;\n", " const int ntime = 10;\n", " const int batchSize = 100;\n", " const int maxepochs = 20;\n", "\n", " int nTotEvts = nevts; // total events to be generated for signal or background\n", "\n", " bool useKeras = true;\n", "\n", "\n", " bool useTMVA_RNN = true;\n", " bool useTMVA_DNN = true;\n", " bool useTMVA_BDT = false;\n", "\n", " std::vector rnn_types = {\"RNN\", \"LSTM\", \"GRU\"};\n", " std::vector use_rnn_type = {1, 1, 1};\n", " if (use_type >=0 && use_type < 3) {\n", " use_rnn_type = {0,0,0};\n", " use_rnn_type[use_type] = 1;\n", " }\n", " bool useGPU = true; // use GPU for TMVA if available\n", "\n", "#ifndef R__HAS_TMVAGPU\n", " useGPU = false;\n", "#ifndef R__HAS_TMVACPU\n", " Warning(\"TMVA_RNN_Classification\", \"TMVA is not build with GPU or CPU multi-thread support. Cannot use TMVA Deep Learning for RNN\");\n", " useTMVA_RNN = false;\n", "#endif\n", "#endif\n", "\n", "\n", " TString archString = (useGPU) ? \"GPU\" : \"CPU\";\n", "\n", " bool writeOutputFile = true;\n", "\n", "\n", "\n", " const char *rnn_type = \"RNN\";\n", "\n", "#ifdef R__HAS_PYMVA\n", " TMVA::PyMethodBase::PyInitialize();\n", "#else\n", " useKeras = false;\n", "#endif\n", "\n", "#ifdef R__USE_IMT\n", " int num_threads = 4; // use max 4 threads" ] }, { "cell_type": "markdown", "id": "a56362b1", "metadata": {}, "source": [ "switch off MT in OpenBLAS to avoid conflict with tbb" ] }, { "cell_type": "code", "execution_count": null, "id": "95ba377b", "metadata": { "collapsed": false }, "outputs": [], "source": [ " gSystem->Setenv(\"OMP_NUM_THREADS\", \"1\");" ] }, { "cell_type": "markdown", "id": "56b8a47d", "metadata": {}, "source": [ "do enable MT running" ] }, { "cell_type": "code", "execution_count": null, "id": "9e2fda10", "metadata": { "collapsed": false }, "outputs": [], "source": [ " if (num_threads >= 0) {\n", " ROOT::EnableImplicitMT(num_threads);\n", " }\n", "#endif\n", "\n", " TMVA::Config::Instance();\n", "\n", " std::cout << \"Running with nthreads = \" << ROOT::GetThreadPoolSize() << std::endl;\n", "\n", " TString inputFileName = \"time_data_t10_d30.root\";\n", "\n", " bool fileExist = !gSystem->AccessPathName(inputFileName);" ] }, { "cell_type": "markdown", "id": "5ad72680", "metadata": {}, "source": [ "if file does not exists create it" ] }, { "cell_type": "code", "execution_count": null, "id": "f51afbe8", "metadata": { "collapsed": false }, "outputs": [], "source": [ " if (!fileExist) {\n", " MakeTimeData(nTotEvts,ntime, ninput);\n", " }\n", "\n", "\n", " auto inputFile = TFile::Open(inputFileName);\n", " if (!inputFile) {\n", " Error(\"TMVA_RNN_Classification\", \"Error opening input file %s - exit\", inputFileName.Data());\n", " return;\n", " }\n", "\n", "\n", " std::cout << \"--- RNNClassification : Using input file: \" << inputFile->GetName() << std::endl;" ] }, { "cell_type": "markdown", "id": "ba82dac8", "metadata": {}, "source": [ "Create a ROOT output file where TMVA will store ntuples, histograms, etc." ] }, { "cell_type": "code", "execution_count": null, "id": "5a40f53f", "metadata": { "collapsed": false }, "outputs": [], "source": [ " TString outfileName(TString::Format(\"data_RNN_%s.root\", archString.Data()));\n", " TFile *outputFile = nullptr;\n", " if (writeOutputFile) outputFile = TFile::Open(outfileName, \"RECREATE\");\n", "\n", " /**\n", " ## Declare Factory\n", "\n", " Create the Factory class. Later you can choose the methods\n", " whose performance you'd like to investigate.\n", "\n", " The factory is the major TMVA object you have to interact with. Here is the list of parameters you need to\n", "pass\n", "\n", " - The first argument is the base of the name of all the output\n", " weightfiles in the directory weight/ that will be created with the\n", " method parameters\n", "\n", " - The second argument is the output file for the training results\n", "\n", " - The third argument is a string option defining some general configuration for the TMVA session.\n", " For example all TMVA output can be suppressed by removing the \"!\" (not) in front of the \"Silent\" argument in\n", "the option string\n", "\n", " **/" ] }, { "cell_type": "markdown", "id": "c447c79a", "metadata": {}, "source": [ "Creating the factory object" ] }, { "cell_type": "code", "execution_count": null, "id": "7243fde2", "metadata": { "collapsed": false }, "outputs": [], "source": [ " TMVA::Factory *factory = new TMVA::Factory(\"TMVAClassification\", outputFile,\n", " \"!V:!Silent:Color:!DrawProgressBar:Transformations=None:!Correlations:\"\n", " \"AnalysisType=Classification:ModelPersistence\");\n", " TMVA::DataLoader *dataloader = new TMVA::DataLoader(\"dataset\");\n", "\n", " TTree *signalTree = (TTree *)inputFile->Get(\"sgn\");\n", " TTree *background = (TTree *)inputFile->Get(\"bkg\");\n", "\n", " const int nvar = ninput * ntime;" ] }, { "cell_type": "markdown", "id": "abd0f887", "metadata": {}, "source": [ "add variables - use new AddVariablesArray function" ] }, { "cell_type": "code", "execution_count": null, "id": "c15c35d5", "metadata": { "collapsed": false }, "outputs": [], "source": [ " for (auto i = 0; i < ntime; i++) {\n", " dataloader->AddVariablesArray(Form(\"vars_time%d\", i), ninput);\n", " }\n", "\n", " dataloader->AddSignalTree(signalTree, 1.0);\n", " dataloader->AddBackgroundTree(background, 1.0);" ] }, { "cell_type": "markdown", "id": "ec327ffa", "metadata": {}, "source": [ "check given input" ] }, { "cell_type": "code", "execution_count": null, "id": "fbc6972e", "metadata": { "collapsed": false }, "outputs": [], "source": [ " auto &datainfo = dataloader->GetDataSetInfo();\n", " auto vars = datainfo.GetListOfVariables();\n", " std::cout << \"number of variables is \" << vars.size() << std::endl;\n", " for (auto &v : vars)\n", " std::cout << v << \",\";\n", " std::cout << std::endl;\n", "\n", " int nTrainSig = 0.8 * nTotEvts;\n", " int nTrainBkg = 0.8 * nTotEvts;" ] }, { "cell_type": "markdown", "id": "a2dbfbba", "metadata": {}, "source": [ "build the string options for DataLoader::PrepareTrainingAndTestTree" ] }, { "cell_type": "code", "execution_count": null, "id": "b88606b0", "metadata": { "collapsed": false }, "outputs": [], "source": [ " TString prepareOptions = TString::Format(\"nTrain_Signal=%d:nTrain_Background=%d:SplitMode=Random:SplitSeed=100:NormMode=NumEvents:!V:!CalcCorrelations\", nTrainSig, nTrainBkg);" ] }, { "cell_type": "markdown", "id": "9d35e136", "metadata": {}, "source": [ "Apply additional cuts on the signal and background samples (can be different)" ] }, { "cell_type": "code", "execution_count": null, "id": "a8382be0", "metadata": { "collapsed": false }, "outputs": [], "source": [ " TCut mycuts = \"\"; // for example: TCut mycuts = \"abs(var1)<0.5 && abs(var2-0.5)<1\";\n", " TCut mycutb = \"\";\n", "\n", " dataloader->PrepareTrainingAndTestTree(mycuts, mycutb, prepareOptions);\n", "\n", " std::cout << \"prepared DATA LOADER \" << std::endl;\n", "\n", " /**\n", " ## Book TMVA recurrent models\n", "\n", " Book the different types of recurrent models in TMVA (SimpleRNN, LSTM or GRU)\n", "\n", "**/\n", "\n", " if (useTMVA_RNN) {\n", "\n", " for (int i = 0; i < 3; ++i) {\n", "\n", " if (!use_rnn_type[i])\n", " continue;\n", "\n", " const char *rnn_type = rnn_types[i].c_str();\n", "\n", " /// define the inputlayout string for RNN\n", " /// the input data should be organize as following:\n", " //// input layout for RNN: time x ndim\n", "\n", " TString inputLayoutString = TString::Format(\"InputLayout=%d|%d\", ntime, ninput);\n", "\n", " /// Define RNN layer layout\n", " /// it should be LayerType (RNN or LSTM or GRU) | number of units | number of inputs | time steps | remember output (typically no=0 | return full sequence\n", " TString rnnLayout = TString::Format(\"%s|10|%d|%d|0|1\", rnn_type, ninput, ntime);\n", "\n", " /// add after RNN a reshape layer (needed top flatten the output) and a dense layer with 64 units and a last one\n", " /// Note the last layer is linear because when using Crossentropy a Sigmoid is applied already\n", " TString layoutString = TString(\"Layout=\") + rnnLayout + TString(\",RESHAPE|FLAT,DENSE|64|TANH,LINEAR\");\n", "\n", " /// Defining Training strategies. Different training strings can be concatenate. Use however only one\n", " TString trainingString1 = TString::Format(\"LearningRate=1e-3,Momentum=0.0,Repetitions=1,\"\n", " \"ConvergenceSteps=5,BatchSize=%d,TestRepetitions=1,\"\n", " \"WeightDecay=1e-2,Regularization=None,MaxEpochs=%d,\"\n", " \"Optimizer=ADAM,DropConfig=0.0+0.+0.+0.\",\n", " batchSize,maxepochs);\n", "\n", " TString trainingStrategyString(\"TrainingStrategy=\");\n", " trainingStrategyString += trainingString1; // + \"|\" + trainingString2\n", "\n", " /// Define the full RNN Noption string adding the final options for all network\n", " TString rnnOptions(\"!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=None:\"\n", " \"WeightInitialization=XAVIERUNIFORM:ValidationSize=0.2:RandomSeed=1234\");\n", "\n", " rnnOptions.Append(\":\");\n", " rnnOptions.Append(inputLayoutString);\n", " rnnOptions.Append(\":\");\n", " rnnOptions.Append(layoutString);\n", " rnnOptions.Append(\":\");\n", " rnnOptions.Append(trainingStrategyString);\n", " rnnOptions.Append(\":\");\n", " rnnOptions.Append(TString::Format(\"Architecture=%s\", archString.Data()));\n", "\n", " TString rnnName = \"TMVA_\" + TString(rnn_type);\n", " factory->BookMethod(dataloader, TMVA::Types::kDL, rnnName, rnnOptions);\n", "\n", " }\n", " }\n", "\n", " /**\n", " ## Book TMVA fully connected dense layer models\n", "\n", " **/\n", "\n", " if (useTMVA_DNN) {\n", " // Method DL with Dense Layer\n", " TString inputLayoutString = TString::Format(\"InputLayout=1|1|%d\", ntime * ninput);\n", "\n", " TString layoutString(\"Layout=DENSE|64|TANH,DENSE|TANH|64,DENSE|TANH|64,LINEAR\");\n", " // Training strategies.\n", " TString trainingString1(\"LearningRate=1e-3,Momentum=0.0,Repetitions=1,\"\n", " \"ConvergenceSteps=10,BatchSize=256,TestRepetitions=1,\"\n", " \"WeightDecay=1e-4,Regularization=None,MaxEpochs=20\"\n", " \"DropConfig=0.0+0.+0.+0.,Optimizer=ADAM\");\n", " TString trainingStrategyString(\"TrainingStrategy=\");\n", " trainingStrategyString += trainingString1; // + \"|\" + trainingString2\n", "\n", " // General Options.\n", " TString dnnOptions(\"!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=None:\"\n", " \"WeightInitialization=XAVIER:RandomSeed=0\");\n", "\n", " dnnOptions.Append(\":\");\n", " dnnOptions.Append(inputLayoutString);\n", " dnnOptions.Append(\":\");\n", " dnnOptions.Append(layoutString);\n", " dnnOptions.Append(\":\");\n", " dnnOptions.Append(trainingStrategyString);\n", " dnnOptions.Append(\":\");\n", " dnnOptions.Append(archString);\n", "\n", " TString dnnName = \"TMVA_DNN\";\n", " factory->BookMethod(dataloader, TMVA::Types::kDL, dnnName, dnnOptions);\n", " }\n", "\n", " /**\n", " ## Book Keras recurrent models\n", "\n", " Book the different types of recurrent models in Keras (SimpleRNN, LSTM or GRU)\n", "\n", " **/\n", "\n", " if (useKeras) {\n", "\n", " for (int i = 0; i < 3; i++) {\n", "\n", " if (use_rnn_type[i]) {\n", "\n", " TString modelName = TString::Format(\"model_%s.keras\", rnn_types[i].c_str());\n", " TString trainedModelName = TString::Format(\"trained_model_%s.keras\", rnn_types[i].c_str());\n", "\n", " Info(\"TMVA_RNN_Classification\", \"Building recurrent keras model using a %s layer\", rnn_types[i].c_str());\n", " // create python script which can be executed\n", " // create 2 conv2d layer + maxpool + dense\n", " TMacro m;\n", " m.AddLine(\"import tensorflow\");\n", " m.AddLine(\"from tensorflow.keras.models import Sequential\");\n", " m.AddLine(\"from tensorflow.keras.optimizers import Adam\");\n", " m.AddLine(\"from tensorflow.keras.layers import Input, Dense, Dropout, Flatten, SimpleRNN, GRU, LSTM, Reshape, \"\n", " \"BatchNormalization\");\n", " m.AddLine(\"\");\n", " m.AddLine(\"model = Sequential() \");\n", " m.AddLine(\"model.add(Reshape((10, 30), input_shape = (10*30, )))\");\n", " // add recurrent neural network depending on type / Use option to return the full output\n", " if (rnn_types[i] == \"LSTM\")\n", " m.AddLine(\"model.add(LSTM(units=10, return_sequences=True) )\");\n", " else if (rnn_types[i] == \"GRU\")\n", " m.AddLine(\"model.add(GRU(units=10, return_sequences=True) )\");\n", " else\n", " m.AddLine(\"model.add(SimpleRNN(units=10, return_sequences=True) )\");\n", "\n", " // m.AddLine(\"model.add(BatchNormalization())\");\n", " m.AddLine(\"model.add(Flatten())\"); // needed if returning the full time output sequence\n", " m.AddLine(\"model.add(Dense(64, activation = 'tanh')) \");\n", " m.AddLine(\"model.add(Dense(2, activation = 'sigmoid')) \");\n", " m.AddLine(\n", " \"model.compile(loss = 'binary_crossentropy', optimizer = Adam(learning_rate = 0.001), weighted_metrics = ['accuracy'])\");\n", " m.AddLine(TString::Format(\"modelName = '%s'\", modelName.Data()));\n", " m.AddLine(\"model.save(modelName)\");\n", " m.AddLine(\"model.summary()\");\n", "\n", " m.SaveSource(\"make_rnn_model.py\");\n", " // execute python script to make the model\n", " auto ret = (TString *)gROOT->ProcessLine(\"TMVA::Python_Executable()\");\n", " TString python_exe = (ret) ? *(ret) : \"python\";\n", " gSystem->Exec(python_exe + \" make_rnn_model.py\");\n", "\n", " if (gSystem->AccessPathName(modelName)) {\n", " Warning(\"TMVA_RNN_Classification\", \"Error creating Keras recurrent model file - Skip using Keras\");\n", " useKeras = false;\n", " } else {\n", " // book PyKeras method only if Keras model could be created\n", " Info(\"TMVA_RNN_Classification\", \"Booking Keras %s model\", rnn_types[i].c_str());\n", " factory->BookMethod(dataloader, TMVA::Types::kPyKeras,\n", " TString::Format(\"PyKeras_%s\", rnn_types[i].c_str()),\n", " TString::Format(\"!H:!V:VarTransform=None:FilenameModel=%s:tf.keras:\"\n", " \"FilenameTrainedModel=%s:NumEpochs=%d:BatchSize=%d\",\n", " modelName.Data(), trainedModelName.Data(), maxepochs, batchSize));\n", " }\n", " }\n", " }\n", " }\n", "\n", " // use BDT in case not using Keras or TMVA DL\n", " if (!useKeras || !useTMVA_BDT)\n", " useTMVA_BDT = true;\n", "\n", " /**\n", " ## Book TMVA BDT\n", " **/\n", "\n", " if (useTMVA_BDT) {\n", "\n", " factory->BookMethod(dataloader, TMVA::Types::kBDT, \"BDTG\",\n", " \"!H:!V:NTrees=100:MinNodeSize=2.5%:BoostType=Grad:Shrinkage=0.10:UseBaggedBoost:\"\n", " \"BaggedSampleFraction=0.5:nCuts=20:\"\n", " \"MaxDepth=2\");\n", "\n", " }\n", "\n", " /// Train all methods\n", " factory->TrainAllMethods();\n", "\n", " std::cout << \"nthreads = \" << ROOT::GetThreadPoolSize() << std::endl;\n", "\n", " // ---- Evaluate all MVAs using the set of test events\n", " factory->TestAllMethods();\n", "\n", " // ----- Evaluate and compare performance of all configured MVAs\n", " factory->EvaluateAllMethods();\n", "\n", " // check method\n", "\n", " // plot ROC curve\n", " auto c1 = factory->GetROCCurve(dataloader);\n", " c1->Draw();\n", "\n", " if (outputFile) outputFile->Close();" ] }, { "cell_type": "markdown", "id": "e0c7f7cc", "metadata": {}, "source": [ "Draw all canvases " ] }, { "cell_type": "code", "execution_count": null, "id": "83a6d1b4", "metadata": { "collapsed": false }, "outputs": [], "source": [ "gROOT->GetListOfCanvases()->Draw()" ] } ], "metadata": { "kernelspec": { "display_name": "ROOT C++", "language": "c++", "name": "root" }, "language_info": { "codemirror_mode": "text/x-c++src", "file_extension": ".C", "mimetype": " text/x-c++src", "name": "c++" } }, "nbformat": 4, "nbformat_minor": 5 }