RNTupleInspector.cxx

Go to the documentation of this file.

/// \file RNTupleInspector.cxx
/// \author Florine de Geus <florine.willemijn.de.geus@cern.ch>
/// \date 2023-01-09
/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
/// is welcome!
 
/*************************************************************************
 * Copyright (C) 1995-2023, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/
 
#include <ROOT/RColumnElementBase.hxx>
#include <ROOT/RError.hxx>
#include <ROOT/RPageStorageFile.hxx>
#include <ROOT/RNTupleDescriptor.hxx>
#include <ROOT/RNTupleInspector.hxx>
#include "ROOT/RNTupleUtils.hxx"
 
#include <TFile.h>
 
#include <algorithm>
#include <cstring>
#include <deque>
#include <exception>
#include <functional>
#include <iomanip>
#include <iostream>
 
using ROOT::Internal::RColumnElementBase;
 
ROOT::Experimental::RNTupleInspector::RNTupleInspector(std::unique_ptr<ROOT::Internal::RPageSource> pageSource)
   : fPageSource(std::move(pageSource))
{
   fPageSource->Attach();
   fPageSource->LoadAllPageLists();
   auto descriptorGuard = fPageSource->GetSharedDescriptorGuard();
   fDescriptor = descriptorGuard->Clone();
 
   CollectColumnInfo();
   CollectFieldTreeInfo(fDescriptor.GetFieldZeroId());
}
 
// NOTE: outlined to avoid including RPageStorage in the header
ROOT::Experimental::RNTupleInspector::~RNTupleInspector() = default;
 
void ROOT::Experimental::RNTupleInspector::CollectColumnInfo()
{
   fCompressedSize = 0;
   fUncompressedSize = 0;
 
   std::vector<DescriptorId_t> clusterIds;
   for (const auto &cgDesc : fDescriptor.GetClusterGroupIterable()) {
      R__ASSERT(cgDesc.HasClusterDetails());
      clusterIds.insert(clusterIds.end(), cgDesc.GetClusterIds().begin(), cgDesc.GetClusterIds().end());
   }
 
   for (const auto &colDesc : fDescriptor.GetColumnIterable()) {
      if (colDesc.IsAliasColumn())
         continue;
 
      auto colId = colDesc.GetPhysicalId();
 
      // We generate the default memory representation for the given column type in order
      // to report the size _in memory_ of column elements.
      std::uint32_t elemSize = RColumnElementBase::Generate(colDesc.GetType())->GetSize();
      std::uint64_t nElems = 0;
      std::vector<std::uint64_t> compressedPageSizes{};
 
      for (auto cid : clusterIds) {
         const auto &clusterDescriptor = fDescriptor.GetClusterDescriptor(cid);
         if (!clusterDescriptor.ContainsColumn(colId)) {
            continue;
         }
 
         auto columnRange = clusterDescriptor.GetColumnRange(colId);
         if (columnRange.IsSuppressed())
            continue;
 
         nElems += columnRange.GetNElements();
 
         if (!fCompressionSettings && columnRange.GetCompressionSettings()) {
            fCompressionSettings = columnRange.GetCompressionSettings();
         } else if (fCompressionSettings && columnRange.GetCompressionSettings() &&
                    (*fCompressionSettings != *columnRange.GetCompressionSettings())) {
            // Note that currently all clusters and columns are compressed with the same settings and it is not yet
            // possible to do otherwise. This means that currently, this exception should never be thrown, but this
            // could change in the future.
            throw RException(R__FAIL("compression setting mismatch between column ranges (" +
                                     std::to_string(*fCompressionSettings) + " vs " +
                                     std::to_string(*columnRange.GetCompressionSettings()) +
                                     ") for column with physical ID " + std::to_string(colId)));
         }
 
         const auto &pageRange = clusterDescriptor.GetPageRange(colId);
 
         for (const auto &page : pageRange.GetPageInfos()) {
            compressedPageSizes.emplace_back(page.GetLocator().GetNBytesOnStorage());
            fUncompressedSize += page.GetNElements() * elemSize;
         }
      }
 
      fCompressedSize +=
         std::accumulate(compressedPageSizes.begin(), compressedPageSizes.end(), static_cast<std::uint64_t>(0));
      fColumnInfo.emplace(colId, RColumnInspector(colDesc, compressedPageSizes, elemSize, nElems));
   }
}
 
ROOT::Experimental::RNTupleInspector::RFieldTreeInspector
ROOT::Experimental::RNTupleInspector::CollectFieldTreeInfo(ROOT::DescriptorId_t fieldId)
{
   std::uint64_t compressedSize = 0;
   std::uint64_t uncompressedSize = 0;
 
   for (const auto &colDescriptor : fDescriptor.GetColumnIterable(fieldId)) {
      auto colInfo = GetColumnInspector(colDescriptor.GetPhysicalId());
      compressedSize += colInfo.GetCompressedSize();
      uncompressedSize += colInfo.GetUncompressedSize();
   }
 
   for (const auto &subFieldDescriptor : fDescriptor.GetFieldIterable(fieldId)) {
      auto subFieldId = subFieldDescriptor.GetId();
 
      auto subFieldInfo = CollectFieldTreeInfo(subFieldId);
 
      compressedSize += subFieldInfo.GetCompressedSize();
      uncompressedSize += subFieldInfo.GetUncompressedSize();
   }
 
   auto fieldInfo = RFieldTreeInspector(fDescriptor.GetFieldDescriptor(fieldId), compressedSize, uncompressedSize);
   fFieldTreeInfo.emplace(fieldId, fieldInfo);
   return fieldInfo;
}
 
std::vector<ROOT::DescriptorId_t>
ROOT::Experimental::RNTupleInspector::GetAllColumnsOfField(ROOT::DescriptorId_t fieldId) const
{
   std::vector<ROOT::DescriptorId_t> colIds;
   std::deque<ROOT::DescriptorId_t> fieldIdQueue{fieldId};
 
   while (!fieldIdQueue.empty()) {
      auto currId = fieldIdQueue.front();
      fieldIdQueue.pop_front();
 
      for (const auto &col : fDescriptor.GetColumnIterable(currId)) {
         if (col.IsAliasColumn()) {
            continue;
         }
 
         colIds.emplace_back(col.GetPhysicalId());
      }
 
      for (const auto &fld : fDescriptor.GetFieldIterable(currId)) {
         fieldIdQueue.push_back(fld.GetId());
      }
   }
 
   return colIds;
}
 
std::unique_ptr<ROOT::Experimental::RNTupleInspector>
ROOT::Experimental::RNTupleInspector::Create(const ROOT::RNTuple &sourceNTuple)
{
   auto pageSource = ROOT::Internal::RPageSourceFile::CreateFromAnchor(sourceNTuple);
   return std::unique_ptr<RNTupleInspector>(new RNTupleInspector(std::move(pageSource)));
}
 
std::unique_ptr<ROOT::Experimental::RNTupleInspector>
ROOT::Experimental::RNTupleInspector::Create(std::string_view ntupleName, std::string_view sourceFileName)
{
   auto pageSource = ROOT::Internal::RPageSource::Create(ntupleName, sourceFileName);
   return std::unique_ptr<RNTupleInspector>(new RNTupleInspector(std::move(pageSource)));
}
 
std::string ROOT::Experimental::RNTupleInspector::GetCompressionSettingsAsString() const
{
   if (!fCompressionSettings)
      return "unknown";
 
   int algorithm = *fCompressionSettings / 100;
   int level = *fCompressionSettings - (algorithm * 100);
 
   return RCompressionSetting::AlgorithmToString(static_cast<RCompressionSetting::EAlgorithm::EValues>(algorithm)) +
          " (level " + std::to_string(level) + ")";
}
 
//------------------------------------------------------------------------------
 
const ROOT::Experimental::RNTupleInspector::RColumnInspector &
ROOT::Experimental::RNTupleInspector::GetColumnInspector(ROOT::DescriptorId_t physicalColumnId) const
{
   if (physicalColumnId > fDescriptor.GetNPhysicalColumns()) {
      throw RException(R__FAIL("No column with physical ID " + std::to_string(physicalColumnId) + " present"));
   }
 
   return fColumnInfo.at(physicalColumnId);
}
 
size_t ROOT::Experimental::RNTupleInspector::GetColumnCountByType(ROOT::ENTupleColumnType colType) const
{
   size_t typeCount = 0;
 
   for (auto &[colId, colInfo] : fColumnInfo) {
      if (colInfo.GetType() == colType) {
         ++typeCount;
      }
   }
 
   return typeCount;
}
 
std::vector<ROOT::DescriptorId_t>
ROOT::Experimental::RNTupleInspector::GetColumnsByType(ROOT::ENTupleColumnType colType)
{
   std::vector<ROOT::DescriptorId_t> colIds;
 
   for (const auto &[colId, colInfo] : fColumnInfo) {
      if (colInfo.GetType() == colType)
         colIds.emplace_back(colId);
   }
 
   return colIds;
}
 
std::vector<ROOT::ENTupleColumnType> ROOT::Experimental::RNTupleInspector::GetColumnTypes()
{
   std::set<ROOT::ENTupleColumnType> colTypes;
 
   for (const auto &[colId, colInfo] : fColumnInfo) {
      colTypes.emplace(colInfo.GetType());
   }
 
   return std::vector(colTypes.begin(), colTypes.end());
}
 
void ROOT::Experimental::RNTupleInspector::PrintColumnTypeInfo(ENTupleInspectorPrintFormat format, std::ostream &output)
{
   struct ColumnTypeInfo {
      std::uint64_t nElems = 0;
      std::uint64_t compressedSize = 0;
      std::uint64_t uncompressedSize = 0;
      std::uint64_t nPages = 0;
      std::uint32_t count = 0;
 
      void operator+=(const RColumnInspector &colInfo)
      {
         this->count++;
         this->nElems += colInfo.GetNElements();
         this->compressedSize += colInfo.GetCompressedSize();
         this->uncompressedSize += colInfo.GetUncompressedSize();
         this->nPages += colInfo.GetNPages();
      }
 
      // Helper method to calculate compression factor
      float GetCompressionFactor() const
      {
         if (compressedSize == 0)
            return 1.0;
         return static_cast<float>(uncompressedSize) / static_cast<float>(compressedSize);
      }
   };
 
   std::map<ENTupleColumnType, ColumnTypeInfo> colTypeInfo;
 
   // Collect information for each column
   for (const auto &[colId, colInfo] : fColumnInfo) {
      colTypeInfo[colInfo.GetType()] += colInfo;
   }
 
   switch (format) {
   case ENTupleInspectorPrintFormat::kTable:
      output << " column type    | count   | # elements  | compressed bytes | uncompressed bytes | compression ratio | "
                "# pages \n"
             << "----------------|---------|-------------|------------------|--------------------|-------------------|-"
                "------\n";
      for (const auto &[colType, typeInfo] : colTypeInfo)
         output << std::setw(15) << RColumnElementBase::GetColumnTypeName(colType) << " |" << std::setw(8)
                << typeInfo.count << " |" << std::setw(12) << typeInfo.nElems << " |" << std::setw(17)
                << typeInfo.compressedSize << " |" << std::setw(19) << typeInfo.uncompressedSize << " |" << std::fixed
                << std::setprecision(3) << std::setw(18) << typeInfo.GetCompressionFactor() << " |" << std::setw(6)
                << typeInfo.nPages << " \n";
      break;
   case ENTupleInspectorPrintFormat::kCSV:
      output << "columnType,count,nElements,compressedSize,uncompressedSize,compressionFactor,nPages\n";
      for (const auto &[colType, typeInfo] : colTypeInfo) {
         output << RColumnElementBase::GetColumnTypeName(colType) << "," << typeInfo.count << "," << typeInfo.nElems
                << "," << typeInfo.compressedSize << "," << typeInfo.uncompressedSize << "," << std::fixed
                << std::setprecision(3) << typeInfo.GetCompressionFactor() << "," << typeInfo.nPages << '\n';
      }
      break;
   default: R__ASSERT(false && "Invalid print format");
   }
}
 
std::unique_ptr<TH1D>
ROOT::Experimental::RNTupleInspector::GetColumnTypeInfoAsHist(ROOT::Experimental::ENTupleInspectorHist histKind,
                                                              std::string_view histName, std::string_view histTitle)
{
   if (histName.empty()) {
      switch (histKind) {
      case ENTupleInspectorHist::kCount: histName = "colTypeCountHist"; break;
      case ENTupleInspectorHist::kNElems: histName = "colTypeElemCountHist"; break;
      case ENTupleInspectorHist::kCompressedSize: histName = "colTypeCompSizeHist"; break;
      case ENTupleInspectorHist::kUncompressedSize: histName = "colTypeUncompSizeHist"; break;
      default: throw RException(R__FAIL("Unknown histogram type"));
      }
   }
 
   if (histTitle.empty()) {
      switch (histKind) {
      case ENTupleInspectorHist::kCount: histTitle = "Column count by type"; break;
      case ENTupleInspectorHist::kNElems: histTitle = "Number of elements by column type"; break;
      case ENTupleInspectorHist::kCompressedSize: histTitle = "Compressed size by column type"; break;
      case ENTupleInspectorHist::kUncompressedSize: histTitle = "Uncompressed size by column type"; break;
      default: throw RException(R__FAIL("Unknown histogram type"));
      }
   }
 
   auto hist = std::make_unique<TH1D>(std::string(histName).c_str(), std::string(histTitle).c_str(), 1, 0, 1);
 
   double data;
   for (const auto &[colId, colInfo] : fColumnInfo) {
      switch (histKind) {
      case ENTupleInspectorHist::kCount: data = 1.; break;
      case ENTupleInspectorHist::kNElems: data = colInfo.GetNElements(); break;
      case ENTupleInspectorHist::kCompressedSize: data = colInfo.GetCompressedSize(); break;
      case ENTupleInspectorHist::kUncompressedSize: data = colInfo.GetUncompressedSize(); break;
      default: throw RException(R__FAIL("Unknown histogram type"));
      }
 
      hist->AddBinContent(hist->GetXaxis()->FindBin(RColumnElementBase::GetColumnTypeName(colInfo.GetType())), data);
   }
 
   return hist;
}
 
std::unique_ptr<TH1D>
ROOT::Experimental::RNTupleInspector::GetPageSizeDistribution(ROOT::DescriptorId_t physicalColumnId,
                                                              std::string histName, std::string histTitle, size_t nBins)
{
   if (histTitle.empty())
      histTitle = "Page size distribution for column with ID " + std::to_string(physicalColumnId);
 
   return GetPageSizeDistribution({physicalColumnId}, histName, histTitle, nBins);
}
 
std::unique_ptr<TH1D> ROOT::Experimental::RNTupleInspector::GetPageSizeDistribution(ROOT::ENTupleColumnType colType,
                                                                                    std::string histName,
                                                                                    std::string histTitle, size_t nBins)
{
   if (histName.empty())
      histName = "pageSizeHistCol" + std::string{RColumnElementBase::GetColumnTypeName(colType)};
   if (histTitle.empty())
      histTitle =
         "Page size distribution for columns with type " + std::string{RColumnElementBase::GetColumnTypeName(colType)};
 
   auto perTypeHist = GetPageSizeDistribution({colType}, histName, histTitle, nBins);
 
   if (perTypeHist->GetNhists() < 1)
      return std::make_unique<TH1D>(histName.c_str(), histTitle.c_str(), 64, 0, 0);
 
   auto hist = std::unique_ptr<TH1D>(dynamic_cast<TH1D *>(perTypeHist->GetHists()->First()));
 
   hist->SetName(histName.c_str());
   hist->SetTitle(histTitle.c_str());
   hist->SetXTitle("Page size (B)");
   hist->SetYTitle("N_{pages}");
   return hist;
}
 
std::unique_ptr<TH1D>
ROOT::Experimental::RNTupleInspector::GetPageSizeDistribution(std::initializer_list<ROOT::DescriptorId_t> colIds,
                                                              std::string histName, std::string histTitle, size_t nBins)
{
   auto hist = std::make_unique<TH1D>();
 
   if (histName.empty())
      histName = "pageSizeHist";
   hist->SetName(histName.c_str());
   if (histTitle.empty())
      histTitle = "Page size distribution";
   hist->SetTitle(histTitle.c_str());
   hist->SetXTitle("Page size (B)");
   hist->SetYTitle("N_{pages}");
 
   std::vector<std::uint64_t> pageSizes;
   std::for_each(colIds.begin(), colIds.end(), [this, &pageSizes](const auto colId) {
      auto colInfo = GetColumnInspector(colId);
      pageSizes.insert(pageSizes.end(), colInfo.GetCompressedPageSizes().begin(),
                       colInfo.GetCompressedPageSizes().end());
   });
 
   if (!pageSizes.empty()) {
      auto histMinMax = std::minmax_element(pageSizes.begin(), pageSizes.end());
      hist->SetBins(nBins, *histMinMax.first,
                    *histMinMax.second + ((*histMinMax.second - *histMinMax.first) / static_cast<double>(nBins)));
 
      for (const auto pageSize : pageSizes) {
         hist->Fill(pageSize);
      }
   }
 
   return hist;
}
 
std::unique_ptr<THStack>
ROOT::Experimental::RNTupleInspector::GetPageSizeDistribution(std::initializer_list<ROOT::ENTupleColumnType> colTypes,
                                                              std::string histName, std::string histTitle, size_t nBins)
{
   if (histName.empty())
      histName = "pageSizeHist";
   if (histTitle.empty())
      histTitle = "Per-column type page size distribution";
 
   auto stackedHist = std::make_unique<THStack>(histName.c_str(), histTitle.c_str());
 
   double histMin = std::numeric_limits<double>::max();
   double histMax = 0;
   std::map<ROOT::ENTupleColumnType, std::vector<std::uint64_t>> pageSizes;
 
   std::vector<ROOT::ENTupleColumnType> colTypeVec = colTypes;
   if (std::empty(colTypes)) {
      colTypeVec = GetColumnTypes();
   }
 
   for (const auto colType : colTypeVec) {
      auto colIds = GetColumnsByType(colType);
 
      if (colIds.empty())
         continue;
 
      std::vector<std::uint64_t> pageSizesForColType;
      std::for_each(colIds.cbegin(), colIds.cend(), [this, &pageSizesForColType](const auto colId) {
         auto colInfo = GetColumnInspector(colId);
         pageSizesForColType.insert(pageSizesForColType.end(), colInfo.GetCompressedPageSizes().begin(),
                                    colInfo.GetCompressedPageSizes().end());
      });
      if (pageSizesForColType.empty())
         continue;
 
      pageSizes.emplace(colType, pageSizesForColType);
 
      auto histMinMax = std::minmax_element(pageSizesForColType.begin(), pageSizesForColType.end());
      histMin = std::min(histMin, static_cast<double>(*histMinMax.first));
      histMax = std::max(histMax, static_cast<double>(*histMinMax.second));
   }
 
   for (const auto &[colType, pageSizesForColType] : pageSizes) {
      auto hist = std::make_unique<TH1D>(
         TString::Format("%s%s", histName.c_str(), RColumnElementBase::GetColumnTypeName(colType)),
         RColumnElementBase::GetColumnTypeName(colType), nBins, histMin,
         histMax + ((histMax - histMin) / static_cast<double>(nBins)));
 
      for (const auto pageSize : pageSizesForColType) {
         hist->Fill(pageSize);
      }
 
      stackedHist->Add(hist.release());
   }
 
   return stackedHist;
}
 
std::unique_ptr<THStack> ROOT::Experimental::RNTupleInspector::GetPagesPerClusterDistribution(
   std::initializer_list<ROOT::ENTupleColumnType> colTypes, std::string_view histName, std::string_view histTitle,
   size_t nBins)
{
   if (histName.empty())
      histName = "pagesPerClusterHist";
   if (histTitle.empty())
      histTitle = "#pages per cluster";
 
   auto stackedHist = std::make_unique<THStack>(std::string(histName).c_str(), std::string(histTitle).c_str());
   // Each element of the vector contains the sum of all pages of all columns of each type.
   std::vector<std::unordered_map<ENTupleColumnType, std::uint64_t>> pagesPerClusterPerColumnType;
 
   std::vector<ROOT::ENTupleColumnType> colTypeVec = colTypes;
   if (std::empty(colTypes)) {
      colTypeVec = GetColumnTypes();
   }
 
   R__ASSERT(fDescriptor.GetNClusters() == fDescriptor.GetNActiveClusters());
   for (const auto &clDesc : fDescriptor.GetActiveClusterIterable()) {
      auto &pagesPerColumnTypeInThisCluster = pagesPerClusterPerColumnType.emplace_back();
      for (const auto &colRange : clDesc.GetColumnRangeIterable()) {
         const auto colId = colRange.GetPhysicalColumnId();
         const auto &colDesc = fDescriptor.GetColumnDescriptor(colId);
         const auto nPages = clDesc.GetPageRange(colId).GetPageInfos().size();
         auto &nPagesForThisType = pagesPerColumnTypeInThisCluster[colDesc.GetType()];
         nPagesForThisType += nPages;
      }
   }
 
   double histMin = std::numeric_limits<double>::max();
   double histMax = 0;
   for (const auto &pagesPerType : pagesPerClusterPerColumnType) {
      auto [min, max] = std::minmax_element(pagesPerType.begin(), pagesPerType.end(),
                                            [](const auto &a, const auto &b) { return a.second < b.second; });
      histMin = std::min<double>(histMin, min->second);
      histMax = std::max<double>(histMax, max->second);
   }
 
   std::array<std::unique_ptr<TH1L>, static_cast<std::size_t>(ENTupleColumnType::kMax)> histsPerType = {};
   for (const auto &pagesPerType : pagesPerClusterPerColumnType) {
      for (const auto &[colType, nPages] : pagesPerType) {
         auto &hist = histsPerType[static_cast<std::size_t>(colType)];
         if (!hist) {
            hist = std::make_unique<TH1L>(ROOT::Internal::RColumnElementBase::GetColumnTypeName(colType),
                                          ROOT::Internal::RColumnElementBase::GetColumnTypeName(colType), nBins,
                                          histMin, histMax + ((histMax - histMin) / static_cast<double>(nBins)));
         }
         hist->Fill(nPages);
      }
   }
 
   for (auto &hist : histsPerType) {
      if (hist) {
         stackedHist->Add(hist.release());
      }
   }
 
   return stackedHist;
}
 
//------------------------------------------------------------------------------
 
const ROOT::Experimental::RNTupleInspector::RFieldTreeInspector &
ROOT::Experimental::RNTupleInspector::GetFieldTreeInspector(ROOT::DescriptorId_t fieldId) const
{
   if (fieldId >= fDescriptor.GetNFields()) {
      throw RException(R__FAIL("No field with ID " + std::to_string(fieldId) + " present"));
   }
 
   return fFieldTreeInfo.at(fieldId);
}
 
const ROOT::Experimental::RNTupleInspector::RFieldTreeInspector &
ROOT::Experimental::RNTupleInspector::GetFieldTreeInspector(std::string_view fieldName) const
{
   auto fieldId = fDescriptor.FindFieldId(fieldName);
 
   if (fieldId == kInvalidDescriptorId) {
      throw RException(R__FAIL("Could not find field `" + std::string(fieldName) + "`"));
   }
 
   return GetFieldTreeInspector(fieldId);
}
 
size_t ROOT::Experimental::RNTupleInspector::GetFieldCountByType(const std::regex &typeNamePattern,
                                                                 bool includeSubfields) const
{
   size_t typeCount = 0;
 
   for (auto &[fldId, fldInfo] : fFieldTreeInfo) {
      if (!includeSubfields && fldInfo.GetDescriptor().GetParentId() != fDescriptor.GetFieldZeroId()) {
         continue;
      }
 
      if (std::regex_match(fldInfo.GetDescriptor().GetTypeName(), typeNamePattern)) {
         typeCount++;
      }
   }
 
   return typeCount;
}
 
std::vector<ROOT::DescriptorId_t>
ROOT::Experimental::RNTupleInspector::GetFieldsByName(const std::regex &fieldNamePattern, bool searchInSubfields) const
{
   std::vector<ROOT::DescriptorId_t> fieldIds;
 
   for (auto &[fldId, fldInfo] : fFieldTreeInfo) {
 
      if (!searchInSubfields && fldInfo.GetDescriptor().GetParentId() != fDescriptor.GetFieldZeroId()) {
         continue;
      }
 
      if (std::regex_match(fldInfo.GetDescriptor().GetFieldName(), fieldNamePattern)) {
         fieldIds.emplace_back(fldId);
      }
   }
 
   return fieldIds;
}
 
void ROOT::Experimental::RNTupleInspector::PrintFieldTreeAsDot(const ROOT::RFieldDescriptor &fieldDescriptor,
                                                               std::ostream &output) const
{
   const auto &tupleDescriptor = GetDescriptor();
   const bool isZeroField = fieldDescriptor.GetParentId() == ROOT::kInvalidDescriptorId;
   if (isZeroField) {
      output << "digraph D {\n";
      output << "node[shape=box]\n";
   }
   const std::string &nodeId = (isZeroField) ? "0" : std::to_string(fieldDescriptor.GetId() + 1);
   const std::string &description = fieldDescriptor.GetFieldDescription();
   const std::uint32_t &version = fieldDescriptor.GetFieldVersion();
 
   auto htmlEscape = [&](const std::string &in) -> std::string {
      std::string out;
      out.reserve(in.size());
      for (const char &c : in) {
         switch (c) {
         case '&': out += "&amp;"; break;
         case '<': out += "&lt;"; break;
         case '>': out += "&gt;"; break;
         case '\"': out += "&quot;"; break;
         case '\'': out += "&#39;"; break;
         default: out += c; break;
         }
      }
      return out;
   };
 
   output << nodeId << "[label=<";
   if (!isZeroField) {
      output << "<b>Name: </b>" << htmlEscape(fieldDescriptor.GetFieldName()) << "<br></br>";
      output << "<b>Type: </b>" << htmlEscape(fieldDescriptor.GetTypeName()) << "<br></br>";
      output << "<b>ID: </b>" << std::to_string(fieldDescriptor.GetId()) << "<br></br>";
      if (description != "")
         output << "<b>Description: </b>" << htmlEscape(description) << "<br></br>";
      if (version != 0)
         output << "<b>Version: </b>" << version << "<br></br>";
   } else
      output << "<b>RFieldZero</b>";
   output << ">]\n";
   for (const auto &childFieldId : fieldDescriptor.GetLinkIds()) {
      const auto &childFieldDescriptor = tupleDescriptor.GetFieldDescriptor(childFieldId);
      output << nodeId + "->" + std::to_string(childFieldDescriptor.GetId() + 1) + "\n";
      PrintFieldTreeAsDot(childFieldDescriptor, output);
   }
   if (isZeroField)
      output << "}";
}
 
namespace {
 
struct SpeedscopeFrame {
   std::string fString;
   std::uint64_t fOpeningPosition = 0;
   std::uint64_t fClosingPosition = 0;
};
 
static void PrintSpeedscopeFrames(const std::vector<SpeedscopeFrame> &frames, std::ostream &output)
{
   output << "{\n";
   output << "   \"$schema\":\"https://www.speedscope.app/file-format-schema.json\",\n";
   output << "   \"shared\":{\n";
   output << "      \"frames\":[\n";
 
   for (std::size_t i = 0; i < frames.size(); ++i) {
      output << "         { \"name\":\"" << frames[i].fString << "\" }" << (i + 1 < frames.size() ? ",\n" : "\n");
   }
 
   output << "      ]\n";
   output << "   },\n";
   output << "   \"profiles\":[\n";
   output << "      {\n";
   output << "         \"type\":\"evented\",\n";
   output << "         \"name\":\"Flattened Timeline\",\n";
   output << "         \"unit\":\"bytes\",\n";
   output << "         \"startValue\":0,\n";
   output << "         \"endValue\":" << frames.back().fClosingPosition << ",\n";
   output << "         \"events\":[\n";
 
   bool first = true;
 
   // Parameter idx Index of the frame being processed
   // Parameter limit
   //   - If the frame is not root: Closing Position of its father
   //   - If the frame is root: Closing Position of the last element of frames
   // Returns the next index to be processed
   std::function<std::size_t(std::size_t, std::uint32_t)> processRecursive = [&](std::size_t nextIdxToProcess,
                                                                                 std::uint32_t limit) -> std::size_t {
      while (nextIdxToProcess < frames.size() && frames[nextIdxToProcess].fOpeningPosition < limit) {
         const std::size_t currentIdx = nextIdxToProcess;
 
         if (!first)
            output << ",\n";
 
         output << "            {\"type\":\"O\",\"frame\":" << currentIdx
                << ",\"at\":" << frames[currentIdx].fOpeningPosition << "}";
         first = false;
 
         nextIdxToProcess = processRecursive(nextIdxToProcess + 1, frames[currentIdx].fClosingPosition);
 
         output << ",\n            {\"type\":\"C\",\"frame\":" << currentIdx
                << ",\"at\":" << frames[currentIdx].fClosingPosition << "}";
      }
      return nextIdxToProcess;
   };
 
   processRecursive(0, frames.back().fClosingPosition);
 
   output << "\n         ]\n";
   output << "      }\n";
   output << "   ]\n";
   output << "}\n";
}
} // namespace
 
void ROOT::Experimental::RNTupleInspector::PrintSchemaProfile(std::ostream &output,
                                                              [[maybe_unused]] ESchemaProfileFormat format) const
{
   // There is only one format at the moment
   assert(format == ESchemaProfileFormat::kSpeedscopeJSON);
 
   const auto &tupleDescriptor = GetDescriptor();
   ROOT::DescriptorId_t rootId = tupleDescriptor.GetFieldZeroId();
   const auto &rootFieldDescriptor = tupleDescriptor.GetFieldDescriptor(rootId);
 
   std::vector<SpeedscopeFrame> frames;
   std::uint64_t positionCursor = 0;
 
   // Returns size of the visited field
   auto visitFieldsRecursive = [&](auto &self, const ROOT::RFieldDescriptor &fieldDescriptor) -> std::size_t {
      SpeedscopeFrame fieldSpeedscopeFrame;
      fieldSpeedscopeFrame.fString =
         tupleDescriptor.GetQualifiedFieldName(fieldDescriptor.GetId()) + " (" + fieldDescriptor.GetTypeName() + ")";
      fieldSpeedscopeFrame.fOpeningPosition = positionCursor;
      frames.push_back(fieldSpeedscopeFrame);
 
      std::size_t fieldSpeedscopeFrameIndex = frames.size() - 1;
 
      std::size_t subTreeSize = 0;
      const auto &childIds = fieldDescriptor.GetLinkIds();
 
      for (const auto &childFieldId : childIds) {
         const auto &childFieldDescriptor = tupleDescriptor.GetFieldDescriptor(childFieldId);
         subTreeSize += self(self, childFieldDescriptor);
      }
 
      for (const auto &columnDescriptor : tupleDescriptor.GetColumnIterable(fieldDescriptor.GetId())) {
         const auto &columnInfo = GetColumnInspector(columnDescriptor.GetPhysicalId());
         std::size_t columnSize = columnInfo.GetCompressedSize();
 
         SpeedscopeFrame columnSpeedscopeFrame;
         columnSpeedscopeFrame.fString =
            "[col#" + std::to_string(columnDescriptor.GetPhysicalId()) + "] " +
            tupleDescriptor.GetQualifiedFieldName(fieldDescriptor.GetId()) + " (" +
            ROOT::Internal::RColumnElementBase::GetColumnTypeName(columnDescriptor.GetType()) + ")";
         columnSpeedscopeFrame.fOpeningPosition = positionCursor;
         positionCursor += columnSize;
         columnSpeedscopeFrame.fClosingPosition = positionCursor;
         frames.push_back(columnSpeedscopeFrame);
         subTreeSize += columnSize;
      }
 
      frames[fieldSpeedscopeFrameIndex].fClosingPosition = positionCursor;
 
      return subTreeSize;
   };
 
   const auto &topLevelIds = rootFieldDescriptor.GetLinkIds();
   for (const auto &childId : topLevelIds) {
      const auto &childFieldDescriptor = tupleDescriptor.GetFieldDescriptor(childId);
      visitFieldsRecursive(visitFieldsRecursive, childFieldDescriptor);
   }
 
   PrintSpeedscopeFrames(frames, output);
}
 
void ROOT::Experimental::RNTupleInspector::PrintDiskProfile(std::ostream &output,
                                                            [[maybe_unused]] ESchemaProfileFormat format) const
{
   // There is only one format at the moment
   assert(format == ESchemaProfileFormat::kSpeedscopeJSON);
 
   const auto *pageSourceFile = dynamic_cast<const ROOT::Internal::RPageSourceFile *>(fPageSource.get());
   // GetAnchorFromFile() only supports file-based backend, so better to check early
   if (!pageSourceFile)
      throw RException(R__FAIL("Disk profile is only supported for file-based page sources"));
   const auto anchor = ROOT::Internal::GetAnchorFromFile(*pageSourceFile);
   if (!anchor)
      R__LOG_WARNING(ROOT::Internal::NTupleLog()) << "Cannot retrieve RNTuple anchor";
 
   const auto &descriptor = GetDescriptor();
 
   struct RDiskPageLeaf {
      std::uint64_t fPosition = 0;
      std::uint64_t fSize = 0;
      std::string fName;
      std::array<DescriptorId_t, 3> fAncestors;
   };
   static constexpr std::array<const char *, 3> kAncestorsNames = {"cluster group", "cluster", "column range"};
   std::vector<RDiskPageLeaf> pageLeaves;
 
   // Collect all pageLeaves in whichever order the iterator provides
   for (const auto &clusterGroupDescriptor : descriptor.GetClusterGroupIterable()) {
      const auto groupId = clusterGroupDescriptor.GetId();
 
      for (const auto clusterId : clusterGroupDescriptor.GetClusterIds()) {
         const auto &clusterDescriptor = descriptor.GetClusterDescriptor(clusterId);
 
         for (const auto &columnRange : clusterDescriptor.GetColumnRangeIterable()) {
            const auto columnId = columnRange.GetPhysicalColumnId();
 
            const auto &pageRange = clusterDescriptor.GetPageRange(columnId);
            for (const auto &pageInfo : pageRange.GetPageInfos()) {
               const auto &locator = pageInfo.GetLocator();
 
               RDiskPageLeaf pageLeaf;
               pageLeaf.fPosition = locator.GetPosition<std::uint64_t>();
               pageLeaf.fSize = locator.GetNBytesOnStorage() +
                                pageInfo.HasChecksum() * ROOT::Internal::RPageStorage::kNBytesPageChecksum;
               pageLeaf.fName = "[page @" + std::to_string(pageLeaf.fPosition) + "]";
               pageLeaf.fAncestors = {groupId, clusterId, columnId};
               pageLeaves.push_back(pageLeaf);
            }
         }
      }
   }
 
   // Sort pageLeafs by on-disk address
   std::sort(pageLeaves.begin(), pageLeaves.end(),
             [](const RDiskPageLeaf &a, const RDiskPageLeaf &b) { return a.fPosition < b.fPosition; });
 
   // Remove aliases (the ntuple specification allows complete, but not partial, overlap between pages)
   pageLeaves.erase(
      std::unique(pageLeaves.begin(), pageLeaves.end(),
                  [](const RDiskPageLeaf &a, const RDiskPageLeaf &b) { return a.fPosition == b.fPosition; }),
      pageLeaves.end());
 
   std::vector<SpeedscopeFrame> frames;
 
   // Construct frame for ntuple header
   if (anchor) {
      SpeedscopeFrame headerFrame;
      headerFrame.fString = "ntuple header";
      headerFrame.fOpeningPosition = anchor->GetSeekHeader();
      headerFrame.fClosingPosition = anchor->GetSeekHeader() + anchor->GetNBytesHeader();
      frames.push_back(headerFrame);
   }
 
   struct ROpenFrame {
      ROOT::DescriptorId_t fId = 0; // clusterGroup, cluster, columnRange id
      std::size_t fIndex = 0;       // index in frames vector
   };
   std::vector<ROpenFrame> openFrames;
   std::uint64_t previouspageLeafEnd = 0;
 
   // Construct frames from the bottom (leafs ordered by disk address) upwards
   for (const auto &pageLeaf : pageLeaves) {
      std::size_t sharedDepth = 0;
 
      // How many of the currently open ancestors does this pageLeaf share?
      while (sharedDepth < openFrames.size() && sharedDepth < pageLeaf.fAncestors.size() &&
             openFrames[sharedDepth].fId == pageLeaf.fAncestors[sharedDepth]) {
         sharedDepth++;
      }
 
      // Close ancestors not shared with this pageLeaf (innermost first order)
      while (openFrames.size() > sharedDepth) {
         frames[openFrames.back().fIndex].fClosingPosition = previouspageLeafEnd;
         openFrames.pop_back();
      }
 
      // Open the ancestors this pageLeaf needs (outermost first order)
      for (std::size_t depth = sharedDepth; depth < pageLeaf.fAncestors.size(); ++depth) {
         SpeedscopeFrame ancestorFrame;
         ancestorFrame.fString =
            "[" + std::string(kAncestorsNames[depth]) + " " + std::to_string(pageLeaf.fAncestors[depth]) + "]";
         ancestorFrame.fOpeningPosition = pageLeaf.fPosition;
         frames.push_back(ancestorFrame);
 
         ROpenFrame openFrame;
         openFrame.fId = pageLeaf.fAncestors[depth];
         openFrame.fIndex = frames.size() - 1;
         openFrames.push_back(openFrame);
      }
 
      // Emit the pageLeaf itself
      SpeedscopeFrame pageLeafFrame;
      pageLeafFrame.fString = pageLeaf.fName;
      pageLeafFrame.fOpeningPosition = pageLeaf.fPosition;
      pageLeafFrame.fClosingPosition = pageLeaf.fPosition + pageLeaf.fSize;
      frames.push_back(pageLeafFrame);
 
      previouspageLeafEnd = pageLeaf.fPosition + pageLeaf.fSize;
   }
 
   // Close whatever is still open after the last pageLeaf
   while (!openFrames.empty()) {
      frames[openFrames.back().fIndex].fClosingPosition = previouspageLeafEnd;
      openFrames.pop_back();
   }
 
   // Construct frames for page lists
   for (const auto &clusterGroupDescriptor : descriptor.GetClusterGroupIterable()) {
      const auto locator = clusterGroupDescriptor.GetPageListLocator();
 
      SpeedscopeFrame pageListFrame;
      pageListFrame.fString = "[page list " + std::to_string(clusterGroupDescriptor.GetId()) + "]";
      pageListFrame.fOpeningPosition = locator.GetPosition<std::uint64_t>();
      pageListFrame.fClosingPosition = locator.GetPosition<std::uint64_t>() + locator.GetNBytesOnStorage();
      frames.push_back(pageListFrame);
   }
 
   // Construct frame for ntuple footer
   if (anchor) {
      SpeedscopeFrame footerFrame;
      footerFrame.fString = "ntuple footer";
      footerFrame.fOpeningPosition = anchor->GetSeekFooter();
      footerFrame.fClosingPosition = anchor->GetSeekFooter() + anchor->GetNBytesFooter();
      frames.push_back(footerFrame);
   }
 
   PrintSpeedscopeFrames(frames, output);
}