From 77305662736affe854378de4ad6180034132c1a0 Mon Sep 17 00:00:00 2001 From: Joshua Koenig Date: Wed, 10 Aug 2022 10:51:08 +0200 Subject: [PATCH 001/289] [EMCAL-566] print detailed message if gaus fit fails - cellID is now printed if the fit of the time distribution of this cell fails and in addition, the type of the failure can be accessed and is printed in case of the time calibration - Usefull and needed for debugging and in case something goes wrong during data taking - Small fixes in the fitGaus method: corrected caluclation of the bin width, number of bins and the maimum bin content. This is only needed in case their are less than 3 bins filled in the histogram which should not happen in practice --- .../include/CommonUtils/BoostHistogramUtils.h | 117 ++++++++---------- Common/Utils/src/BoostHistogramUtils.cxx | 21 ++++ .../EMCALCalibration/EMCALCalibExtractor.h | 6 +- 3 files changed, 77 insertions(+), 67 deletions(-) diff --git a/Common/Utils/include/CommonUtils/BoostHistogramUtils.h b/Common/Utils/include/CommonUtils/BoostHistogramUtils.h index 68812115770b2..e663314b9741a 100644 --- a/Common/Utils/include/CommonUtils/BoostHistogramUtils.h +++ b/Common/Utils/include/CommonUtils/BoostHistogramUtils.h @@ -28,7 +28,6 @@ #include "Rtypes.h" #include "TLinearFitter.h" #include "TVectorD.h" -#include "TMath.h" #include "TF1.h" #include "Foption.h" #include "HFitInterface.h" @@ -211,12 +210,18 @@ class fitResult * \brief Error code for invalid result in the fitGaus process */ enum class FitGausError_t { - FIT_ERROR, ///< Fit procedure returned invalid result + FIT_ERROR_MAX, ///< Gaus fit failed! yMax too large + FIT_ERROR_MIN, ///< Gaus fit failed! yMax < 4 + FIT_ERROR_ENTRIES, ///< Gaus fit failed! entries < 12 + FIT_ERROR_KTOL_MEAN, ///< Gaus fit failed! std::abs(par[1]) < kTol + FIT_ERROR_KTOL_SIGMA, ///< Gaus fit failed! std::abs(par[2]) < kTol + FIT_ERROR_KTOL_RMS ///< Gaus fit failed! RMS < kTol + }; /// \brief Printing an error message when then fit returns an invalid result /// \param errorcode Error of the type FitGausError_t, thrown when fit result is invalid. -std::string createErrorMessage(o2::utils::FitGausError_t errorcode); +std::string createErrorMessageFitGaus(o2::utils::FitGausError_t errorcode); /// \brief Function to fit histogram to a gaussian using iterators. /// \param first begin iterator of the histogram @@ -235,31 +240,36 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir if (ignoreUnderOverflowBin) { first++; last--; + axisfirst++; } - static TLinearFitter fitter(3, "pol2"); - static TMatrixD mat(3, 3); - static Double_t kTol = mat.GetTol(); + TLinearFitter fitter(3, "pol2"); + TMatrixD mat(3, 3); + double kTol = mat.GetTol(); fitter.StoreData(kFALSE); fitter.ClearPoints(); TVectorD par(3); TMatrixD A(3, 3); TMatrixD b(3, 1); - T rms = TMath::RMS(first, last); - // Markus: return type of std::max_element is an iterator, cannot cast implicitly to double - // better use auto - // pointer needs to be dereferenced afterwards - auto xMax = std::max_element(first, last); - auto xMin = std::min_element(first, last); - auto nbins = last - first; - const double binWidth = double(*xMax - *xMin) / double(nbins); - - Float_t meanCOG = 0; - Float_t rms2COG = 0; - Float_t sumCOG = 0; - Float_t entries = 0; - Int_t nfilled = 0; + // return type of std::max_element is an iterator, cannot cast implicitly to double + // pointer needs to be dereferenced afterwards + auto yMax = std::max_element(first, last); + auto nbins = std::distance(first, last); + auto getBinWidth = [](BinCenterView axisiter) { + double binCenter1 = *axisiter; + axisiter++; + double binCenter2 = *axisiter; + return std::abs(binCenter1 - binCenter2); + }; + double binWidth = getBinWidth(axisfirst); + + float meanCOG = 0; + float rms2COG = 0; + float sumCOG = 0; + + float entries = 0; + int nfilled = 0; for (auto iter = first; iter != last; iter++) { entries += *iter; @@ -268,17 +278,11 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir } } - if (*xMax < 4) { - LOG(warning) << "Gaus fit failed! xMax < 4"; - throw FitGausError_t::FIT_ERROR; + if (*yMax < 4) { + throw FitGausError_t::FIT_ERROR_MIN; } if (entries < 12) { - LOG(warning) << "Gaus fit failed! entries < 12"; - throw FitGausError_t::FIT_ERROR; - } - if (rms < kTol) { - LOG(warning) << "Gaus fit failed! rms < kTol"; - throw FitGausError_t::FIT_ERROR; + throw FitGausError_t::FIT_ERROR_ENTRIES; } // create the result, first fill it with all 0's @@ -289,7 +293,7 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir // then set the third parameter to entries result.at(3) = entries; - Int_t npoints = 0; + int npoints = 0; // in this loop: increase iter and axisiter (iterator for bin center and bincontent) auto axisiter = axisfirst; for (auto iter = first; iter != last; iter++, axisiter++) { @@ -300,7 +304,7 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir continue; } double x = *axisiter; - // take logarith of gaussian in order to obtain a pol2 + // take logarithm of gaussian in order to obtain a pol2 double y = std::log(*iter); // first order taylor series of log(x) to approimate the errors df(x)/dx * err(f(x)) double ey = std::sqrt(fabs(*iter)) / fabs(*iter); @@ -312,13 +316,13 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir A(npoints, 2) = x * x; b(npoints, 0) = y; meanCOG += x * nbins; - rms2COG += x * nbins * x; + rms2COG += x * x * nbins; sumCOG += nbins; } npoints++; } } - Double_t chi2 = 0; + double chi2 = 0; if (npoints >= 3) { if (npoints == 3) { // analytic calculation of the parameters for three points @@ -334,58 +338,43 @@ std::vector fitGaus(Iterator first, Iterator last, BinCenterView axisfir fitter.Eval(); fitter.GetParameters(par); fitter.GetCovarianceMatrix(mat); - result.at(4) = (fitter.GetChisquare() / Double_t(npoints)); + result.at(4) = (fitter.GetChisquare() / double(npoints)); } - if (TMath::Abs(par[1]) < kTol) { - LOG(warning) << "Gaus fit failed! TMath::Abs(par[1]) < kTol"; - throw FitGausError_t::FIT_ERROR; + if (std::abs(par[1]) < kTol) { + throw FitGausError_t::FIT_ERROR_KTOL_MEAN; } - if (TMath::Abs(par[2]) < kTol) { - LOG(warning) << "Gaus fit failed! TMath::Abs(par[2]) < kTol"; - throw FitGausError_t::FIT_ERROR; + if (std::abs(par[2]) < kTol) { + throw FitGausError_t::FIT_ERROR_KTOL_SIGMA; } // calculate parameters for gaus from pol2 fit T param1 = T(par[1] / (-2. * par[2])); result.at(1) = param1; - result.at(2) = T(1. / TMath::Sqrt(TMath::Abs(-2. * par[2]))); + result.at(2) = T(1. / std::sqrt(std::abs(-2. * par[2]))); auto lnparam0 = par[0] - par[1] * par[1] / (4 * par[2]); if (lnparam0 > 307) { - LOG(warning) << "Gaus fit failed! lnparam0 > 307"; - throw FitGausError_t::FIT_ERROR; + throw FitGausError_t::FIT_ERROR_MAX; } - result.at(0) = T(TMath::Exp(lnparam0)); + result.at(0) = T(std::exp(lnparam0)); return result; - } - if (npoints == 2) { + } else if (npoints == 2) { // use center of gravity for 2 points meanCOG /= sumCOG; rms2COG /= sumCOG; - /* - result.setParameter<0>(xMax); - result.setParameter<1>(meanCOG); - result.setParameter<2>(TMath::Sqrt(TMath::Abs(meanCOG * meanCOG - rms2COG))); - result.setChi2(-2); - */ - result.at(0) = *xMax; + + result.at(0) = *yMax; result.at(1) = meanCOG; - result.at(2) = TMath::Sqrt(TMath::Abs(meanCOG * meanCOG - rms2COG)); + result.at(2) = std::sqrt(std::abs(meanCOG * meanCOG - rms2COG)); result.at(4) = -2; - } - if (npoints == 1) { + } else if (npoints == 1) { meanCOG /= sumCOG; - /* - result.setParameter<0>(xMax); - result.setParameter<1>(meanCOG); - result.setParameter<2>(binWidth / TMath::Sqrt(12)); - result.setChi2(-1); - */ - result.at(0) = *xMax; + + result.at(0) = *yMax; result.at(1) = meanCOG; - result.at(2) = binWidth / TMath::Sqrt(12); + result.at(2) = binWidth / std::sqrt(12); result.at(4) = -1; } diff --git a/Common/Utils/src/BoostHistogramUtils.cxx b/Common/Utils/src/BoostHistogramUtils.cxx index c661fc750d25a..c5c9d97742976 100644 --- a/Common/Utils/src/BoostHistogramUtils.cxx +++ b/Common/Utils/src/BoostHistogramUtils.cxx @@ -19,6 +19,27 @@ namespace o2 namespace utils { +std::string createErrorMessageFitGaus(o2::utils::FitGausError_t errorcode) +{ + switch (errorcode) { + case FitGausError_t::FIT_ERROR_MIN: + return "Gaus fit failed! xMax < 4"; + case FitGausError_t::FIT_ERROR_MAX: + return "Gaus fit failed! xMax too large"; + case FitGausError_t::FIT_ERROR_ENTRIES: + return "Gaus fit failed! entries < 12"; + case FitGausError_t::FIT_ERROR_KTOL_MEAN: + return "Gaus fit failed! std::abs(par[1]) < kTol"; + case FitGausError_t::FIT_ERROR_KTOL_SIGMA: + return "Gaus fit failed! std::abs(par[2]) < kTol"; + case FitGausError_t::FIT_ERROR_KTOL_RMS: + return "Gaus fit failed! RMS < kTol"; + default: + return "Gaus fit failed! Unknown error code"; + } + return "Gaus fit failed! Unknown error code"; +} + boostHisto1d_VarAxis boosthistoFromRoot_1D(TH1D* inHist1D) { // first setup the proper boost histogram diff --git a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h index 9beaf110054cf..881d09b1156f1 100644 --- a/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h +++ b/Detectors/EMCAL/calibration/include/EMCALCalibration/EMCALCalibExtractor.h @@ -140,8 +140,7 @@ class EMCALCalibExtractor maxElementIndex = 0; } float maxElementCenter = 0.5 * (boostHist1d.axis(0).bin(maxElementIndex).upper() + boostHist1d.axis(0).bin(maxElementIndex).lower()); - float timeInterval = 25; // in ns - boostHist1d = boost::histogram::algorithm::reduce(boostHist1d, boost::histogram::algorithm::shrink(maxElementCenter - timeInterval, maxElementCenter + timeInterval)); + boostHist1d = boost::histogram::algorithm::reduce(boostHist1d, boost::histogram::algorithm::shrink(maxElementCenter - restrictFitRangeToMax, maxElementCenter + restrictFitRangeToMax)); } try { @@ -149,7 +148,8 @@ class EMCALCalibExtractor mean = fitValues.at(1); // add mean to time calib params TCP.addTimeCalibParam(i, mean, 0); - } catch (o2::utils::FitGausError_t) { + } catch (o2::utils::FitGausError_t err) { + LOG(warning) << createErrorMessageFitGaus(err) << "; for cell " << i << " (Will take the parameter of the previous cell: " << mean << "ns)"; TCP.addTimeCalibParam(i, mean, 0); // take calib value of last cell; or 400 ns shift default value } } From b2175be0ffc23299c2b979a26ea9459c1ce1eed3 Mon Sep 17 00:00:00 2001 From: Chiara Zampolli Date: Mon, 15 Aug 2022 17:44:26 +0200 Subject: [PATCH 002/289] MeanVertex X and Y coordinate depend on Z (#9315) * MeanVertex X and Y coordinate depend on Z And make MeanVertex object derive from VertexBase * some reworking * answers to comments * Fix compilation and typo * Prefer switch to general method --- DataFormats/Calibration/CMakeLists.txt | 5 +- .../DataFormatsCalibration/MeanVertexObject.h | 99 +++-- .../Calibration/src/MeanVertexObject.cxx | 52 +++ .../MeanVertexCalibrator.h | 36 +- .../DetectorsCalibration/MeanVertexData.h | 60 +-- .../DetectorsCalibration/MeanVertexParams.h | 2 +- .../Calibration/src/MeanVertexCalibrator.cxx | 388 +++++++++++++----- Detectors/Calibration/src/MeanVertexData.cxx | 54 +-- .../workflow/src/MeanVertexCalibratorSpec.cxx | 4 +- 9 files changed, 434 insertions(+), 266 deletions(-) diff --git a/DataFormats/Calibration/CMakeLists.txt b/DataFormats/Calibration/CMakeLists.txt index 189b613f12095..7c905ac350fab 100644 --- a/DataFormats/Calibration/CMakeLists.txt +++ b/DataFormats/Calibration/CMakeLists.txt @@ -10,7 +10,10 @@ # or submit itself to any jurisdiction. o2_add_library(DataFormatsCalibration - SOURCES src/MeanVertexObject.cxx) + SOURCES src/MeanVertexObject.cxx + PUBLIC_LINK_LIBRARIES O2::ReconstructionDataFormats + O2::Framework) o2_target_root_dictionary(DataFormatsCalibration HEADERS include/DataFormatsCalibration/MeanVertexObject.h) + diff --git a/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h b/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h index 2b13ccebda12d..bbda9be7dccc2 100644 --- a/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h +++ b/DataFormats/Calibration/include/DataFormatsCalibration/MeanVertexObject.h @@ -15,29 +15,39 @@ #include #include "Rtypes.h" +#include "Framework/Logger.h" +#include "ReconstructionDataFormats/Vertex.h" + namespace o2 { namespace dataformats { -class MeanVertexObject +class MeanVertexObject : public VertexBase { public: - MeanVertexObject(float x, float y, float z, float sigmax, float sigmay, float sigmaz) + MeanVertexObject(float x, float y, float z, float sigmax, float sigmay, float sigmaz, float slopeX, float slopeY) { - mPos[0] = x; - mPos[1] = y; - mPos[2] = z; - mSigma[0] = sigmax; - mSigma[1] = sigmay; - mSigma[2] = sigmaz; + gpu::gpustd::array cov; + cov[CovElems::kCovXX] = sigmax; + cov[CovElems::kCovYY] = sigmay; + cov[CovElems::kCovZZ] = sigmaz; + setXYZ(x, y, z); + setCov(cov); + mSlopeX = slopeX; + mSlopeY = slopeY; } - MeanVertexObject(std::array pos, std::array sigma) + MeanVertexObject(std::array pos, std::array sigma, float slopeX, float slopeY) { - for (int i = 0; i < 3; i++) { - mPos[i] = pos[i]; - mSigma[i] = sigma[i]; - } + math_utils::Point3D p(pos[0], pos[1], pos[2]); + gpu::gpustd::array cov; + cov[CovElems::kCovXX] = sigma[0]; + cov[CovElems::kCovYY] = sigma[1]; + cov[CovElems::kCovZZ] = sigma[2]; + setPos(p); + setCov(cov); + mSlopeX = slopeX; + mSlopeY = slopeY; } MeanVertexObject() = default; ~MeanVertexObject() = default; @@ -46,42 +56,57 @@ class MeanVertexObject MeanVertexObject& operator=(MeanVertexObject& other) = default; MeanVertexObject& operator=(MeanVertexObject&& other) = default; - void setX(float val) { mPos[0] = val; } - void setY(float val) { mPos[1] = val; } - void setZ(float val) { mPos[2] = val; } - void setPos(std::array val) + void set(int icoord, float val); + void setSigma(int icoord, float val); + void setSigmaX(float val) { setSigmaX2(val); } + void setSigmaY(float val) { setSigmaY2(val); } + void setSigmaZ(float val) { setSigmaZ2(val); } + void setSigma(std::array val) { - for (int i = 0; i < 3; i++) { - mPos[i] = val[i]; - } + setSigmaX2(val[0]); + setSigmaY2(val[1]); + setSigmaZ2(val[2]); } + void setSlopeX(float val) { mSlopeX = val; } + void setSlopeY(float val) { mSlopeY = val; } - float getX() const { return mPos[0]; } - float getY() const { return mPos[1]; } - float getZ() const { return mPos[2]; } - const std::array& getPos() const { return mPos; } + math_utils::Point3D& getPos() { return getXYZ(); } + math_utils::Point3D getPos() const { return getXYZ(); } + float getSigmaX() const { return getSigmaX2(); } + float getSigmaY() const { return getSigmaY2(); } + float getSigmaZ() const { return getSigmaZ2(); } + const gpu::gpustd::array& getSigma() const { return getCov(); } - void setSigmaX(float val) { mSigma[0] = val; } - void setSigmaY(float val) { mSigma[1] = val; } - void setSigmaZ(float val) { mSigma[2] = val; } - void setSigma(std::array val) + float getSlopeX() const { return mSlopeX; } + float getSlopeY() const { return mSlopeY; } + + float getXAtZ(float z) { return getX() + mSlopeX * (z - getZ()); } + float getYAtZ(float z) { return getY() + mSlopeY * (z - getZ()); } + + void print() const; + std::string asString() const; + + VertexBase getMeanVertex(float z) { - for (int i = 0; i < 3; i++) { - mSigma[i] = val[i]; - } + VertexBase v = *this; + v.setXYZ(getXAtZ(z), getYAtZ(z), z); + return v; } - float getSigmaX() const { return mSigma[0]; } - float getSigmaY() const { return mSigma[1]; } - float getSigmaZ() const { return mSigma[2]; } - const std::array& getSigma() const { return mSigma; } + const VertexBase& getMeanVertex() const + { + return (const VertexBase&)(*this); + } private: - std::array mPos; // position of mean vertex - std::array mSigma; // sigma of mean vertex + float mSlopeX{0.f}; // slope of x = f(z) + float mSlopeY{0.f}; // slope of y = f(z) ClassDefNV(MeanVertexObject, 1); }; + +std::ostream& operator<<(std::ostream& os, const o2::dataformats::MeanVertexObject& o); + } // namespace dataformats } // namespace o2 diff --git a/DataFormats/Calibration/src/MeanVertexObject.cxx b/DataFormats/Calibration/src/MeanVertexObject.cxx index 3a08d8eba108a..e9f1be7a15e74 100644 --- a/DataFormats/Calibration/src/MeanVertexObject.cxx +++ b/DataFormats/Calibration/src/MeanVertexObject.cxx @@ -10,3 +10,55 @@ // or submit itself to any jurisdiction. #include "DataFormatsCalibration/MeanVertexObject.h" + +namespace o2 +{ +namespace dataformats +{ + +void MeanVertexObject::set(int icoord, float val) +{ + if (icoord == 0) { + setX(val); + } else if (icoord == 1) { + setY(val); + } else if (icoord == 2) { + setZ(val); + } else { + LOG(fatal) << "Coordinate out of bound to set vtx " << icoord << ", should be in [0, 2]"; + } +} + +void MeanVertexObject::setSigma(int icoord, float val) +{ + if (icoord == 0) { + setSigmaX2(val); + } else if (icoord == 1) { + setSigmaY2(val); + } else if (icoord == 2) { + setSigmaZ2(val); + } else { + LOG(fatal) << "Coordinate out of bound to set sigma via MeanVtx " << icoord << ", should be in [0, 2]"; + } +} + +std::string MeanVertexObject::asString() const +{ + return fmt::format("Slopes {{{:+.4e},{:+.4e}}}", mSlopeX, mSlopeY); +} + +std::ostream& operator<<(std::ostream& os, const o2::dataformats::MeanVertexObject& o) +{ + // stream itself + os << o.asString(); + return os; +} + +void MeanVertexObject::print() const +{ + VertexBase::print(); + std::cout << *this << std::endl; +} + +} // namespace dataformats +} // namespace o2 diff --git a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h index 93438df293a7a..056b1bdfae556 100644 --- a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h +++ b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexCalibrator.h @@ -37,19 +37,26 @@ class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration; public: - MeanVertexCalibrator(int minEnt = 500, bool useFit = false, int nBinsX = 100, float rangeX = 1.f, - int nBinsY = 100, float rangeY = 1.f, int nBinsZ = 100, float rangeZ = 20.f, - int nSlotsSMA = 5) : mMinEntries(minEnt), mUseFit(useFit), mNBinsX(nBinsX), mRangeX(rangeX), mNBinsY(nBinsY), mRangeY(rangeY), mNBinsZ(nBinsZ), mRangeZ(rangeZ), mSMAslots(nSlotsSMA) + MeanVertexCalibrator(int minEnt = 500, int nBinsX = 100, float rangeX = 1.f, + int nBinsY = 100, float rangeY = 1.f, int nBinsZ = 100, float rangeZ = 20.f, uint32_t nPointsForSlope = 10, + int nSlotsSMA = 5) : mMinEntries(minEnt), mNBinsX(nBinsX), mRangeX(rangeX), mNBinsY(nBinsY), mRangeY(rangeY), mNBinsZ(nBinsZ), mRangeZ(rangeZ), mSMAslots(nSlotsSMA), mNPointsForSlope(nPointsForSlope) { - mSMAdata.init(useFit, nBinsX, rangeX, nBinsY, rangeY, nBinsZ, rangeZ); + mBinWidthX = 2 * rangeX / nBinsX; + mBinWidthY = 2 * rangeY / nBinsY; + mBinWidthZ = 2 * rangeZ / nBinsZ; + mBinWidthXInv = 1. / mBinWidthX; + mBinWidthYInv = 1. / mBinWidthY; + mBinWidthZInv = 1. / mBinWidthZ; } ~MeanVertexCalibrator() final = default; bool hasEnoughData(const Slot& slot) const final { - LOG(info) << "container entries = " << slot.getContainer()->entries << ", minEntries = " << mMinEntries; - return slot.getContainer()->entries >= mMinEntries; + if (mVerbose) { + LOG(info) << "container entries = " << slot.getContainer()->entries << ", minEntries = " << mMinEntries * 2; + } + return slot.getContainer()->entries >= mMinEntries * 2; // we need in fact that the min number of entries is 2x the required ones because we will do the slices in z, and we need at least two to fit } void initOutput() final; void finalizeSlot(Slot& slot) final; @@ -68,6 +75,12 @@ class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration& vectOut, const std::vector& vectIn, int nbins, float min, float max, float binWidthInv); + void printVector(std::vector& vect, float minRange, float maxRange, float binWidth); + void printVector(float* vect, int sizeVect, float minRange, float maxRange, float binWidth); + private: int mMinEntries = 0; int mNBinsX = 0; @@ -76,8 +89,14 @@ class MeanVertexCalibrator final : public o2::calibration::TimeSlotCalibration> mTmpMVobjDqTime; // This is the deque for the start and end time of the // slots used for the SMA - std::deque mTmpMVdataDq; // This is the vector of Mean Vertex data to be used for the simple - // moving average - MeanVertexData mSMAdata; // This is to do the SMA when we keep the histos bool mVerbose = false; // Whether to log in verbose mode ClassDefOverride(MeanVertexCalibrator, 1); diff --git a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h index 9caa33e213664..1ce6efeedc4ea 100644 --- a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h +++ b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexData.h @@ -26,52 +26,15 @@ namespace calibration struct MeanVertexData { using PVertex = o2::dataformats::PrimaryVertex; - float rangeX = 1.f; - float rangeY = 1.f; - float rangeZ = 20.f; - int nbinsX = 100; - int nbinsY = 100; - int nbinsZ = 100; - float v2BinX = nbinsX / (2 * rangeX); - float v2BinY = nbinsY / (2 * rangeY); - float v2BinZ = nbinsZ / (2 * rangeZ); int entries = 0; - std::vector histoX{0}; - std::vector histoY{0}; - std::vector histoZ{0}; - bool useFit = false; + std::vector> histoVtx{0}; bool mVerbose = false; MeanVertexData(); ~MeanVertexData() { - useFit = false; - nbinsX = 1000; - rangeX = 10.f; - nbinsY = 1000; - rangeY = 10.f; - nbinsZ = 1000; - rangeZ = 10.f; - v2BinX = 0.f; - v2BinY = 0.f; - v2BinZ = 0.f; - histoX.clear(); - histoY.clear(); - histoZ.clear(); - } - - MeanVertexData(bool buseFit, int nbX, float rX, int nbY, float rY, int nbZ, float rZ) : useFit(buseFit), nbinsX(nbX), rangeX(rX), v2BinX(0), nbinsY(nbY), rangeY(rY), v2BinY(0), nbinsZ(nbZ), rangeZ(rZ), v2BinZ(0) - { - if (rX <= 0. || nbX < 1 || rY <= 0. || nbY < 1 || rZ <= 0. || nbZ < 1) { - throw std::runtime_error("Wrong initialization of the histogram"); - } - v2BinX = nbinsX / (2 * rangeX); - v2BinY = nbinsY / (2 * rangeY); - v2BinZ = nbinsZ / (2 * rangeZ); - histoX.resize(nbinsX, 0.); - histoY.resize(nbinsY, 0.); - histoZ.resize(nbinsZ, 0.); + histoVtx.clear(); } MeanVertexData(MeanVertexData&& other) = default; @@ -79,25 +42,6 @@ struct MeanVertexData { MeanVertexData& operator=(MeanVertexData& other) = default; MeanVertexData& operator=(MeanVertexData&& other) = default; - //_____________________________________________ - void init(bool buseFit, int nbX, float rX, int nbY, float rY, int nbZ, float rZ) - { - - useFit = buseFit; - nbinsX = nbX; - rangeX = rX; - nbinsY = nbY; - rangeY = rY; - nbinsZ = nbZ; - rangeZ = rZ; - v2BinX = nbinsX / (2 * rangeX); - v2BinY = nbinsY / (2 * rangeY); - v2BinZ = nbinsZ / (2 * rangeZ); - histoX.resize(nbinsX, 0.); - histoY.resize(nbinsY, 0.); - histoZ.resize(nbinsZ, 0.); - } - //_____________________________________________ size_t getEntries() const { return entries; } diff --git a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h index b7ee88aee5378..a0cf14800a59c 100644 --- a/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h +++ b/Detectors/Calibration/include/DetectorsCalibration/MeanVertexParams.h @@ -33,9 +33,9 @@ struct MeanVertexParams : public o2::conf::ConfigurableParamHelper; using o2::math_utils::fitGaus; using clbUtils = o2::calibration::Utils; using MeanVertexObject = o2::dataformats::MeanVertexObject; +using CovMatrix = ROOT::Math::SMatrix>; void MeanVertexCalibrator::initOutput() { @@ -35,137 +36,292 @@ void MeanVertexCalibrator::initOutput() } //_____________________________________________ -void MeanVertexCalibrator::finalizeSlot(Slot& slot) +void MeanVertexCalibrator::printVector(float* vect, int sizeVect, float minRange, float maxRange, float binWidth) { - // Extract results for the single slot - o2::calibration::MeanVertexData* c = slot.getContainer(); - LOG(info) << "Finalize slot " << slot.getTFStart() << " <= TF <= " << slot.getTFEnd() << " with " - << c->getEntries() << " entries"; - mTmpMVobjDqTime.emplace_back(slot.getStartTimeMS(), slot.getEndTimeMS()); + for (int i = 0; i < sizeVect; ++i) { + LOG(info) << "i-th bin [" << minRange + i * binWidth << ", " << minRange + (i + 1) * binWidth << "] = " << i << ", content of histogram = " << vect[i]; + } + LOG(info) << "Printing to be used as a vector holding the content"; + for (int i = 0; i < sizeVect; ++i) { + LOG(info) << "vect[" << i << "] = " << vect[i] << ";"; + } + LOG(info) << "Printing to be used to fill a ROOT histogram"; + for (int i = 0; i < sizeVect; ++i) { + if (vect[i] != 0) { + LOG(info) << "h->SetBinContent(" << i + 1 << ", " << vect[i] << ");"; + } + } +} - if (mUseFit) { - MeanVertexObject mvo; - // x coordinate - std::vector fitValues; - float* array = &c->histoX[0]; - if (mVerbose) { - LOG(info) << "**** Printing content of MeanVertex object for x coordinate"; - for (int i = 0; i < c->nbinsX; i++) { - LOG(info) << "i = " << i << ", content of histogram = " << c->histoX[i]; - } +//_____________________________________________ +void MeanVertexCalibrator::printVector(std::vector& vect, float minRange, float maxRange, float binWidth) +{ + printVector(&vect[0], vect.size(), minRange, maxRange, binWidth); +} + +//_____________________________________________ +void MeanVertexCalibrator::binVector(std::vector& vectOut, const std::vector& vectIn, int nbins, float min, float max, float binWidthInv) +{ + vectOut.clear(); + vectOut.resize(nbins); + for (int i = 0; i < vectIn.size(); ++i) { + if (vectIn[i] < min) { + continue; } - double fitres = fitGaus(c->nbinsX, array, -(c->rangeX), c->rangeX, fitValues); - if (fitres >= 0) { - LOG(info) << "X: Fit result (of single Slot) => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; - } else { - LOG(warning) << "X: Fit failed with result = " << fitres; + int bin = (vectIn[i] - min) * binWidthInv; + vectOut[bin]++; + } +} + +//_____________________________________________ +void MeanVertexCalibrator::fitMeanVertex(o2::calibration::MeanVertexData* c, MeanVertexObject& mvo) +{ + // now we do the fits in slices of Z + // we fit as soon as we have enough entries in z + double fitres; + // first we order the vector + std::sort(c->histoVtx.begin(), c->histoVtx.end(), [](std::array a, std::array b) { return b[2] > a[2]; }); + if (mVerbose) { + LOG(info) << "Printing ordered vertices"; + for (int i = 0; i < c->histoVtx.size(); ++i) { + LOG(info) << "x = " << c->histoVtx[i][0] << ", y = " << c->histoVtx[i][1] << ", z = " << c->histoVtx[i][2]; } - mvo.setX(fitValues[1]); - mvo.setSigmaX(fitValues[2]); + } - // y coordinate - array = &c->histoY[0]; + std::vector htmpX; + std::vector htmpY; + std::vector htmpZ; + std::vector> fitResSlicesX; + std::vector covMatrixX; + std::vector> fitResSlicesY; + std::vector covMatrixY; + std::vector binnedVect; + std::vector meanZvect; + int startZ = 0; + int counter = 0; + auto minEntriesPerPoint = std::max((unsigned long int)mMinEntries, c->histoVtx.size() / mNPointsForSlope); + if (mVerbose) { + LOG(info) << "Beginning: startZ = " << startZ << " c->histoVtx.size() = " << c->histoVtx.size(); + } + while (startZ <= c->histoVtx.size()) { if (mVerbose) { - LOG(info) << "**** Printing content of MeanVertex object for y coordinate"; - for (int i = 0; i < c->nbinsY; i++) { - LOG(info) << "i = " << i << ", content of histogram = " << c->histoY[i]; - } - } - fitres = fitGaus(c->nbinsY, array, -(c->rangeY), c->rangeY, fitValues); - if (fitres >= 0) { - LOG(info) << "Y: Fit result (of single Slot) => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; - } else { - LOG(warning) << "Y: Fit failed with result = " << fitres; + LOG(info) << "Beginning of while: startZ = " << startZ << " c->histoVtx.size() = " << c->histoVtx.size(); } - mvo.setY(fitValues[1]); - mvo.setSigmaY(fitValues[2]); + double meanZ = 0; + int counts = 0; + for (int ii = startZ; ii <= c->histoVtx.size(); ++ii) { + if (mVerbose) { + // LOG(info) << "htmpX.size() = " << htmpX.size() << " ii = " << ii << " c->histoVtx.size() = " << c->histoVtx.size(); + } + if (htmpX.size() < minEntriesPerPoint) { + if (mVerbose) { + // LOG(info) << "filling X with c->histoVtx[" << ii << "][0] = " << c->histoVtx[ii][0]; + // LOG(info) << "filling Y with c->histoVtx[" << ii << "][0] = " << c->histoVtx[ii][1]; + } + htmpX.push_back(c->histoVtx[ii][0]); + htmpY.push_back(c->histoVtx[ii][1]); + meanZ += c->histoVtx[ii][2]; + ++counts; + } else { + if (mVerbose) { + LOG(info) << "fitting "; + } + // we can fit and restart filling + // X: + fitResSlicesX.push_back({}); + covMatrixX.push_back({}); + if (mVerbose) { + LOG(info) << "Fitting X for counter " << counter << ", will use " << mNBinsX << " bins, from " << -mRangeX << " to " << mRangeX; + for (int i = 0; i < htmpX.size(); ++i) { + LOG(info) << "vect[" << i << "] = " << htmpX[i] << ";"; + } + } + binVector(binnedVect, htmpX, mNBinsX, -mRangeX, mRangeX, mBinWidthXInv); + if (mVerbose) { + LOG(info) << " Printing output binned vector for X:"; + printVector(binnedVect, -(mRangeX), mRangeX, mBinWidthX); + } + fitres = fitGaus(mNBinsX, &binnedVect[0], -(mRangeX), mRangeX, fitResSlicesX.back(), &covMatrixX.back()); + if (fitres != 10) { + LOG(info) << "X, counter " << counter << ": Fit result (z slice [" << c->histoVtx[startZ][2] << ", " << c->histoVtx[ii][2] << "[) => " << fitres << ". Mean = " << fitResSlicesX[counter][1] << " Sigma = " << fitResSlicesX[counter][2] << ", covMatrix = " << covMatrixX[counter](2, 2); + } else { + LOG(error) << "X, counter " << counter << ": Fit failed with result = " << fitres; + } + htmpX.clear(); - // z coordinate - array = &c->histoZ[0]; - if (mVerbose) { - LOG(info) << "**** Printing content of MeanVertex object for z coordinate"; - for (int i = 0; i < c->nbinsZ; i++) { - LOG(info) << "i = " << i << ", content of histogram = " << c->histoZ[i]; + // Y: + fitResSlicesY.push_back({}); + covMatrixY.push_back({}); + if (mVerbose) { + LOG(info) << "Fitting Y for counter " << counter << ", will use " << mNBinsY << " bins, from " << -(mRangeY) << " to " << mRangeY; + for (int i = 0; i < htmpY.size(); ++i) { + LOG(info) << i << " : " << htmpY[i]; + } + } + binnedVect.clear(); + binVector(binnedVect, htmpY, mNBinsY, -(mRangeY), mRangeY, mBinWidthYInv); + if (mVerbose) { + LOG(info) << " Printing output binned vector for Y:"; + printVector(binnedVect, -(mRangeY), mRangeY, mBinWidthY); + } + fitres = fitGaus(mNBinsY, &binnedVect[0], -(mRangeY), mRangeY, fitResSlicesY.back(), &covMatrixY.back()); + if (fitres != 10) { + LOG(info) << "Y, counter " << counter << ": Fit result (z slice [" << c->histoVtx[startZ][2] << ", " << c->histoVtx[ii][2] << "[) => " << fitres << ". Mean = " << fitResSlicesY[counter][1] << " Sigma = " << fitResSlicesY[counter][2] << ", covMatrix = " << covMatrixY[counter](2, 2); + } else { + LOG(error) << "Y, counter " << counter << ": Fit failed with result = " << fitres; + } + htmpY.clear(); + + // Z: let's calculate the mean position + if (mVerbose) { + LOG(info) << "Z, counter " << counter << ": " << meanZ / counts; + } + ++counter; + meanZvect.push_back(meanZ / counts); + break; } } - fitres = fitGaus(c->nbinsZ, array, -(c->rangeZ), c->rangeZ, fitValues); - if (fitres >= 0) { - LOG(info) << "Z: Fit result (of single Slot) => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; - } else { - LOG(warning) << "Z: Fit failed with result = " << fitres; - } - mvo.setZ(fitValues[1]); - mvo.setSigmaZ(fitValues[2]); - - // now we add the object to the deque - mTmpMVobjDq.push_back(std::move(mvo)); - } else { - mTmpMVdataDq.push_back(std::move(*c)); - mSMAdata.merge(&mTmpMVdataDq.back()); - if (mTmpMVobjDqTime.size() > mSMAslots) { - mSMAdata.subtract(&mTmpMVdataDq.front()); - mTmpMVdataDq.pop_front(); + startZ += mMinEntries * counter; + if (mVerbose) { + LOG(info) << "End of while: startZ = " << startZ << " c->histoVtx.size() = " << c->histoVtx.size(); } } - // output object - MeanVertexObject mvo; + // fitting main mean vtx Z + for (int ii = 0; ii < c->histoVtx.size(); ++ii) { + htmpZ.push_back(c->histoVtx[ii][2]); + } + binVector(binnedVect, htmpZ, mNBinsZ, -(mRangeZ), mRangeZ, mBinWidthZInv); + fitMeanVertexCoord(2, mNBinsZ, &binnedVect[0], -(mRangeZ), mRangeZ, mvo); + htmpZ.clear(); + binnedVect.clear(); - if (mUseFit) { - doSimpleMovingAverage(mTmpMVobjDq, mSMAMVobj); - } else { - // now we need to fit, on the merged data + // now we update the error on x + double sumX = 0, sumY = 0, weightSumX = 0, weightSumY = 0; + for (int iFit = 0; iFit < counter; ++iFit) { if (mVerbose) { - LOG(info) << "**** Printing content of SMA MVData object for x coordinate"; - for (int i = 0; i < mSMAdata.nbinsX; i++) { - LOG(info) << "i = " << i << ", content of histogram = " << mSMAdata.histoX[i]; - } + LOG(info) << "SigmaX = " << fitResSlicesX[iFit][2] << " error = " << covMatrixX[iFit](2, 2); + LOG(info) << "SigmaY = " << fitResSlicesY[iFit][2] << " error = " << covMatrixY[iFit](2, 2); } - std::vector fitValues; - float* array = &mSMAdata.histoX[0]; - double fitres = fitGaus(mSMAdata.nbinsX, array, -(mSMAdata.rangeX), mSMAdata.rangeX, fitValues); - if (fitres >= 0) { - LOG(info) << "X: Fit result (of merged Slots) => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; - } else { - LOG(warning) << "X: Fit failed with result = " << fitres; + if (covMatrixX[iFit](2, 2) != 0) { + double weightSigma = 1. / covMatrixX[iFit](2, 2); // covMatrix is already an error squared + sumX += (fitResSlicesX[iFit][2] * weightSigma); + weightSumX += weightSigma; + } + if (covMatrixY[iFit](2, 2) != 0) { + double weightSigma = 1. / covMatrixY[iFit](2, 2); // covMatrix is already an error squared + sumY += (fitResSlicesY[iFit][2] * weightSigma); + weightSumY += weightSigma; } - mSMAMVobj.setX(fitValues[1]); - mSMAMVobj.setSigmaX(fitValues[2]); + } + if (mVerbose) { + LOG(info) << "sumX = " << sumX; + LOG(info) << "weightSumX = " << weightSumX; + LOG(info) << "sumY = " << sumY; + LOG(info) << "weightSumY = " << weightSumY; + } + + double sigmaX = 0; + if (weightSumX != 0) { + sigmaX = sumX / weightSumX; + } + double sigmaY = 0; + if (weightSumY != 0) { + sigmaY = sumY / weightSumY; + } + if (mVerbose) { + LOG(info) << "SigmaX for MeanVertex = " << sigmaX; + LOG(info) << "SigmaY for MeanVertex = " << sigmaY; + } + mvo.setSigmaX(sigmaX); + mvo.setSigmaY(sigmaY); - // y coordinate + // now we get the slope for the x-coordinate dependence on z + TLinearFitter lf(1, "pol1"); + lf.StoreData(kFALSE); + for (int i = 0; i < fitResSlicesX.size(); ++i) { if (mVerbose) { - LOG(info) << "**** Printing content of SMA MVData object for y coordinate"; - for (int i = 0; i < mSMAdata.nbinsY; i++) { - LOG(info) << "i = " << i << ", content of histogram = " << mSMAdata.histoY[i]; - } - } - array = &mSMAdata.histoY[0]; - fitres = fitGaus(mSMAdata.nbinsY, array, -(mSMAdata.rangeY), mSMAdata.rangeY, fitValues); - if (fitres >= 0) { - LOG(info) << "Y: Fit result (of merged Slots) => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; - } else { - LOG(warning) << "Y: Fit failed with result = " << fitres; + LOG(info) << "Adding point " << i << ": zvtx = " << meanZvect[i] << " xvtx = " << fitResSlicesX[i][2]; } - mSMAMVobj.setY(fitValues[1]); - mSMAMVobj.setSigmaY(fitValues[2]); + lf.AddPoint(&meanZvect[i], fitResSlicesX[i][1]); + } + lf.Eval(); + double slopeX = lf.GetParameter(1); + mvo.setSlopeX(slopeX); + mvo.setX(mvo.getZ() * slopeX + lf.GetParameter(0)); + lf.ClearPoints(); - // z coordinate + // now slope for the y-coordinate dependence on z + for (int i = 0; i < fitResSlicesX.size(); ++i) { if (mVerbose) { - LOG(info) << "**** Printing content of SMA MVData object for z coordinate"; - for (int i = 0; i < mSMAdata.nbinsZ; i++) { - LOG(info) << "i = " << i << ", content of histogram = " << mSMAdata.histoZ[i]; - } + LOG(info) << "Adding point " << i << ": zvtx = " << meanZvect[i] << " yvtx = " << fitResSlicesY[i][2]; } - array = &mSMAdata.histoZ[0]; - fitres = fitGaus(mSMAdata.nbinsZ, array, -(mSMAdata.rangeZ), mSMAdata.rangeZ, fitValues); - if (fitres >= 0) { - LOG(info) << "Z: Fit result (of merged Slots) => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; + lf.AddPoint(&meanZvect[i], fitResSlicesY[i][1]); + } + lf.Eval(); + double slopeY = lf.GetParameter(1); + mvo.setSlopeY(slopeY); + mvo.setY(mvo.getZ() * slopeY + lf.GetParameter(0)); + if (mVerbose) { + LOG(info) << "slope X = " << slopeX; + LOG(info) << "slope Y = " << slopeY; + } +} +//_____________________________________________ +void MeanVertexCalibrator::fitMeanVertexCoord(int icoord, int nbins, float* array, float minRange, float maxRange, MeanVertexObject& mvo) +{ + // fit mean vertex coordinate icoord + std::vector fitValues; + float binWidth = 0; + if (mVerbose) { + LOG(info) << "**** Printing content of MeanVertex object for coordinate " << icoord; + if (icoord == 0) { + binWidth = mBinWidthX; + } else if (icoord == 1) { + binWidth = mBinWidthY; } else { - LOG(warning) << "Z: Fit failed with result = " << fitres; + binWidth = mBinWidthZ; } - mSMAMVobj.setZ(fitValues[1]); - mSMAMVobj.setSigmaZ(fitValues[2]); + printVector(array, nbins, minRange, maxRange, binWidth); } + double fitres = fitGaus(nbins, array, minRange, maxRange, fitValues); + if (fitres != -4) { + LOG(info) << "coordinate " << icoord << ": Fit result of full statistics => " << fitres << ". Mean = " << fitValues[1] << " Sigma = " << fitValues[2]; + } else { + LOG(error) << "coordinate " << icoord << ": Fit failed with result = " << fitres; + } + switch (icoord) { + case 0: + mvo.setX(fitValues[1]); + mvo.setSigmaX(fitValues[2]); + break; + case 1: + mvo.setY(fitValues[1]); + mvo.setSigmaY(fitValues[2]); + break; + case 2: + mvo.setZ(fitValues[1]); + mvo.setSigmaZ(fitValues[2]); + break; + } +} + +//_____________________________________________ +void MeanVertexCalibrator::finalizeSlot(Slot& slot) +{ + // Extract results for the single slot + o2::calibration::MeanVertexData* c = slot.getContainer(); + LOG(info) << "Finalize slot " << slot.getTFStart() << " <= TF <= " << slot.getTFEnd() << " with " + << c->getEntries() << " entries"; + mTmpMVobjDqTime.emplace_back(slot.getStartTimeMS(), slot.getEndTimeMS()); + MeanVertexObject mvo; + // fitting + fitMeanVertex(c, mvo); + // now we add the object to the deque + mTmpMVobjDq.push_back(std::move(mvo)); + + // moving average + doSimpleMovingAverage(mTmpMVobjDq, mSMAMVobj); if (mTmpMVobjDqTime.size() > mSMAslots) { mTmpMVobjDqTime.pop_front(); @@ -177,6 +333,10 @@ void MeanVertexCalibrator::finalizeSlot(Slot& slot) auto flName = o2::ccdb::CcdbApi::generateFileName(clName); mInfoVector.emplace_back("GLO/Calib/MeanVertex", clName, flName, md, startValidity - 10 * o2::ccdb::CcdbObjectInfo::SECOND, startValidity + o2::ccdb::CcdbObjectInfo::MONTH); mMeanVertexVector.emplace_back(mSMAMVobj); + if (mVerbose) { + LOG(info) << "Printing MeanVertex Object:"; + mSMAMVobj.print(); + } slot.print(); } @@ -214,6 +374,12 @@ void MeanVertexCalibrator::doSimpleMovingAverage(std::deque& dq, MVObj sma.setSigmaX((sma.getSigmaX() * (dq.size() - 1) + dq.back().getSigmaX()) / dq.size()); sma.setSigmaY((sma.getSigmaY() * (dq.size() - 1) + dq.back().getSigmaY()) / dq.size()); sma.setSigmaZ((sma.getSigmaZ() * (dq.size() - 1) + dq.back().getSigmaZ()) / dq.size()); + sma.setSlopeX((sma.getSlopeX() * (dq.size() - 1) + dq.back().getSlopeX()) / dq.size()); + sma.setSlopeY((sma.getSlopeY() * (dq.size() - 1) + dq.back().getSlopeY()) / dq.size()); + if (mVerbose) { + LOG(info) << "Printing from simple moving average, when we have not collected enough objects yet:"; + sma.print(); + } return; } @@ -225,9 +391,16 @@ void MeanVertexCalibrator::doSimpleMovingAverage(std::deque& dq, MVObj sma.setSigmaX(sma.getSigmaX() + (dq[dq.size() - 1].getSigmaX() - dq[0].getSigmaX()) / mSMAslots); sma.setSigmaY(sma.getSigmaY() + (dq[dq.size() - 1].getSigmaY() - dq[0].getSigmaY()) / mSMAslots); sma.setSigmaZ(sma.getSigmaZ() + (dq[dq.size() - 1].getSigmaZ() - dq[0].getSigmaZ()) / mSMAslots); + sma.setSlopeX(sma.getSlopeX() + (dq[dq.size() - 1].getSlopeX() - dq[0].getSlopeX()) / mSMAslots); + sma.setSlopeY(sma.getSlopeY() + (dq[dq.size() - 1].getSlopeY() - dq[0].getSlopeY()) / mSMAslots); dq.pop_front(); + if (mVerbose) { + LOG(info) << "Printing from simple moving average:"; + sma.print(); + } + return; } @@ -236,7 +409,8 @@ Slot& MeanVertexCalibrator::emplaceNewSlot(bool front, TFType tstart, TFType ten { auto& cont = getSlots(); auto& slot = front ? cont.emplace_front(tstart, tend) : cont.emplace_back(tstart, tend); - slot.setContainer(std::make_unique(mUseFit, mNBinsX, mRangeX, mNBinsY, mRangeY, mNBinsZ, mRangeZ)); + // slot.setContainer(std::make_unique(mNBinsX, mRangeX, mNBinsY, mRangeY, mNBinsZ, mRangeZ)); + slot.setContainer(std::make_unique()); return slot; } diff --git a/Detectors/Calibration/src/MeanVertexData.cxx b/Detectors/Calibration/src/MeanVertexData.cxx index d28976735d9e9..1ad4bbceabbcc 100644 --- a/Detectors/Calibration/src/MeanVertexData.cxx +++ b/Detectors/Calibration/src/MeanVertexData.cxx @@ -52,34 +52,8 @@ void MeanVertexData::fill(const gsl::span data) if (mVerbose) { LOG(info) << "i = " << i << " --> x = " << x << ", y = " << y << ", z = " << z; } - auto dx = x + rangeX; - uint32_t binx = dx < 0 ? 0xffffffff : (x + rangeX) * v2BinX; - auto dy = y + rangeY; - uint32_t biny = dy < 0 ? 0xffffffff : (y + rangeY) * v2BinY; - auto dz = z + rangeZ; - uint32_t binz = dz < 0 ? 0xffffffff : (z + rangeZ) * v2BinZ; - if (mVerbose) { - LOG(info) << "dx = " << dx << ", dy = " << dy << ", dz = " << dz; - LOG(info) << "rangeX = " << rangeX << ", rangeY = " << rangeY << ", rangeZ = " << rangeZ; - LOG(info) << "v2BinX = " << v2BinX << ", v2BinY = " << v2BinY << ", v2BinZ = " << v2BinZ; - LOG(info) << "binx = " << binx << ", biny = " << biny << ", binz = " << binz; - } - if (binx < nbinsX && biny < nbinsY && binz < nbinsZ) { // do not account for vertices outside the histo ranges - histoX[binx]++; - histoY[biny]++; - histoZ[binz]++; - entries++; - } - } - - for (int i = 0; i < histoX.size(); i++) { - LOG(debug) << "histoX, bin " << i << ": entries = " << histoX[i]; - } - for (int i = 0; i < histoY.size(); i++) { - LOG(debug) << "histoY, bin " << i << ": entries = " << histoY[i]; - } - for (int i = 0; i < histoZ.size(); i++) { - LOG(debug) << "histoZ, bin " << i << ": entries = " << histoZ[i]; + entries++; + histoVtx.push_back({x, y, z}); } } @@ -88,19 +62,7 @@ void MeanVertexData::subtract(const MeanVertexData* prev) { // remove entries from prev - assert(histoX.size() == prev->histoX.size()); - assert(histoY.size() == prev->histoY.size()); - assert(histoZ.size() == prev->histoZ.size()); - - for (int i = histoX.size(); i--;) { - histoX[i] -= prev->histoX[i]; - } - for (int i = histoY.size(); i--;) { - histoY[i] -= prev->histoY[i]; - } - for (int i = histoZ.size(); i--;) { - histoZ[i] -= prev->histoZ[i]; - } + histoVtx.erase(histoVtx.begin(), histoVtx.begin() + prev->entries); entries -= prev->entries; } @@ -108,15 +70,7 @@ void MeanVertexData::subtract(const MeanVertexData* prev) void MeanVertexData::merge(const MeanVertexData* prev) { // merge data of 2 slots - assert(histoX.size() == prev->histoX.size()); - assert(histoY.size() == prev->histoY.size()); - assert(histoZ.size() == prev->histoZ.size()); - - for (int i = histoX.size(); i--;) { - histoX[i] += prev->histoX[i]; - histoY[i] += prev->histoY[i]; - histoZ[i] += prev->histoZ[i]; - } + histoVtx.insert(histoVtx.end(), prev->histoVtx.begin(), prev->histoVtx.end()); entries += prev->entries; } diff --git a/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx b/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx index fec9d7f75d779..bfe0efc9e378c 100644 --- a/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx +++ b/Detectors/Calibration/workflow/src/MeanVertexCalibratorSpec.cxx @@ -39,10 +39,10 @@ void MeanVertexCalibDevice::init(InitContext& ic) int nbZ = params->nbinsZ; float rangeZ = params->rangeZ; int nSlots4SMA = params->nSlots4SMA; - bool useFit = params->useFit; auto slotL = params->tfPerSlot; auto delay = params->maxTFdelay; - mCalibrator = std::make_unique(minEnt, useFit, nbX, rangeX, nbY, rangeY, nbZ, rangeZ, nSlots4SMA); + auto nPointsForSlope = params->nPointsForSlope; + mCalibrator = std::make_unique(minEnt, nbX, rangeX, nbY, rangeY, nbZ, rangeZ, nSlots4SMA, nPointsForSlope); mCalibrator->setSlotLength(slotL); mCalibrator->setMaxSlotsDelay(delay); bool useVerboseMode = ic.options().get("use-verbose-mode"); From 85d5fe1b40bd51768377f2ffa441aca747bcb537 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 27 Oct 2022 09:12:58 +0200 Subject: [PATCH 003/289] Reduce verbosity to InfoLogger --- Detectors/GlobalTracking/src/MatchGlobalFwd.cxx | 8 ++++---- Detectors/TRD/reconstruction/src/CruRawReader.cxx | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Detectors/GlobalTracking/src/MatchGlobalFwd.cxx b/Detectors/GlobalTracking/src/MatchGlobalFwd.cxx index 2106b8b636272..9f68908cf66e9 100644 --- a/Detectors/GlobalTracking/src/MatchGlobalFwd.cxx +++ b/Detectors/GlobalTracking/src/MatchGlobalFwd.cxx @@ -146,14 +146,14 @@ bool MatchGlobalFwd::prepareMCHData() } mMCHWork.reserve(mMCHTracks.size()); static int BCDiffErrCount = 0; - constexpr int MAXBCDiffErrCount = 5; + constexpr int MAXBCDiffErrCount = 2; for (int irof = 0; irof < nROFs; irof++) { const auto& rofRec = mMCHTrackROFRec[irof]; int nBC = rofRec.getBCData().differenceInBC(mStartIR); if (nBC < 0) { - if (BCDiffErrCount < MAXBCDiffErrCount) { + if (BCDiffErrCount++ < MAXBCDiffErrCount) { LOGP(alarm, "wrong bunches diff. {} for current IR {} wrt 1st TF orbit {} in MCH data", nBC, rofRec.getBCData().asString(), mStartIR.asString()); } } @@ -242,14 +242,14 @@ bool MatchGlobalFwd::prepareMFTData() mMFTWork.reserve(mMFTTracks.size()); static int BCDiffErrCount = 0; - constexpr int MAXBCDiffErrCount = 5; + constexpr int MAXBCDiffErrCount = 2; for (int irof = 0; irof < nROFs; irof++) { const auto& rofRec = mMFTTrackROFRec[irof]; int nBC = rofRec.getBCData().differenceInBC(mStartIR); if (nBC < 0) { - if (BCDiffErrCount < MAXBCDiffErrCount) { + if (BCDiffErrCount++ < MAXBCDiffErrCount) { LOGP(alarm, "wrong bunches diff. {} for current IR {} wrt 1st TF orbit {} in MFT data", nBC, rofRec.getBCData().asString(), mStartIR.asString()); } } diff --git a/Detectors/TRD/reconstruction/src/CruRawReader.cxx b/Detectors/TRD/reconstruction/src/CruRawReader.cxx index 6ff62edaa48ab..903c091a6f5a7 100644 --- a/Detectors/TRD/reconstruction/src/CruRawReader.cxx +++ b/Detectors/TRD/reconstruction/src/CruRawReader.cxx @@ -233,7 +233,7 @@ void CruRawReader::checkDigitHCHeader(int halfChamberIdRef) if (halfChamberIdRef != halfChamberIdHeader) { incrementErrors(DigitHCHeaderMismatch, halfChamberIdRef, fmt::format("HCID mismatch detected. HCID from DigitHCHeader: {}, HCID from RDH: {}", halfChamberIdHeader, halfChamberIdRef)); if (mMaxWarnPrinted > 0) { - LOGF(alarm, "HCID mismatch in DigitHCHeader detected for ref HCID %i. DigitHCHeader says HCID is %i", halfChamberIdRef, halfChamberIdHeader); + LOGF(warning, "HCID mismatch in DigitHCHeader detected for ref HCID %i. DigitHCHeader says HCID is %i", halfChamberIdRef, halfChamberIdHeader); checkNoWarn(); } } From 91588db67dfd3fde030d5ab9bc833ae3410ed268 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 27 Oct 2022 09:15:23 +0200 Subject: [PATCH 004/289] Increase MCH downscaling --- prodtests/full-system-test/dpl-workflow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh index 042f0f1e22266..a0e0260a19b78 100755 --- a/prodtests/full-system-test/dpl-workflow.sh +++ b/prodtests/full-system-test/dpl-workflow.sh @@ -237,7 +237,7 @@ if has_processing_step MUON_SYNC_RECO; then elif [[ $RUNTYPE == "SYNTHETIC" ]]; then CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow="MCHTimeClusterizer.peakSearchSignalOnly=false;MCHDigitFilter.rejectBackground=false;MCHClustering.defaultClusterResolution=0.4;MCHTracking.chamberResolutionX=0.4;MCHTracking.chamberResolutionY=0.4;MCHTracking.sigmaCutForTracking=7.;MCHTracking.sigmaCutForImprovement=6.;" fi - has_detector_reco ITS && [[ $RUNTYPE != "COSMICS" ]] && CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow+="MCHTimeClusterizer.irFramesOnly=true;" + has_detector_reco ITS && [[ $RUNTYPE != "COSMICS" ]] && CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow+="MCHTimeClusterizer.irFramesOnly=true;MCHTimeClusterizer.rofRejectionFraction=0.7;" fi [[ $RUNTYPE == "COSMICS" ]] && [[ -z $CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow ]] && CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow="MFTTracking.FullClusterScan=true" fi From 55c09f9ee0a899d64629f808c8e878d07bef8524 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 27 Oct 2022 09:41:26 +0200 Subject: [PATCH 005/289] More settings can be taken from multiplicity-config --- prodtests/full-system-test/dpl-workflow.sh | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh index a0e0260a19b78..7bc860143c720 100755 --- a/prodtests/full-system-test/dpl-workflow.sh +++ b/prodtests/full-system-test/dpl-workflow.sh @@ -64,6 +64,7 @@ TOF_CONFIG= TOF_INPUT=raw TOF_OUTPUT=clusters ITS_CONFIG_KEY= +MFT_CONFIG= MFT_CONFIG_KEY= TRD_CONFIG= TRD_CONFIG_KEY= @@ -87,11 +88,7 @@ if [[ $SYNCMODE == 1 ]]; then ITS_CONFIG_KEY+="fastMultConfig.cutMultClusLow=30;fastMultConfig.cutMultClusHigh=2000;fastMultConfig.cutMultVtxHigh=500;" elif [[ $BEAMTYPE == "pp" ]]; then ITS_CONFIG_KEY+="fastMultConfig.cutMultClusLow=-1;fastMultConfig.cutMultClusHigh=-1;fastMultConfig.cutMultVtxHigh=-1;ITSVertexerParam.phiCut=0.5;ITSVertexerParam.clusterContributorsCut=3;ITSVertexerParam.tanLambdaCut=0.2;" - if [[ "0$HIGH_RATE_PP" == "01" ]]; then - ITS_CONFIG_KEY+="fastMultConfig.cutRandomFraction=0.97;" - else - ITS_CONFIG_KEY+="fastMultConfig.cutRandomFraction=0.9;" - fi + [[ ! -z $CUT_RANDOM_FRACTION_ITS ]] && ITS_CONFIG_KEY+="fastMultConfig.cutRandomFraction=$CUT_RANDOM_FRACTION_ITS;" fi if has_detector_reco ITS; then MFT_CONFIG_KEY+="MFTTracking.irFramesOnly=1;" @@ -113,6 +110,7 @@ else fi [[ ! -z $NTRDTRKTHREADS ]] && TRD_CONFIG_KEY+="GPU_proc.ompThreads=$NTRDTRKTHREADS;" [[ ! -z $NGPURECOTHREADS ]] && GPU_CONFIG_KEY+="GPU_proc.ompThreads=$NGPURECOTHREADS;" +[[ ! -z $NMFTTHREADS ]] && MFT_CONFIG+=" --nThreads $NMFTTHREADS" if [[ $BEAMTYPE == "PbPb" ]]; then PVERTEXING_CONFIG_KEY+="pvertexer.maxChi2TZDebris=2000;" @@ -237,7 +235,8 @@ if has_processing_step MUON_SYNC_RECO; then elif [[ $RUNTYPE == "SYNTHETIC" ]]; then CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow="MCHTimeClusterizer.peakSearchSignalOnly=false;MCHDigitFilter.rejectBackground=false;MCHClustering.defaultClusterResolution=0.4;MCHTracking.chamberResolutionX=0.4;MCHTracking.chamberResolutionY=0.4;MCHTracking.sigmaCutForTracking=7.;MCHTracking.sigmaCutForImprovement=6.;" fi - has_detector_reco ITS && [[ $RUNTYPE != "COSMICS" ]] && CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow+="MCHTimeClusterizer.irFramesOnly=true;MCHTimeClusterizer.rofRejectionFraction=0.7;" + has_detector_reco ITS && [[ $RUNTYPE != "COSMICS" ]] && CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow+="MCHTimeClusterizer.irFramesOnly=true;" + [[ ! -z $CUT_RANDOM_FRACTION_MCH ]] && CONFIG_EXTRA_PROCESS_o2_mch_reco_workflow+="MCHTimeClusterizer.rofRejectionFraction=$CUT_RANDOM_FRACTION_MCH;" fi [[ $RUNTYPE == "COSMICS" ]] && [[ -z $CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow ]] && CONFIG_EXTRA_PROCESS_o2_mft_reco_workflow="MFTTracking.FullClusterScan=true" fi @@ -366,7 +365,7 @@ has_detectors TPC && [ -z "$DISABLE_ROOT_OUTPUT" ] && add_W o2-tpc-reco-workflow # Reconstruction workflows normally active only in async mode in async mode ($LIST_OF_ASYNC_RECO_STEPS), but can be forced via $WORKFLOW_EXTRA_PROCESSING_STEPS has_detector MID && has_processing_step MID_RECO && add_W o2-mid-reco-workflow "$DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N MIDClusterizer MID REST 1),$(get_N MIDTracker MID REST 1)" has_detector MCH && has_processing_step MCH_RECO && add_W o2-mch-reco-workflow "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N mch-track-finder MCH REST 1 MCHTRK),$(get_N mch-cluster-finder MCH REST 1 MCHCL),$(get_N mch-cluster-transformer MCH REST 1)" "$MCH_CONFIG_KEY" -has_detector MFT && has_processing_step MFT_RECO && add_W o2-mft-reco-workflow "$DISABLE_DIGIT_CLUSTER_INPUT $DISABLE_MC $DISABLE_ROOT_OUTPUT --pipeline $(get_N mft-tracker MFT REST 1 MFTTRK)" "$MFT_CONFIG_KEY;$ITSMFT_STROBES" +has_detector MFT && has_processing_step MFT_RECO && add_W o2-mft-reco-workflow "$DISABLE_DIGIT_CLUSTER_INPUT $DISABLE_MC $DISABLE_ROOT_OUTPUT $MFT_CONFIG --pipeline $(get_N mft-tracker MFT REST 1 MFTTRK)" "$MFT_CONFIG_KEY;$ITSMFT_STROBES" has_detector FDD && has_processing_step FDD_RECO && add_W o2-fdd-reco-workflow "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC" has_detector FV0 && has_processing_step FV0_RECO && add_W o2-fv0-reco-workflow "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC" has_detector ZDC && has_processing_step ZDC_RECO && add_W o2-zdc-digits-reco "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC" From 892d23d32ed97d45674ca3c0bb650e833ec6fb8e Mon Sep 17 00:00:00 2001 From: cortesep <57937610+cortesep@users.noreply.github.com> Date: Mon, 14 Nov 2022 09:32:34 +0100 Subject: [PATCH 006/289] ZDC - Better error reporting during data taking (#10262) * Correct cabling for modules 6 and 7 (ZPC towers) * Improving warning and error messages * Improved formatting * Improved formatting --- .../ZDC/include/DataFormatsZDC/ZDCTDCData.h | 83 ++++++++++++++++--- DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx | 4 + Detectors/ZDC/macro/CreateModuleConfig.C | 8 +- Detectors/ZDC/raw/src/RawReaderZDC.cxx | 32 +++---- Detectors/ZDC/reconstruction/src/DigiReco.cxx | 5 +- 5 files changed, 101 insertions(+), 31 deletions(-) diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h index b159ccdb1a3a2..4e3d0bc9f9d4f 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/ZDCTDCData.h @@ -27,6 +27,30 @@ namespace o2 namespace zdc { +struct ZDCTDCDataErr { + + static uint32_t mErrVal[NTDCChannels]; // Errors in encoding TDC values + static uint32_t mErrAmp[NTDCChannels]; // Errors in encoding TDC amplitudes + static uint32_t mErrId; // Errors with TDC Id + + static void print() + { + if (mErrId > 0) { + LOG(error) << "TDCId was out of range #times = " << mErrId; + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + if (mErrVal[itdc] > 0) { + LOG(error) << "TDCVal itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " was out of range #times = " << mErrVal[itdc]; + } + } + for (int itdc = 0; itdc < NTDCChannels; itdc++) { + if (mErrAmp[itdc] > 0) { + LOG(warning) << "TDCAmp itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " was out of range #times = " << mErrAmp[itdc]; + } + } + } +}; + struct ZDCTDCData { uint8_t id = 0xff; // channel ID @@ -37,42 +61,81 @@ struct ZDCTDCData { ZDCTDCData(uint8_t ida, int16_t vala, int16_t ampa, bool isbeg = false, bool isend = false) { // TDC value and amplitude are encoded externally - id = ida & 0x0f; + id = ida < NTDCChannels ? ida : 0xf; id = id | (isbeg ? 0x80 : 0x00); id = id | (isend ? 0x40 : 0x00); - val = vala; - amp = ampa; + + if (ida < NTDCChannels) { + val = vala; + amp = ampa; + } else { + val = kMaxShort; + amp = kMaxShort; +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDC Id = " << int(ida) << " is out of range"; +#endif + ZDCTDCDataErr::mErrId++; + } } ZDCTDCData(uint8_t ida, float vala, float ampa, bool isbeg = false, bool isend = false) { - // TDC value and amplitude are encoded externally - id = ida & 0x0f; + // TDC value and amplitude are encoded externally but argument is float + id = ida < NTDCChannels ? ida : 0xf; id = id | (isbeg ? 0x80 : 0x00); id = id | (isend ? 0x40 : 0x00); + if (ida >= NTDCChannels) { + val = kMaxShort; + amp = kMaxShort; +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDC Id = " << int(ida) << " is out of range"; +#endif + ZDCTDCDataErr::mErrId++; + return; + } + auto TDCVal = std::nearbyint(vala); auto TDCAmp = std::nearbyint(ampa); if (TDCVal < kMinShort) { - LOG(error) << __func__ << " TDCVal " << int(ida) << " " << ChannelNames[ida] << " = " << TDCVal << " is out of range"; + int itdc = int(id); +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDCVal itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " = " << TDCVal << " is out of range"; +#endif + ZDCTDCDataErr::mErrVal[itdc]++; TDCVal = kMinShort; } + if (TDCVal > kMaxShort) { - LOG(error) << __func__ << " TDCVal " << int(ida) << " " << ChannelNames[ida] << " = " << TDCVal << " is out of range"; + int itdc = int(ida); +#ifdef O2_ZDC_DEBUG + LOG(error) << __func__ << "TDCVal itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " = " << TDCVal << " is out of range"; +#endif + ZDCTDCDataErr::mErrVal[itdc]++; TDCVal = kMaxShort; } + if (TDCAmp < kMinShort) { - LOG(error) << __func__ << " TDCAmp " << int(ida) << " " << ChannelNames[ida] << " = " << TDCAmp << " is out of range"; + int itdc = int(ida); +#ifdef O2_ZDC_DEBUG + LOG(warning) << __func__ << "TDCAmp itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " = " << TDCAmp << " is out of range"; +#endif + ZDCTDCDataErr::mErrAmp[itdc]++; TDCAmp = kMinShort; } + if (TDCAmp > kMaxShort) { - LOG(error) << __func__ << " TDCAmp " << int(ida) << " " << ChannelNames[ida] << " = " << TDCAmp << " is out of range"; + int itdc = int(ida); +#ifdef O2_ZDC_DEBUG + LOG(warning) << __func__ << "TDCAmp itdc=" << itdc << " " << ChannelNames[TDCSignal[itdc]] << " = " << TDCAmp << " is out of range"; +#endif + ZDCTDCDataErr::mErrAmp[itdc]++; TDCAmp = kMaxShort; } val = TDCVal; - amp = ampa; + amp = TDCAmp; } inline float amplitude() const diff --git a/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx b/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx index f78db83a1c983..e86ddddbf2ce7 100644 --- a/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx +++ b/DataFormats/Detectors/ZDC/src/ZDCTDCData.cxx @@ -13,6 +13,10 @@ using namespace o2::zdc; +uint32_t ZDCTDCDataErr::mErrVal[NTDCChannels] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +uint32_t ZDCTDCDataErr::mErrAmp[NTDCChannels] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +uint32_t ZDCTDCDataErr::mErrId = 0; + void o2::zdc::ZDCTDCData::print() const { int itdc = id & 0x0f; diff --git a/Detectors/ZDC/macro/CreateModuleConfig.C b/Detectors/ZDC/macro/CreateModuleConfig.C index f1ff6acaeb8e1..b0a2636180468 100644 --- a/Detectors/ZDC/macro/CreateModuleConfig.C +++ b/Detectors/ZDC/macro/CreateModuleConfig.C @@ -122,8 +122,8 @@ void CreateModuleConfig(long tmin = 0, long tmax = -1, std::string ccdbHost = "" module.id = modID; module.setChannel(0, IdZPCC, 2 * modID, true, true, -5, 6, 4, 12); module.setChannel(1, IdZEM2, 2 * modID, true, true, -5, 6, 4, 12); - module.setChannel(2, IdZPC1, 2 * modID + 1, true, false, -5, 6, 4, 12); - module.setChannel(3, IdZPC2, 2 * modID + 1, true, false, -5, 6, 4, 12); + module.setChannel(2, IdZPC3, 2 * modID + 1, true, false, -5, 6, 4, 12); + module.setChannel(3, IdZPC4, 2 * modID + 1, true, false, -5, 6, 4, 12); // } //------------------------------------------- @@ -133,8 +133,8 @@ void CreateModuleConfig(long tmin = 0, long tmax = -1, std::string ccdbHost = "" module.id = modID; module.setChannel(0, IdZPCC, 2 * modID, false, true, -5, 6, 4, 12); module.setChannel(1, IdZPCSum, 2 * modID, true, false, -5, 6, 4, 12); - module.setChannel(2, IdZPC3, 2 * modID + 1, true, false, -5, 6, 4, 12); - module.setChannel(3, IdZPC4, 2 * modID + 1, true, false, -5, 6, 4, 12); + module.setChannel(2, IdZPC1, 2 * modID + 1, true, false, -5, 6, 4, 12); + module.setChannel(3, IdZPC2, 2 * modID + 1, true, false, -5, 6, 4, 12); // } conf.check(); diff --git a/Detectors/ZDC/raw/src/RawReaderZDC.cxx b/Detectors/ZDC/raw/src/RawReaderZDC.cxx index ab68865b1951f..34b18f1af1f7c 100644 --- a/Detectors/ZDC/raw/src/RawReaderZDC.cxx +++ b/Detectors/ZDC/raw/src/RawReaderZDC.cxx @@ -10,6 +10,7 @@ // or submit itself to any jurisdiction. #include "ZDCRaw/RawReaderZDC.h" +#include namespace o2 { @@ -110,6 +111,8 @@ void RawReaderZDC::process(const EventChData& ch) // pop digits int RawReaderZDC::getDigits(std::vector& digitsBC, std::vector& digitsCh, std::vector& orbitData) { + const char* thefcn = "RawReaderZDC::getDigits"; + if (mModuleConfig == nullptr) { LOG(fatal) << "Missing ModuleConfig"; return 0; @@ -242,20 +245,11 @@ int RawReaderZDC::getDigits(std::vector& digitsBC, std::vector DbgMinimal) { - if (alice_0 != ch.f.Alice_0) { - printf("im=%d ic=%d Alice_0 mt=%u ch=%u\n", im, ic, mt.f.Alice_0, ch.f.Alice_0); - } - if (alice_1 != ch.f.Alice_1) { - printf("im=%d ic=%d Alice_1 mt=%u ch=%u\n", im, ic, mt.f.Alice_1, ch.f.Alice_1); - } - if (alice_2 != ch.f.Alice_2) { - printf("im=%d ic=%d Alice_2 mt=%u ch=%u\n", im, ic, mt.f.Alice_2, ch.f.Alice_2); - } - if (alice_3 != ch.f.Alice_3) { - printf("im=%d ic=%d Alice_3 mt=%u ch=%u\n", im, ic, mt.f.Alice_3, ch.f.Alice_3); - } - } + LOGF(warn, "%s (m,c)=(%d,%d) Alice [0123] %u%s%u %u%s%u %u%s%u %u%s%u", thefcn, im, ic, + alice_0, alice_0 == ch.f.Alice_0 ? "==" : "!=", ch.f.Alice_0, + alice_1, alice_1 == ch.f.Alice_1 ? "==" : "!=", ch.f.Alice_1, + alice_2, alice_2 == ch.f.Alice_2 ? "==" : "!=", ch.f.Alice_2, + alice_3, alice_3 == ch.f.Alice_3 ? "==" : "!=", ch.f.Alice_3); } if (filled_module == false) { mt.f.Auto_m = ch.f.Auto_m; @@ -271,12 +265,18 @@ int RawReaderZDC::getDigits(std::vector& digitsBC, std::vector& digitsBC, std::vector DbgMinimal) || (mVerbosity >= DbgFull)) { bcdata.print(mTriggerMask); diff --git a/Detectors/ZDC/reconstruction/src/DigiReco.cxx b/Detectors/ZDC/reconstruction/src/DigiReco.cxx index 7d6e09f6387ef..191b17de7f906 100644 --- a/Detectors/ZDC/reconstruction/src/DigiReco.cxx +++ b/Detectors/ZDC/reconstruction/src/DigiReco.cxx @@ -345,11 +345,14 @@ void DigiReco::eor() mDbg->Close(); mDbg.reset(); } + + ZDCTDCDataErr::print(); + if (mNLonely > 0) { LOG(warn) << "Detected " << mNLonely << " lonely bunches"; for (int ib = 0; ib < o2::constants::lhc::LHCMaxBunches; ib++) { if (mLonely[ib]) { - LOG(warn) << "lonely " << ib << " " << mLonely[ib] << " T " << mLonelyTrig[ib]; + LOGF(warn, "lonely bunch %4d #times=%u #trig=%u", ib, mLonely[ib], mLonelyTrig[ib]); } } } From 95ef47f7d7469872875dfa159c6c050c7abc937f Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 14 Nov 2022 10:15:12 +0100 Subject: [PATCH 007/289] Adjust Fairlogger-->InfoLogger Severity-->Level mapping --- Framework/DataTakingSupport/src/Plugin.cxx | 24 +++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Framework/DataTakingSupport/src/Plugin.cxx b/Framework/DataTakingSupport/src/Plugin.cxx index 3ab37fd456ff9..58ac9b6f5a9ff 100644 --- a/Framework/DataTakingSupport/src/Plugin.cxx +++ b/Framework/DataTakingSupport/src/Plugin.cxx @@ -74,40 +74,40 @@ auto createInfoLoggerSinkHelper(InfoLogger* logger, InfoLoggerContext* ctx) level = 1; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::error)) { severity = InfoLogger::Severity::Error; - level = 3; + level = 2; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::alarm)) { severity = InfoLogger::Severity::Warning; - level = 4; + level = 6; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::important)) { severity = InfoLogger::Severity::Info; - level = 5; + level = 7; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::warn)) { severity = InfoLogger::Severity::Warning; - level = 6; + level = 11; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::state)) { severity = InfoLogger::Severity::Info; - level = 8; + level = 12; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::info)) { severity = InfoLogger::Severity::Info; - level = 10; + level = 13; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug)) { severity = InfoLogger::Severity::Debug; - level = 11; + level = 14; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug1)) { severity = InfoLogger::Severity::Debug; - level = 12; + level = 15; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug2)) { severity = InfoLogger::Severity::Debug; - level = 13; + level = 16; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug3)) { severity = InfoLogger::Severity::Debug; - level = 14; + level = 17; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::debug4)) { severity = InfoLogger::Severity::Debug; - level = 15; + level = 18; } else if (metadata.severity_name == fair::Logger::SeverityName(fair::Severity::trace)) { severity = InfoLogger::Severity::Debug; - level = 50; + level = 21; } InfoLogger::InfoLoggerMessageOption opt = { From 52a2412c3fbe961f0990259d04a3d0df486f4382 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 15 Nov 2022 14:39:41 +0100 Subject: [PATCH 008/289] Calib workflow: Remove deprecated mode where SACs were router via EPNs --- prodtests/full-system-test/aggregator-workflow.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/prodtests/full-system-test/aggregator-workflow.sh b/prodtests/full-system-test/aggregator-workflow.sh index 7556b0f81867a..95a52594ec1c7 100755 --- a/prodtests/full-system-test/aggregator-workflow.sh +++ b/prodtests/full-system-test/aggregator-workflow.sh @@ -133,15 +133,11 @@ if workflow_has_parameter CALIB_PROXIES; then add_W o2-dpl-raw-proxy "--proxy-name tpcidc --io-threads 2 --dataspec \"$CALIBDATASPEC_TPCIDC_A;$CALIBDATASPEC_TPCIDC_C\" --channel-config \"$CHANNELS_LIST\" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT" "" 0 fi if [[ ! -z $CALIBDATASPEC_TPCSAC ]]; then - if [[ $FLP_TPC_SAC == 1 ]]; then # SAC are coming from FLP 145 - # define port for FLP; should be in 47900 - 47999; if nobody defined it, we use 47901 - [[ -z $TPC_SAC_FLP_PORT ]] && TPC_SAC_FLP_PORT=47901 - FLP_ADDRESS_SAC="tcp://alio2-cr1-flp145-ib:${TPC_SAC_FLP_PORT}" - CHANNEL_SAC="type=pull,name=tpcsac,transport=zeromq,address=$FLP_ADDRESS_SAC,method=connect,rateLogging=10;" - add_W o2-dpl-raw-proxy "--proxy-name tpcsac --dataspec \"$CALIBDATASPEC_TPCSAC\" --channel-config \"$CHANNEL_SAC\" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT" "" 0 - else - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_TPCSAC\" $(get_proxy_connection tpcsac input)" "" 0 - fi + # define port for FLP; should be in 47900 - 47999; if nobody defined it, we use 47901 + [[ -z $TPC_SAC_FLP_PORT ]] && TPC_SAC_FLP_PORT=47901 + FLP_ADDRESS_SAC="tcp://alio2-cr1-flp145-ib:${TPC_SAC_FLP_PORT}" + CHANNEL_SAC="type=pull,name=tpcsac,transport=zeromq,address=$FLP_ADDRESS_SAC,method=connect,rateLogging=10;" + add_W o2-dpl-raw-proxy "--proxy-name tpcsac --dataspec \"$CALIBDATASPEC_TPCSAC\" --channel-config \"$CHANNEL_SAC\" --timeframes-shm-limit $TIMEFRAME_SHM_LIMIT" "" 0 fi elif [[ $AGGREGATOR_TASKS == CALO_TF ]]; then if [[ ! -z $CALIBDATASPEC_CALO_TF ]]; then From 8e24e56f7fd7bc8457c9e89c5426a5db524ccb2a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 15 Nov 2022 14:46:19 +0100 Subject: [PATCH 009/289] Calib workflow: indicate whether output proxies are for sporadic or for timeframe data in get_proxy_connection --- prodtests/full-system-test/aggregator-workflow.sh | 14 +++++++------- prodtests/full-system-test/calib-workflow.sh | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/prodtests/full-system-test/aggregator-workflow.sh b/prodtests/full-system-test/aggregator-workflow.sh index 95a52594ec1c7..2f396705ea370 100755 --- a/prodtests/full-system-test/aggregator-workflow.sh +++ b/prodtests/full-system-test/aggregator-workflow.sh @@ -111,11 +111,11 @@ fi if workflow_has_parameter CALIB_PROXIES; then if [[ $AGGREGATOR_TASKS == BARREL_TF ]]; then if [[ ! -z $CALIBDATASPEC_BARREL_TF ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_BARREL_TF\" $(get_proxy_connection barrel_tf input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_BARREL_TF\" $(get_proxy_connection barrel_tf input timeframe)" "" 0 fi elif [[ $AGGREGATOR_TASKS == BARREL_SPORADIC ]]; then if [[ ! -z $CALIBDATASPEC_BARREL_SPORADIC ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_BARREL_SPORADIC\" $(get_proxy_connection barrel_sp input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_BARREL_SPORADIC\" $(get_proxy_connection barrel_sp input sporadic)" "" 0 fi elif [[ $AGGREGATOR_TASKS == TPC_IDCBOTH_SAC ]]; then if [[ ! -z $CALIBDATASPEC_TPCIDC_A ]] || [[ ! -z $CALIBDATASPEC_TPCIDC_C ]]; then @@ -141,23 +141,23 @@ if workflow_has_parameter CALIB_PROXIES; then fi elif [[ $AGGREGATOR_TASKS == CALO_TF ]]; then if [[ ! -z $CALIBDATASPEC_CALO_TF ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_CALO_TF\" $(get_proxy_connection calo_tf input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_CALO_TF\" $(get_proxy_connection calo_tf input timeframe)" "" 0 fi elif [[ $AGGREGATOR_TASKS == CALO_SPORADIC ]]; then if [[ ! -z $CALIBDATASPEC_CALO_SPORADIC ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_CALO_SPORADIC\" $(get_proxy_connection calo_sp input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_CALO_SPORADIC\" $(get_proxy_connection calo_sp input sporadic)" "" 0 fi elif [[ $AGGREGATOR_TASKS == MUON_TF ]]; then if [[ ! -z $CALIBDATASPEC_MUON_TF ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_MUON_TF\" $(get_proxy_connection muon_tf input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_MUON_TF\" $(get_proxy_connection muon_tf input timeframe)" "" 0 fi elif [[ $AGGREGATOR_TASKS == MUON_SPORADIC ]]; then if [[ ! -z $CALIBDATASPEC_MUON_SPORADIC ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_MUON_SPORADIC\" $(get_proxy_connection muon_sp input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_MUON_SPORADIC\" $(get_proxy_connection muon_sp input sporadic)" "" 0 fi elif [[ $AGGREGATOR_TASKS == FORWARD_TF ]]; then if [[ ! -z $CALIBDATASPEC_FORWARD_TF ]]; then - add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_FORWARD_TF\" $(get_proxy_connection fwd_tf input)" "" 0 + add_W o2-dpl-raw-proxy "--dataspec \"$CALIBDATASPEC_FORWARD_TF\" $(get_proxy_connection fwd_tf input timeframe)" "" 0 fi fi fi diff --git a/prodtests/full-system-test/calib-workflow.sh b/prodtests/full-system-test/calib-workflow.sh index 9cf7c588e6e6b..ed53371bbeb75 100644 --- a/prodtests/full-system-test/calib-workflow.sh +++ b/prodtests/full-system-test/calib-workflow.sh @@ -30,25 +30,25 @@ if [[ $CALIB_FT0_TIMEOFFSET == 1 ]]; then add_W o2-calibration-ft0-time-spectra- # output-proxy for aggregator if workflow_has_parameter CALIB_PROXIES; then if [[ ! -z $CALIBDATASPEC_BARREL_TF ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_BARREL_TF\" $(get_proxy_connection barrel_tf output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_BARREL_TF\" $(get_proxy_connection barrel_tf output timeframe)" "" 0 fi if [[ ! -z $CALIBDATASPEC_BARREL_SPORADIC ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_BARREL_SPORADIC\" $(get_proxy_connection barrel_sp output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_BARREL_SPORADIC\" $(get_proxy_connection barrel_sp output sporadic)" "" 0 fi if [[ ! -z $CALIBDATASPEC_CALO_TF ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_CALO_TF\" $(get_proxy_connection calo_tf output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_CALO_TF\" $(get_proxy_connection calo_tf output timeframe)" "" 0 fi if [[ ! -z $CALIBDATASPEC_CALO_SPORADIC ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_CALO_SPORADIC\" $(get_proxy_connection calo_sp output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_CALO_SPORADIC\" $(get_proxy_connection calo_sp output sporadic)" "" 0 fi if [[ ! -z $CALIBDATASPEC_MUON_TF ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_MUON_TF\" $(get_proxy_connection muon_tf output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_MUON_TF\" $(get_proxy_connection muon_tf output timeframe)" "" 0 fi if [[ ! -z $CALIBDATASPEC_MUON_SPORADIC ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_MUON_SPORADIC\" $(get_proxy_connection muon_sp output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_MUON_SPORADIC\" $(get_proxy_connection muon_sp output sporadic)" "" 0 fi if [[ ! -z $CALIBDATASPEC_FORWARD_TF ]]; then - add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_FORWARD_TF\" $(get_proxy_connection fwd_tf output)" "" 0 + add_W o2-dpl-output-proxy "--dataspec \"$CALIBDATASPEC_FORWARD_TF\" $(get_proxy_connection fwd_tf output timeframe)" "" 0 fi fi From 06a0c4cb5b68dee4c4f6001f880c48182729136c Mon Sep 17 00:00:00 2001 From: David Rohr Date: Tue, 15 Nov 2022 15:13:47 +0100 Subject: [PATCH 010/289] DPL: Add option to not drop old timeSlices based on oldestPossible via env variable --- Framework/Core/src/DataRelayer.cxx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Framework/Core/src/DataRelayer.cxx b/Framework/Core/src/DataRelayer.cxx index ce220719376e1..f88a86b0207c8 100644 --- a/Framework/Core/src/DataRelayer.cxx +++ b/Framework/Core/src/DataRelayer.cxx @@ -272,6 +272,10 @@ void DataRelayer::setOldestPossibleInput(TimesliceId proposed, ChannelIndex chan { auto newOldest = mTimesliceIndex.setOldestPossibleInput(proposed, channel); LOGP(debug, "DataRelayer::setOldestPossibleInput {} from channel {}", newOldest.timeslice.value, newOldest.channel.value); + static bool dontDrop = getenv("DPL_DONT_DROP_OLD_TIMESLICE") && atoi(getenv("DPL_DONT_DROP_OLD_TIMESLICE")); + if (dontDrop) { + return; + } for (size_t si = 0; si < mCache.size() / mInputs.size(); ++si) { auto& variables = mTimesliceIndex.getVariablesForSlot({si}); auto timestamp = VariableContextHelpers::getTimeslice(variables); From a09dcf5c37e181ec189758cc316a4b0f1fed8ea8 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Wed, 23 Nov 2022 10:08:36 +0100 Subject: [PATCH 011/289] Revert "DPL: avoid race condition when resetting output proxy" This reverts commit a292ad8315968b30d19fe88a637c9e3aa663668c. --- Framework/Core/src/CompletionPolicyHelpers.cxx | 7 +------ Framework/Core/src/ExternalFairMQDeviceProxy.cxx | 6 ------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx index 9dd9507179bae..d6d85edd91c96 100644 --- a/Framework/Core/src/CompletionPolicyHelpers.cxx +++ b/Framework/Core/src/CompletionPolicyHelpers.cxx @@ -125,12 +125,7 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAllOrdered(const char* name if (input.header == nullptr) { return CompletionPolicy::CompletionOp::Wait; } - long int startTime = framework::DataRefUtils::getHeader(input)->startTime; - if (startTime == 0) { - LOGP(info, "startTime is 0, which means we have the first message, so we can process it."); - *nextTimeSlice = 0; - } - if (framework::DataRefUtils::isValid(input) && startTime != *nextTimeSlice) { + if (framework::DataRefUtils::isValid(input) && framework::DataRefUtils::getHeader(input)->startTime != *nextTimeSlice) { return CompletionPolicy::CompletionOp::Retry; } } diff --git a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx index 0def2c9f0536f..b1e7e0d498ca8 100644 --- a/Framework/Core/src/ExternalFairMQDeviceProxy.cxx +++ b/Framework/Core/src/ExternalFairMQDeviceProxy.cxx @@ -264,12 +264,6 @@ InjectorFunction dplModelAdaptor(std::vector const& filterSpecs, DPL LOG(error) << "data on input " << msgidx << " does not follow the O2 data model, DataProcessingHeader missing"; continue; } - static size_t currentRunNumber = -1; - if (dh->runNumber != currentRunNumber) { - LOGP(detail, "Run number changed from {} to {}. Resetting DPL timeslice counter", currentRunNumber, dh->runNumber); - currentRunNumber = dh->runNumber; - dplCounter = 0; - } const_cast(dph)->startTime = dplCounter; if (override_creation) { const_cast(dph)->creation = creationVal + (dh->firstTForbit * o2::constants::lhc::LHCOrbitNS * 0.000001f); From 33cb996d03041094fcb3da9ecad21a0c985ba412 Mon Sep 17 00:00:00 2001 From: cortesep <57937610+cortesep@users.noreply.github.com> Date: Sun, 16 Apr 2023 01:53:59 +0200 Subject: [PATCH 012/289] ZDC - Introduction of RDH V7 (#11109) * Propagate flag to select RDH version (#65) * Introduce dataFormat = 2 * Fixes * Fix in binning * Please consider the following formatting changes (#66) * Better error treatment * Please consider the following formatting changes (#67) * Avoid GBT word copy, use pointers * Please consider the following formatting changes (#68) --------- Co-authored-by: ALICE Builder --- .../ZDC/include/DataFormatsZDC/RawEventData.h | 4 +- .../ZDC/raw/include/ZDCRaw/RawReaderZDC.h | 2 +- Detectors/ZDC/raw/src/DumpRaw.cxx | 29 ++++----- Detectors/ZDC/raw/src/RawReaderZDC.cxx | 62 +++++++++++++------ Detectors/ZDC/raw/src/ZDCRawParserDPLSpec.cxx | 28 +++++++-- .../include/ZDCSimulation/Digits2Raw.h | 9 +++ Detectors/ZDC/simulation/src/Digits2Raw.cxx | 27 ++++---- Detectors/ZDC/simulation/src/digi2raw.cxx | 19 +++++- .../ZDC/workflow/src/ZDCDataReaderDPLSpec.cxx | 12 ++-- 9 files changed, 132 insertions(+), 60 deletions(-) diff --git a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h index 5d856f55c1d65..622f3116b7868 100644 --- a/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h +++ b/DataFormats/Detectors/ZDC/include/DataFormatsZDC/RawEventData.h @@ -33,7 +33,9 @@ constexpr unsigned short Id_w0 = 0x0; constexpr unsigned short Id_w1 = 0x1; constexpr unsigned short Id_w2 = 0x2; constexpr unsigned short Id_wn = 0x3; -constexpr int NWPerGBTW = 4; +constexpr int NWPerGBTW = 4; // 4*32bit=128 bit per GBTW +constexpr int NBPerGBTW = 4 * NWPerGBTW; // 16B=128 bit per GBTW +constexpr int PayloadPerGBTW = 10; // 80 bit per GBTW struct __attribute__((__packed__)) ChannelDataV0 { // First GBT word diff --git a/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h b/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h index 482c4c4c38b6d..b27f7e34353c0 100644 --- a/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h +++ b/Detectors/ZDC/raw/include/ZDCRaw/RawReaderZDC.h @@ -70,7 +70,7 @@ class RawReaderZDC void clear(); // decoding binary data into data blocks - void processBinaryData(gsl::span payload, int linkID); // processing data blocks into digits + int processBinaryData(gsl::span payload, int linkID, uint8_t dataFormat); // processing data blocks into digits int processWord(const uint32_t* word); void process(const EventChData& ch); diff --git a/Detectors/ZDC/raw/src/DumpRaw.cxx b/Detectors/ZDC/raw/src/DumpRaw.cxx index c1dc48169cf40..3a6cec3cba6f8 100644 --- a/Detectors/ZDC/raw/src/DumpRaw.cxx +++ b/Detectors/ZDC/raw/src/DumpRaw.cxx @@ -78,9 +78,9 @@ void DumpRaw::init() { gROOT->SetBatch(); auto& sopt = ZDCSimParam::Instance(); - int nbx = (sopt.nBCAheadTrig + 1) * NTimeBinsPerBC; double xmin = -sopt.nBCAheadTrig * NTimeBinsPerBC - 0.5; double xmax = 2 * NTimeBinsPerBC - 0.5; + int nbx = std::round(xmax - xmin); if (mTransmitted == nullptr) { mTransmitted = std::make_unique("ht", "Transmitted channels", NModules, -0.5, NModules - 0.5, NChPerModule, -0.5, NChPerModule - 0.5); } @@ -226,36 +226,37 @@ int DumpRaw::processWord(const uint32_t* word) printf("NULL\n"); return 1; } + // LOGF(info, "GBT word %04x %08x %08x id=%u", *((uint16_t*)&word[2]), word[1], word[0], word[0] & 0x3); if ((word[0] & 0x3) == Id_w0) { - for (int32_t iw = 0; iw < NWPerGBTW; iw++) { - mCh.w[0][iw] = word[iw]; - } + mCh.w[0][NWPerGBTW - 1] = 0; + mCh.w[0][NWPerGBTW - 2] = 0; + memcpy((void*)&mCh.w[0][0], (const void*)word, PayloadPerGBTW); } else if ((word[0] & 0x3) == Id_w1) { if (mCh.f.fixed_0 == Id_w0) { - for (int32_t iw = 0; iw < NWPerGBTW; iw++) { - mCh.w[1][iw] = word[iw]; - } + mCh.w[1][NWPerGBTW - 1] = 0; + mCh.w[1][NWPerGBTW - 2] = 0; + memcpy((void*)&mCh.w[1][0], (const void*)word, PayloadPerGBTW); } else { - LOG(error) << "Wrong word sequence"; + LOGF(error, "Wrong word sequence: %04x %08x %08x id=%u *%u*", *((uint16_t*)&word[2]), word[1], word[0], mCh.f.fixed_0, word[0] & 0x3); mCh.f.fixed_0 = Id_wn; mCh.f.fixed_1 = Id_wn; mCh.f.fixed_2 = Id_wn; } } else if ((word[0] & 0x3) == Id_w2) { if (mCh.f.fixed_0 == Id_w0 && mCh.f.fixed_1 == Id_w1) { - for (int32_t iw = 0; iw < NWPerGBTW; iw++) { - mCh.w[2][iw] = word[iw]; - } + mCh.w[2][NWPerGBTW - 1] = 0; + mCh.w[2][NWPerGBTW - 2] = 0; + memcpy((void*)&mCh.w[2][0], (const void*)word, PayloadPerGBTW); process(mCh); } else { - LOG(error) << "Wrong word sequence"; + LOGF(error, "Wrong word sequence: %04x %08x %08x id=%u %u *%u*", *((uint16_t*)&word[2]), word[1], word[0], mCh.f.fixed_0, mCh.f.fixed_1, word[0] & 0x3); } mCh.f.fixed_0 = Id_wn; mCh.f.fixed_1 = Id_wn; mCh.f.fixed_2 = Id_wn; } else { - // Word not present in payload - LOG(fatal) << "Event format error"; + // Word id not foreseen in payload + LOGF(error, "Event format error on word %04x %08x %08x id=%u", *((uint16_t*)&word[2]), word[1], word[0], word[0] & 0x3); return 1; } return 0; diff --git a/Detectors/ZDC/raw/src/RawReaderZDC.cxx b/Detectors/ZDC/raw/src/RawReaderZDC.cxx index 34b18f1af1f7c..f9e40b28abc24 100644 --- a/Detectors/ZDC/raw/src/RawReaderZDC.cxx +++ b/Detectors/ZDC/raw/src/RawReaderZDC.cxx @@ -32,25 +32,48 @@ void RawReaderZDC::clear() mOrbitData.clear(); } -void RawReaderZDC::processBinaryData(gsl::span payload, int linkID) +int RawReaderZDC::processBinaryData(gsl::span payload, int linkID, uint8_t dataFormat) { if (0 <= linkID && linkID < 16) { size_t payloadSize = payload.size(); - for (int32_t ip = 0; ip < payloadSize; ip += 16) { + if (dataFormat == 2) { + for (int32_t ip = 0; (ip + PayloadPerGBTW) <= payloadSize; ip += PayloadPerGBTW) { #ifndef O2_ZDC_DEBUG - if (mVerbosity >= DbgExtra) { + if (mVerbosity >= DbgExtra) { + o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); + } +#else o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); +#endif + const uint32_t* gbtw = (const uint32_t*)&payload[ip]; + if (gbtw[0] != 0xffffffff && gbtw[1] != 0xffffffff && (*((const uint16_t*)&gbtw[2])) != 0xffff) { + if (processWord(gbtw)) { + return 1; + } + } } + } else if (dataFormat == 0) { + for (int32_t ip = 0; ip < payloadSize; ip += NBPerGBTW) { +#ifndef O2_ZDC_DEBUG + if (mVerbosity >= DbgExtra) { + o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); + } #else - o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); + o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); #endif - processWord((const uint32_t*)&payload[ip]); + if (processWord((const uint32_t*)&payload[ip])) { + return 1; + } + } + } else { + LOG(fatal) << "RawReaderZDC::processBinaryData - Unsupported DataFormat " << dataFormat; } } else { // put here code in case of bad rdh.linkID value LOG(info) << "WARNING! WRONG LINK ID! " << linkID; - return; + return 1; } + return 0; } int RawReaderZDC::processWord(const uint32_t* word) @@ -59,36 +82,37 @@ int RawReaderZDC::processWord(const uint32_t* word) LOG(error) << "NULL pointer"; return 1; } + // LOGF(info, "GBT word %04x %08x %08x id=%u", *((uint16_t*)&word[2]), word[1], word[0], word[0] & 0x3); if ((word[0] & 0x3) == Id_w0) { - for (int32_t iw = 0; iw < NWPerGBTW; iw++) { - mCh.w[0][iw] = word[iw]; - } + mCh.w[0][NWPerGBTW - 1] = 0; + mCh.w[0][NWPerGBTW - 2] = 0; + memcpy((void*)&mCh.w[0][0], (const void*)word, PayloadPerGBTW); } else if ((word[0] & 0x3) == Id_w1) { if (mCh.f.fixed_0 == Id_w0) { - for (int32_t iw = 0; iw < NWPerGBTW; iw++) { - mCh.w[1][iw] = word[iw]; - } + mCh.w[1][NWPerGBTW - 1] = 0; + mCh.w[1][NWPerGBTW - 2] = 0; + memcpy((void*)&mCh.w[1][0], (const void*)word, PayloadPerGBTW); } else { - LOG(error) << "Wrong word sequence"; + LOGF(error, "Wrong word sequence: %04x %08x %08x id=%u *%u*", *((uint16_t*)&word[2]), word[1], word[0], mCh.f.fixed_0, word[0] & 0x3); mCh.f.fixed_0 = Id_wn; mCh.f.fixed_1 = Id_wn; mCh.f.fixed_2 = Id_wn; } } else if ((word[0] & 0x3) == Id_w2) { if (mCh.f.fixed_0 == Id_w0 && mCh.f.fixed_1 == Id_w1) { - for (int32_t iw = 0; iw < NWPerGBTW; iw++) { - mCh.w[2][iw] = word[iw]; - } + mCh.w[2][NWPerGBTW - 1] = 0; + mCh.w[2][NWPerGBTW - 2] = 0; + memcpy((void*)&mCh.w[2][0], (const void*)word, PayloadPerGBTW); process(mCh); } else { - LOG(error) << "Wrong word sequence"; + LOGF(error, "Wrong word sequence: %04x %08x %08x id=%u %u *%u*", *((uint16_t*)&word[2]), word[1], word[0], mCh.f.fixed_0, mCh.f.fixed_1, word[0] & 0x3); } mCh.f.fixed_0 = Id_wn; mCh.f.fixed_1 = Id_wn; mCh.f.fixed_2 = Id_wn; } else { - // Word not present in payload - LOG(fatal) << "Event format error"; + // Word id not foreseen in payload + LOGF(error, "Event format error on word %04x %08x %08x id=%u", *((uint16_t*)&word[2]), word[1], word[0], word[0] & 0x3); return 1; } return 0; diff --git a/Detectors/ZDC/raw/src/ZDCRawParserDPLSpec.cxx b/Detectors/ZDC/raw/src/ZDCRawParserDPLSpec.cxx index d9d5aa8a6aec9..857b1b5400e4c 100644 --- a/Detectors/ZDC/raw/src/ZDCRawParserDPLSpec.cxx +++ b/Detectors/ZDC/raw/src/ZDCRawParserDPLSpec.cxx @@ -33,12 +33,14 @@ #include "DataFormatsZDC/ChannelData.h" #include "DataFormatsZDC/OrbitData.h" #include "DataFormatsZDC/RecEvent.h" +#include "DataFormatsZDC/RawEventData.h" #include "CommonUtils/NameConf.h" #include "CommonUtils/MemFileHelper.h" #include "CCDB/BasicCCDBManager.h" #include "CCDB/CCDBTimeStampUtils.h" #include "ZDCBase/ModuleConfig.h" #include "ZDCRaw/ZDCRawParserDPLSpec.h" +#include "ZDCSimulation/Digits2Raw.h" using namespace o2::framework; @@ -104,12 +106,30 @@ void ZDCRawParserDPLSpec::run(ProcessingContext& pc) size_t payloadSize = it.size(); // offset of payload in the raw page size_t offset = it.offset(); + int dataFormat = o2::raw::RDHUtils::getDataFormat(rdhPtr); #ifdef O2_ZDC_DEBUG - LOG(info) << count << " processBinaryData: size=" << it.size() << " link=" << o2::raw::RDHUtils::getLinkID(rdhPtr); + int linkID = o2::raw::RDHUtils::getLinkID(rdhPtr); + LOG(info) << count << " ZDCRawParserDPLSpec::run: fmt=" << dataFormat << " size=" << it.size() << " link=" << linkID; #endif - for (int32_t ip = 0; ip < payloadSize; ip += 16) { - // o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); - mWorker.processWord((const uint32_t*)&payload[ip]); + if (dataFormat == 2) { + for (int32_t ip = 0; (ip + PayloadPerGBTW) <= payloadSize; ip += PayloadPerGBTW) { + // Assign only the actual payload + uint32_t gbtw[4] = {0x0, 0x0, 0x0, 0x0}; + memcpy((void*)gbtw, (const void*)&payload[ip], PayloadPerGBTW); +#ifdef O2_ZDC_DEBUG + o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)gbtw); +#endif + if (gbtw[0] != 0xffffffff && gbtw[1] != 0xffffffff && (gbtw[2] & 0xffff) != 0xffff) { + mWorker.processWord(gbtw); + } + } + } else if (dataFormat == 0) { + for (int32_t ip = 0; ip < payloadSize; ip += NBPerGBTW) { + // o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&payload[ip]); + mWorker.processWord((const uint32_t*)&payload[ip]); + } + } else { + LOG(error) << "ZDCDataReaderDPLSpec::run - Unsupported DataFormat " << dataFormat; } } } diff --git a/Detectors/ZDC/simulation/include/ZDCSimulation/Digits2Raw.h b/Detectors/ZDC/simulation/include/ZDCSimulation/Digits2Raw.h index a7e8d5f13c3b9..869b00911a1e5 100644 --- a/Detectors/ZDC/simulation/include/ZDCSimulation/Digits2Raw.h +++ b/Detectors/ZDC/simulation/include/ZDCSimulation/Digits2Raw.h @@ -19,6 +19,7 @@ #include #include #include "Headers/RAWDataHeader.h" +#include "Framework/Logger.h" #include "CommonDataFormat/InteractionRecord.h" #include "DetectorsRaw/RawFileWriter.h" #include "DetectorsRaw/HBFUtils.h" @@ -58,6 +59,13 @@ class Digits2Raw // void setContinuous(bool v = true) { mIsContinuous = v; } bool isContinuous() const { return mIsContinuous; } + + void setEnablePadding(bool v) + { + mEnablePadding = v; + } + bool getEnablePadding() const { return mEnablePadding; } + static void print_gbt_word(const uint32_t* word, const ModuleConfig* moduleConfig = nullptr); private: @@ -77,6 +85,7 @@ class Digits2Raw EventData mZDC; /// Output structure bool mIsContinuous = true; /// Continuous (self-triggered) or externally-triggered readout bool mOutputPerLink = false; /// Split output + int mEnablePadding = 0; /// Enable padding to 128 bit const ModuleConfig* mModuleConfig = nullptr; /// Trigger/readout configuration object const SimCondition* mSimCondition = nullptr; /// Pedestal/noise configuration object uint16_t mScalers[NModules][NChPerModule] = {0}; /// ZDC orbit scalers diff --git a/Detectors/ZDC/simulation/src/Digits2Raw.cxx b/Detectors/ZDC/simulation/src/Digits2Raw.cxx index ea083f4943760..bd865fc63c1d4 100644 --- a/Detectors/ZDC/simulation/src/Digits2Raw.cxx +++ b/Detectors/ZDC/simulation/src/Digits2Raw.cxx @@ -61,6 +61,8 @@ void Digits2Raw::processDigits(const std::string& outDir, const std::string& fil mFLPID = uint16_t(0); mEndPointID = uint32_t(0); // TODO: assign FeeID from configuration object + // N.B. Now the electronics has the possibility to reconfigure FEE ID in order to match + // what is expected from simulation. The FEE ID should never change in the future for (int ilink = 0; ilink < NLinks; ilink++) { uint64_t FeeID = uint64_t(ilink); std::string outFileLink = o2::utils::Str::concat_string(outDir, "/", "ZDC"); @@ -237,14 +239,7 @@ inline void Digits2Raw::updatePedestalReference(int bc) for (int32_t ic = 0; ic < NChPerModule; ic++) { // Identify connected channel auto id = mModuleConfig->modules[im].channelID[ic]; - double myped = mzdcPedData[io].data[id] + 32768.; - if (myped < 0) { - myped = 0; - } - if (myped > 65535) { - myped = 65535; - } - mPed[im][ic] = myped; + mPed[im][ic] = *((uint16_t*)&mzdcPedData[io].data[id]); } } } else if (mEmpty[bc] > 0 && mEmpty[bc] != mLastNEmpty) { @@ -495,8 +490,14 @@ void Digits2Raw::writeDigits() uint64_t FeeID = 2 * im + ic / 2; if (mModuleConfig->modules[im].readChannel[ic]) { for (int32_t iw = 0; iw < o2::zdc::NWPerBc; iw++) { - gsl::span payload{reinterpret_cast(&mZDC.data[im][ic].w[iw][0]), data_size}; - mWriter.addData(FeeID, mCruID, mLinkID, mEndPointID, ir, payload); + if (mEnablePadding) { + gsl::span payload{reinterpret_cast(&mZDC.data[im][ic].w[iw][0]), data_size}; + mWriter.addData(FeeID, mCruID, mLinkID, mEndPointID, ir, payload); + } else { + gsl::span payload{reinterpret_cast(&mZDC.data[im][ic].w[iw][0]), PayloadPerGBTW}; + o2::zdc::Digits2Raw::print_gbt_word((const uint32_t*)&mZDC.data[im][ic].w[iw][0]); + mWriter.addData(FeeID, mCruID, mLinkID, mEndPointID, ir, payload); + } } addedChData[ic] = true; } @@ -560,10 +561,8 @@ void Digits2Raw::print_gbt_word(const uint32_t* word, const ModuleConfig* module ULong64_t msb = val >> 64; uint32_t a = word[0]; uint32_t b = word[1]; - uint32_t c = word[2]; - // uint32_t d=(msb>>32)&0xffffffff; - // printf("\n%llx %llx ",lsb,msb); - // printf("\n%8x %8x %8x %8x ",d,c,b,a); + uint16_t c = *((uint16_t*)&word[2]); + // printf("\nGBTW: %04x %08x %08x\n",c,b,a); if ((a & 0x3) == 0) { uint32_t myorbit = (val >> 48) & 0xffffffff; uint32_t mybc = (val >> 36) & 0xfff; diff --git a/Detectors/ZDC/simulation/src/digi2raw.cxx b/Detectors/ZDC/simulation/src/digi2raw.cxx index b4c6cdc2e254c..61d778092f29c 100644 --- a/Detectors/ZDC/simulation/src/digi2raw.cxx +++ b/Detectors/ZDC/simulation/src/digi2raw.cxx @@ -38,7 +38,7 @@ namespace bpo = boost::program_options; -void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV = 4, +void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV = 7, bool enablePadding = false, const std::string& ccdbHost = "", int superPageSizeInB = 1024 * 1024); int main(int argc, char** argv) @@ -60,6 +60,7 @@ int main(int argc, char** argv) add_option("output-dir,o", bpo::value()->default_value("./"), "output directory for raw data"); uint32_t defRDH = o2::raw::RDHUtils::getVersion(); add_option("rdh-version,r", bpo::value()->default_value(defRDH), "RDH version to use"); + add_option("enable-padding", bpo::value()->default_value(false)->implicit_value(true), "enable GBT word padding to 128 bits even for RDH V7"); add_option("hbfutils-config,u", bpo::value()->default_value(std::string(o2::base::NameConf::DIGITIZATIONCONFIGFILE)), "config file for HBFUtils (or none)"); add_option("configKeyValues", bpo::value()->default_value(""), "comma-separated configKeyValues"); @@ -94,6 +95,7 @@ int main(int argc, char** argv) vm["verbosity"].as(), vm["file-for"].as(), vm["rdh-version"].as(), + vm["enable-padding"].as(), ccdbHost); o2::raw::HBFUtils::Instance().print(); @@ -101,8 +103,12 @@ int main(int argc, char** argv) return 0; } -void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, const std::string& ccdbHost, int superPageSizeInB) +void digi2raw(const std::string& inpName, const std::string& outDir, int verbosity, const std::string& fileFor, uint32_t rdhV, bool enablePadding, const std::string& ccdbHost, int superPageSizeInB) { + if (rdhV < 7 && !enablePadding) { + enablePadding = true; + LOG(info) << "padding is always ON for RDH version " << rdhV; + } //std::string ccdbHost = "http://ccdb-test.cern.ch:8080"; auto& mgr = o2::ccdb::BasicCCDBManager::instance(); mgr.setURL(ccdbHost); @@ -126,6 +132,8 @@ void digi2raw(const std::string& inpName, const std::string& outDir, int verbosi LOG(info) << "Loaded simulation configuration for timestamp " << mgr.getTimestamp(); simCondition->print(); + LOG(info) << "RDHVersion " << rdhV << " padding: " << enablePadding; + const auto* ctx = o2::steer::DigitizationContext::loadFromFile("collisioncontext.root"); const auto& bcfill = ctx->getBunchFilling(); auto bf = ctx->getBunchFilling(); @@ -139,6 +147,7 @@ void digi2raw(const std::string& inpName, const std::string& outDir, int verbosi o2::zdc::Digits2Raw d2r; d2r.setFileFor(fileFor); + d2r.setEnablePadding(enablePadding); d2r.setVerbosity(verbosity); auto& wr = d2r.getWriter(); std::string inputGRP = o2::base::NameConf::getGRPFileName(); @@ -146,7 +155,11 @@ void digi2raw(const std::string& inpName, const std::string& outDir, int verbosi wr.setContinuousReadout(grp->isDetContinuousReadOut(o2::detectors::DetID::ZDC)); // must be set explicitly wr.setSuperPageSize(superPageSizeInB); wr.useRDHVersion(rdhV); - + wr.useRDHDataFormat(enablePadding ? 0 : 2); + if (!enablePadding) { // CRU page alignment padding is used only if no GBT word padding is used + wr.setAlignmentSize(16); + wr.setAlignmentPaddingFiller(0xff); + } o2::raw::assertOutputDirectory(outDir); std::string outDirName(outDir); diff --git a/Detectors/ZDC/workflow/src/ZDCDataReaderDPLSpec.cxx b/Detectors/ZDC/workflow/src/ZDCDataReaderDPLSpec.cxx index cb5e5e4dd0ab0..7caf605d8ef38 100644 --- a/Detectors/ZDC/workflow/src/ZDCDataReaderDPLSpec.cxx +++ b/Detectors/ZDC/workflow/src/ZDCDataReaderDPLSpec.cxx @@ -86,7 +86,7 @@ void ZDCDataReaderDPLSpec::run(ProcessingContext& pc) DPLRawParser parser(pc.inputs(), o2::framework::select("zdc:ZDC/RAWDATA")); uint64_t count = 0; - static uint64_t nErr[3] = {0}; + static uint64_t nErr[4] = {0}; for (auto it = parser.begin(), end = parser.end(); it != end; ++it) { // Processing each page auto rdhPtr = reinterpret_cast(it.raw()); @@ -105,10 +105,14 @@ void ZDCDataReaderDPLSpec::run(ProcessingContext& pc) } else { gsl::span payload(it.data(), it.size()); auto lid = o2::raw::RDHUtils::getLinkID(rdhPtr); + auto dataFormat = o2::raw::RDHUtils::getDataFormat(rdhPtr); #ifdef O2_ZDC_DEBUG LOG(info) << count << " processBinaryData: size=" << it.size() << " link=" << lid; #endif - mRawReader.processBinaryData(payload, lid); + if (mRawReader.processBinaryData(payload, lid, dataFormat)) { + nErr[3]++; + break; + } } } count++; @@ -123,10 +127,10 @@ void ZDCDataReaderDPLSpec::run(ProcessingContext& pc) if (nErr[2] > 0) { LOG(warning) << "ZDCDataReaderDPLSpec::run - No payload occurrences " << nErr[2]; } - if (nErr[0] == 0) { + if (nErr[0] == 0 && nErr[3] == 0) { mRawReader.accumulateDigits(); } else { - LOG(warning) << "Not sending output "; + LOG(warning) << "Not sending output"; } mRawReader.makeSnapshot(pc); } From 0eb56c5e8e854a26d2a471e7f678a4478af67be6 Mon Sep 17 00:00:00 2001 From: Ruben Shahoyan Date: Thu, 20 Apr 2023 11:37:01 +0200 Subject: [PATCH 013/289] Fix processing of nodata=1/internal=1 ITS/MFT triggers (#11194) * Fix processing of nodata=1/internal=1 ITS/MFT triggers * Apply suggestions from code review --------- Co-authored-by: shahoian Co-authored-by: Ole Schmidt --- .../include/ITSMFTReconstruction/GBTLink.h | 4 +++- .../include/ITSMFTReconstruction/RUDecodeData.h | 16 ++++++++-------- .../common/reconstruction/src/RUDecodeData.cxx | 4 +++- .../reconstruction/src/RawPixelDecoder.cxx | 9 +++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/GBTLink.h b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/GBTLink.h index d5c529b8c0824..d6d88a32f8b0c 100644 --- a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/GBTLink.h +++ b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/GBTLink.h @@ -295,7 +295,9 @@ GBTLink::CollectedDataStatus GBTLink::collectROFCableData(const Mapping& chmap) extTrigVec->emplace_back(PhysTrigger{o2::InteractionRecord(uint16_t(gbtTrgTmp->bc), uint32_t(gbtTrgTmp->orbit)), uint64_t(gbtTrgTmp->triggerType)}); } } - continue; + if (gbtTrgTmp->internal == 0) { // external trigger, may have others + continue; + } } auto gbtC = reinterpret_cast(&currRawPiece->data[dataOffset]); if (gbtC->isCalibrationWord()) { diff --git a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/RUDecodeData.h b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/RUDecodeData.h index 5f451c4fd3575..de1d5bb4197f1 100644 --- a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/RUDecodeData.h +++ b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/RUDecodeData.h @@ -34,13 +34,13 @@ struct RUDecodeData { static constexpr int MaxChipsPerRU = 196; // max number of chips the RU can readout static constexpr int MaxLinksPerRU = 3; // max number of GBT links per RU - std::array cableData; // cable data in compressed ALPIDE format - std::vector chipsData; // fully decoded data in 1st nChipsFired chips - std::array links; // link entry RSTODO: consider removing this and using pointer - std::array cableHWID; // HW ID of cable whose data is in the corresponding slot of cableData - std::array cableLinkID; // ID of the GBT link transmitting this cable data - std::array cableLinkPtr; // Ptr of the GBT link transmitting this cable data - std::unordered_map linkHBFToDump; // FEEID<<32+hbfEntry to dump in case of error + std::array cableData{}; // cable data in compressed ALPIDE format + std::vector chipsData{}; // fully decoded data in 1st nChipsFired chips + std::array links{}; // link entry RSTODO: consider removing this and using pointer + std::array cableHWID{}; // HW ID of cable whose data is in the corresponding slot of cableData + std::array cableLinkID{}; // ID of the GBT link transmitting this cable data + std::array cableLinkPtr{}; // Ptr of the GBT link transmitting this cable data + std::unordered_map linkHBFToDump{}; // FEEID<<32+hbfEntry to dump in case of error int ruSWID = -1; // SW (stave) ID int nChipsFired = 0; // number of chips with data or with errors int lastChipChecked = 0; // last chips checked among nChipsFired @@ -49,7 +49,7 @@ struct RUDecodeData { int nLinksDone = 0; // number of links finished for this TF int verbosity = 0; // verbosity level, for -1,0 print only summary data, for 1: print once every error GBTCalibData calibData{}; // calibration info from GBT calibration word - std::unordered_map> chipErrorsTF; // vector of chip decoding errors seen in the given TF + std::unordered_map> chipErrorsTF{}; // vector of chip decoding errors seen in the given TF const RUInfo* ruInfo = nullptr; RUDecodeData() diff --git a/Detectors/ITSMFT/common/reconstruction/src/RUDecodeData.cxx b/Detectors/ITSMFT/common/reconstruction/src/RUDecodeData.cxx index 5330e78f4c79a..2261c1e8c6423 100644 --- a/Detectors/ITSMFT/common/reconstruction/src/RUDecodeData.cxx +++ b/Detectors/ITSMFT/common/reconstruction/src/RUDecodeData.cxx @@ -32,7 +32,9 @@ namespace itsmft void RUDecodeData::clear() { for (int i = ruInfo->nCables; i--;) { - cableData[i].clear(); + if (cableLinkPtr[i] && !cableLinkPtr[i]->rofJumpWasSeen) { + cableData[i].clear(); + } } nChipsFired = 0; nNonEmptyLinks = 0; diff --git a/Detectors/ITSMFT/common/reconstruction/src/RawPixelDecoder.cxx b/Detectors/ITSMFT/common/reconstruction/src/RawPixelDecoder.cxx index 883fbb434bca5..fcc7e42a8f7e4 100644 --- a/Detectors/ITSMFT/common/reconstruction/src/RawPixelDecoder.cxx +++ b/Detectors/ITSMFT/common/reconstruction/src/RawPixelDecoder.cxx @@ -181,6 +181,9 @@ bool RawPixelDecoder::doIRMajorityPoll() mIRPoll[link.ir]++; } else if (link.status == GBTLink::StoppedOnEndOfData || link.status == GBTLink::AbortedOnError) { link.statusInTF = GBTLink::StoppedOnEndOfData; + if (mVerbosity != GBTLink::Verbosity::Silent) { + LOGP(info, "doIRMajorityPoll: {} DONE, status = {}", link.describe(), int(link.status)); + } mNLinksDone++; } } @@ -194,9 +197,15 @@ bool RawPixelDecoder::doIRMajorityPoll() } mInteractionRecordHB = mInteractionRecord; if (mInteractionRecord.isDummy()) { + if (mVerbosity != GBTLink::Verbosity::Silent) { + LOG(info) << "doIRMajorityPoll: did not find any valid IR"; + } return false; } mInteractionRecordHB.bc = 0; + if (mVerbosity != GBTLink::Verbosity::Silent) { + LOG(info) << "doIRMajorityPoll: " << mInteractionRecordHB.asString() << " majority = " << majIR << " for " << mNLinksInTF << " links seen, LinksDone = " << mNLinksDone; + } return true; } From 66f7be3b81c75db3630b814b5988305ba3923ae5 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Mon, 22 May 2023 23:49:49 +0200 Subject: [PATCH 014/289] [EMCAL-551] With default -fstrict-aliasing option that comes with -O2, only void* and char* may alias other data types, so the reinterpret_cast in getDataRepresentation is undefined behavior --- DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Cell.h | 2 +- DataFormats/Detectors/EMCAL/src/Cell.cxx | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Cell.h b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Cell.h index a3a67b88dd389..d441c7a795cf9 100644 --- a/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Cell.h +++ b/DataFormats/Detectors/EMCAL/include/DataFormatsEMCAL/Cell.h @@ -183,7 +183,7 @@ class Cell CellData* getDataRepresentation() { return reinterpret_cast(mCellWords); } const CellData* getDataRepresentation() const { return reinterpret_cast(mCellWords); } - uint16_t mCellWords[3]; ///< data word + char mCellWords[6]; ///< data word ClassDefNV(Cell, 1); }; diff --git a/DataFormats/Detectors/EMCAL/src/Cell.cxx b/DataFormats/Detectors/EMCAL/src/Cell.cxx index 4132d99c2a224..edcb9e294d6e1 100644 --- a/DataFormats/Detectors/EMCAL/src/Cell.cxx +++ b/DataFormats/Detectors/EMCAL/src/Cell.cxx @@ -25,12 +25,12 @@ const float TIME_SHIFT = 600., Cell::Cell() { - memset(mCellWords, 0, sizeof(uint16_t) * 3); + memset(mCellWords, 0, 6); } Cell::Cell(short tower, float energy, float time, ChannelType_t ctype) { - memset(mCellWords, 0, sizeof(uint16_t) * 3); + memset(mCellWords, 0, 6); setTower(tower); setTimeStamp(time); setEnergy(energy); From 9b887b81b70766bb8dd082ba93f391e212695d13 Mon Sep 17 00:00:00 2001 From: shahoian Date: Wed, 31 May 2023 11:36:26 +0200 Subject: [PATCH 015/289] Curl writeCallBack allocates memory once according to Content-Length from header --- CCDB/src/CcdbApi.cxx | 50 +++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/CCDB/src/CcdbApi.cxx b/CCDB/src/CcdbApi.cxx index c9770f61c507c..678633694a61e 100644 --- a/CCDB/src/CcdbApi.cxx +++ b/CCDB/src/CcdbApi.cxx @@ -1555,16 +1555,17 @@ void CcdbApi::loadFileToMemory(o2::pmr::vector& dest, std::string const& p // navigate sequence of URLs until TFile content is found; object is extracted and returned void CcdbApi::navigateURLsAndLoadFileToMemory(o2::pmr::vector& dest, CURL* curl_handle, std::string const& url, std::map* headers) const { - // a global internal data structure that can be filled with HTTP header information - // static --> to avoid frequent alloc/dealloc as optimization - // not sure if thread_local takes away that benefit - static thread_local std::multimap headerData; - // let's see first of all if the url is something specific that curl cannot handle if (url.find("alien:/", 0) != std::string::npos) { return loadFileToMemory(dest, url, nullptr); // headers loaded from the file in case of the snapshot reading only } // otherwise make an HTTP/CURL request + struct HeaderObjectPair_t { + std::map header; + o2::pmr::vector* object = nullptr; + int counter = 0; + } hoPair{{}, &dest, 0}; + bool errorflag = false; auto signalError = [&chunk = dest, &errorflag]() { chunk.clear(); @@ -1572,14 +1573,25 @@ void CcdbApi::navigateURLsAndLoadFileToMemory(o2::pmr::vector& dest, CURL* errorflag = true; }; auto writeCallBack = [](void* contents, size_t size, size_t nmemb, void* chunkptr) { - o2::pmr::vector& chunk = *static_cast*>(chunkptr); - size_t realsize = size * nmemb; + auto& ho = *static_cast(chunkptr); + auto& chunk = *ho.object; + size_t realsize = size * nmemb, sz = 0; + ho.counter++; try { - chunk.reserve(chunk.size() + realsize); + if (chunk.capacity() < chunk.size() + realsize) { + auto cl = ho.header.find("Content-Length"); + if (cl != ho.header.end()) { + sz = std::max(chunk.size() + realsize, (size_t)std::stol(cl->second)); + } else { + sz = chunk.size() + realsize; + LOGP(debug, "SIZE IS NOT IN HEADER, allocate {}", sz); + } + chunk.reserve(sz); + } char* contC = (char*)contents; chunk.insert(chunk.end(), contC, contC + realsize); } catch (std::exception e) { - LOGP(info, "failed to expand by {} bytes chunk provided to CURL: {}", realsize, e.what()); + LOGP(alarm, "failed to reserve {} bytes in CURL write callback (realsize = {}): {}", sz, realsize, e.what()); realsize = 0; } return realsize; @@ -1587,17 +1599,17 @@ void CcdbApi::navigateURLsAndLoadFileToMemory(o2::pmr::vector& dest, CURL* // specify URL to get curl_easy_setopt(curl_handle, CURLOPT_URL, url.c_str()); - initCurlOptionsForRetrieve(curl_handle, (void*)&dest, writeCallBack, false); - curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback); - headerData.clear(); - curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&headerData); + initCurlOptionsForRetrieve(curl_handle, (void*)&hoPair, writeCallBack, false); + curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, header_map_callback); + hoPair.header.clear(); + curl_easy_setopt(curl_handle, CURLOPT_HEADERDATA, (void*)&hoPair.header); curlSetSSLOptions(curl_handle); auto res = CURL_perform(curl_handle); long response_code = -1; if (res == CURLE_OK && curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &response_code) == CURLE_OK) { if (headers) { - for (auto& p : headerData) { + for (auto& p : hoPair.header) { (*headers)[p.first] = p.second; } } @@ -1621,14 +1633,14 @@ void CcdbApi::navigateURLsAndLoadFileToMemory(o2::pmr::vector& dest, CURL* }; std::vector locs; - auto iter = headerData.find("Location"); - if (iter != headerData.end()) { + auto iter = hoPair.header.find("Location"); + if (iter != hoPair.header.end()) { locs.push_back(complement_Location(iter->second)); } // add alternative locations (not yet included) - auto iter2 = headerData.find("Content-Location"); - if (iter2 != headerData.end()) { - auto range = headerData.equal_range("Content-Location"); + auto iter2 = hoPair.header.find("Content-Location"); + if (iter2 != hoPair.header.end()) { + auto range = hoPair.header.equal_range("Content-Location"); for (auto it = range.first; it != range.second; ++it) { if (std::find(locs.begin(), locs.end(), it->second) == locs.end()) { locs.push_back(complement_Location(it->second)); From 1314c4a2d6f1831b7134e242b04a962f0764661b Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sun, 18 Jun 2023 19:48:39 +0200 Subject: [PATCH 016/289] DPL: Create consumeWhenAnyWithAllConditions completion policy, and revert consumeWhenAny to original behavior --- .../Framework/CompletionPolicyHelpers.h | 9 +++++++ .../Core/src/CompletionPolicyHelpers.cxx | 25 +++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/Framework/Core/include/Framework/CompletionPolicyHelpers.h b/Framework/Core/include/Framework/CompletionPolicyHelpers.h index 5c40e3060b655..c096fb4cbe5bf 100644 --- a/Framework/Core/include/Framework/CompletionPolicyHelpers.h +++ b/Framework/Core/include/Framework/CompletionPolicyHelpers.h @@ -51,6 +51,15 @@ struct CompletionPolicyHelpers { } static CompletionPolicy consumeWhenAny(std::string matchName); + /// When any of the parts of the record have been received, consume them. + static CompletionPolicy consumeWhenAnyWithAllConditions(const char* name, CompletionPolicy::Matcher matcher); + /// Default matcher applies for all devices + static CompletionPolicy consumeWhenAnyWithAllConditions(CompletionPolicy::Matcher matcher = [](auto const&) -> bool { return true; }) + { + return consumeWhenAnyWithAllConditions("consume-any-all-conditions", matcher); + } + static CompletionPolicy consumeWhenAnyWithAllConditions(std::string matchName); + /// When any of the parts of the record have been received, process the existing and free the associated payloads. /// This allows freeing things as early as possible, while still being able to wait /// all the parts before disposing the timeslice completely diff --git a/Framework/Core/src/CompletionPolicyHelpers.cxx b/Framework/Core/src/CompletionPolicyHelpers.cxx index 403261a5a21d3..89febf0e17985 100644 --- a/Framework/Core/src/CompletionPolicyHelpers.cxx +++ b/Framework/Core/src/CompletionPolicyHelpers.cxx @@ -198,6 +198,27 @@ CompletionPolicy CompletionPolicyHelpers::consumeExistingWhenAny(const char* nam } CompletionPolicy CompletionPolicyHelpers::consumeWhenAny(const char* name, CompletionPolicy::Matcher matcher) +{ + auto callback = [](InputSpan const& inputs) -> CompletionPolicy::CompletionOp { + for (auto& input : inputs) { + if (input.header != nullptr) { + return CompletionPolicy::CompletionOp::Consume; + } + } + return CompletionPolicy::CompletionOp::Wait; + }; + return CompletionPolicy{name, matcher, callback, false}; +} + +CompletionPolicy CompletionPolicyHelpers::consumeWhenAny(std::string matchName) +{ + auto matcher = [matchName](DeviceSpec const& device) -> bool { + return std::regex_match(device.name.begin(), device.name.end(), std::regex(matchName)); + }; + return consumeWhenAny(matcher); +} + +CompletionPolicy CompletionPolicyHelpers::consumeWhenAnyWithAllConditions(const char* name, CompletionPolicy::Matcher matcher) { auto callback = [](InputSpan const& inputs, std::vector const& specs) -> CompletionPolicy::CompletionOp { bool canConsume = false; @@ -258,12 +279,12 @@ CompletionPolicy CompletionPolicyHelpers::consumeWhenAny(const char* name, Compl return CompletionPolicy{name, matcher, callback, false}; } -CompletionPolicy CompletionPolicyHelpers::consumeWhenAny(std::string matchName) +CompletionPolicy CompletionPolicyHelpers::consumeWhenAnyWithAllConditions(std::string matchName) { auto matcher = [matchName](DeviceSpec const& device) -> bool { return std::regex_match(device.name.begin(), device.name.end(), std::regex(matchName)); }; - return consumeWhenAny(matcher); + return consumeWhenAnyWithAllConditions(matcher); } CompletionPolicy CompletionPolicyHelpers::processWhenAny(const char* name, CompletionPolicy::Matcher matcher) From fdb78f6023aa6cd3d3d8e0dcb9a8bf55addae0b8 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sat, 17 Jun 2023 14:35:32 +0200 Subject: [PATCH 017/289] Suppress negative start orbit warning for SYNTHETIC runs --- .../include/DetectorsCalibration/TimeSlotCalibration.h | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h index 48057547b5156..2eb436ab357c8 100644 --- a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h +++ b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h @@ -196,7 +196,12 @@ class TimeSlotCalibration { long orb = long(mCurrentTFInfo.firstTForbit) - long(o2::base::GRPGeomHelper::getNHBFPerTF() * mCurrentTFInfo.tfCounter); if (orb < 0) { - LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit); + static bool suppressRunStartWarning = false; + if (mCurrentTFInfo.firstTForbit < 32 && mCurrentTFInfo.firstTForbit > 0) { + suppressRunStartWarning = true; + } else if (!suppressRunStartWarning && mCurrentTFInfo.firstTForbit >= 32) { // Otherwise this is most likely a SYNTHETIC run starting at orbit 0, so we don't need to print errors. + LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit); + } orb = 0; } return uint32_t(orb); From 25311fc81115e011029a07b32f7d4c9181819118 Mon Sep 17 00:00:00 2001 From: Michal Date: Sun, 18 Jun 2023 20:30:40 +0200 Subject: [PATCH 018/289] CCDBDownloader: More error checks #11489 --- CCDB/include/CCDB/CCDBDownloader.h | 5 ++ CCDB/src/CCDBDownloader.cxx | 123 ++++++++++++++++++----------- 2 files changed, 83 insertions(+), 45 deletions(-) diff --git a/CCDB/include/CCDB/CCDBDownloader.h b/CCDB/include/CCDB/CCDBDownloader.h index cad70fd8abb5a..e4382dbdd3fa1 100644 --- a/CCDB/include/CCDB/CCDBDownloader.h +++ b/CCDB/include/CCDB/CCDBDownloader.h @@ -187,6 +187,11 @@ class CCDBDownloader */ bool mIsExternalLoop; + /** + * Used in debug to detect whether uv loop closed prematurely. + */ + bool mIsClosing = false; + /** * Current amount of handles which are performed on. */ diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx index 13fda43511b6e..f502b1550c76a 100644 --- a/CCDB/src/CCDBDownloader.cxx +++ b/CCDB/src/CCDBDownloader.cxx @@ -12,8 +12,6 @@ #include #include -#include - #include #include #include @@ -26,10 +24,34 @@ #include #include #include +#include namespace o2::ccdb { +void uvErrorCheck(int code) +{ + if (code != 0) { + char buf[1000]; + uv_strerror_r(code, buf, 1000); + LOG(error) << "CCDBDownloader: UV error - " << buf; + } +} + +void curlEasyErrorCheck(CURLcode code) +{ + if (code != CURLE_OK) { + LOG(error) << "CCDBDownloader: CURL error - " << curl_easy_strerror(code); + } +} + +void curlMultiErrorCheck(CURLMcode code) +{ + if (code != CURLM_OK) { + LOG(error) << "CCDBDownloader: CURL error - " << curl_multi_strerror(code); + } +} + CCDBDownloader::CCDBDownloader(uv_loop_t* uv_loop) { if (uv_loop) { @@ -43,8 +65,8 @@ CCDBDownloader::CCDBDownloader(uv_loop_t* uv_loop) // Preparing timer to be used by curl mTimeoutTimer = new uv_timer_t(); mTimeoutTimer->data = this; - uv_loop_init(mUVLoop); - uv_timer_init(mUVLoop, mTimeoutTimer); + uvErrorCheck(uv_loop_init(mUVLoop)); + uvErrorCheck(uv_timer_init(mUVLoop, mTimeoutTimer)); mHandleMap[(uv_handle_t*)mTimeoutTimer] = true; // Preparing curl handle @@ -54,9 +76,9 @@ CCDBDownloader::CCDBDownloader(uv_loop_t* uv_loop) // uv_loop runs only when there are active handles, this handle guarantees the loop won't close immedietly after starting auto timerCheckQueueHandle = new uv_timer_t(); timerCheckQueueHandle->data = this; - uv_timer_init(mUVLoop, timerCheckQueueHandle); + uvErrorCheck(uv_timer_init(mUVLoop, timerCheckQueueHandle)); mHandleMap[(uv_handle_t*)timerCheckQueueHandle] = true; - uv_timer_start(timerCheckQueueHandle, checkStopSignal, 100, 100); + uvErrorCheck(uv_timer_start(timerCheckQueueHandle, checkStopSignal, 100, 100)); mLoopThread = new std::thread(&CCDBDownloader::runLoop, this); } @@ -64,25 +86,26 @@ CCDBDownloader::CCDBDownloader(uv_loop_t* uv_loop) void CCDBDownloader::initializeMultiHandle() { mCurlMultiHandle = curl_multi_init(); - curl_multi_setopt(mCurlMultiHandle, CURLMOPT_SOCKETFUNCTION, handleSocket); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_SOCKETFUNCTION, handleSocket)); auto socketData = &mSocketData; socketData->curlm = mCurlMultiHandle; socketData->CD = this; - curl_multi_setopt(mCurlMultiHandle, CURLMOPT_SOCKETDATA, socketData); - curl_multi_setopt(mCurlMultiHandle, CURLMOPT_TIMERFUNCTION, startTimeout); - curl_multi_setopt(mCurlMultiHandle, CURLMOPT_TIMERDATA, mTimeoutTimer); - curl_multi_setopt(mCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, mMaxHandlesInUse); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_SOCKETDATA, socketData)); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_TIMERFUNCTION, startTimeout)); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_TIMERDATA, mTimeoutTimer)); + curlMultiErrorCheck(curl_multi_setopt(mCurlMultiHandle, CURLMOPT_MAX_TOTAL_CONNECTIONS, mMaxHandlesInUse)); } CCDBDownloader::~CCDBDownloader() { + mIsClosing = true; // Cleanup and close all socket timers (curl_multi_cleanup will take care of the sockets) for (auto socketTimerPair : mSocketTimerMap) { auto timer = socketTimerPair.second; if (timer->data) { delete (DataForClosingSocket*)timer->data; } - uv_timer_stop(socketTimerPair.second); + uvErrorCheck(uv_timer_stop(socketTimerPair.second)); uv_close((uv_handle_t*)socketTimerPair.second, onUVClose); } // all timers have been closed --> so clear this map (otherwise it may get accessed in different callbacks again @@ -102,7 +125,7 @@ CCDBDownloader::~CCDBDownloader() while (UV_EBUSY == uv_loop_close(mUVLoop)) { mCloseLoop = false; uv_walk(mUVLoop, closeHandles, this); - uv_run(mUVLoop, UV_RUN_ONCE); + uvErrorCheck(uv_run(mUVLoop, UV_RUN_ONCE)); } delete mUVLoop; } @@ -110,7 +133,7 @@ CCDBDownloader::~CCDBDownloader() // delete timer // delete mTimeoutTimer; ---> not necessay (done elsewhere??) - curl_multi_cleanup(mCurlMultiHandle); + curlMultiErrorCheck(curl_multi_cleanup(mCurlMultiHandle)); } void closeHandles(uv_handle_t* handle, void* arg) @@ -134,7 +157,7 @@ void CCDBDownloader::checkStopSignal(uv_timer_t* handle) // Check for closing signal auto CD = (CCDBDownloader*)handle->data; if (CD->mCloseLoop) { - uv_timer_stop(handle); + uvErrorCheck(uv_timer_stop(handle)); uv_stop(CD->mUVLoop); } CD->checkForThreadsToJoin(); @@ -145,14 +168,16 @@ void CCDBDownloader::closesocketCallback(void* clientp, curl_socket_t item) auto CD = (CCDBDownloader*)clientp; if (CD->mSocketTimerMap.find(item) != CD->mSocketTimerMap.end()) { auto timer = CD->mSocketTimerMap[item]; - uv_timer_stop(timer); + uvErrorCheck(uv_timer_stop(timer)); // we are getting rid of the uv_timer_t pointer ... so we need // to free possibly attached user data pointers as well. Counteracts action of opensocketCallback if (timer->data) { delete (DataForClosingSocket*)timer->data; } CD->mSocketTimerMap.erase(item); - close(item); + if (close(item) == -1) { + LOG(error) << "CCDBDownloader: Socket failed to close"; + } } } @@ -160,9 +185,12 @@ curl_socket_t opensocketCallback(void* clientp, curlsocktype purpose, struct cur { auto CD = (CCDBDownloader*)clientp; auto sock = socket(address->family, address->socktype, address->protocol); + if (sock == -1) { + LOG(error) << "CCDBDownloader: Socket failed to open"; + } CD->mSocketTimerMap[sock] = new uv_timer_t(); - uv_timer_init(CD->mUVLoop, CD->mSocketTimerMap[sock]); + uvErrorCheck(uv_timer_init(CD->mUVLoop, CD->mSocketTimerMap[sock])); CD->mHandleMap[(uv_handle_t*)CD->mSocketTimerMap[sock]] = true; auto data = new DataForClosingSocket(); @@ -187,9 +215,11 @@ void CCDBDownloader::closeSocketByTimer(uv_timer_t* handle) auto sock = data->socket; if (CD->mSocketTimerMap.find(sock) != CD->mSocketTimerMap.end()) { - uv_timer_stop(CD->mSocketTimerMap[sock]); + uvErrorCheck(uv_timer_stop(CD->mSocketTimerMap[sock])); CD->mSocketTimerMap.erase(sock); - close(sock); + if (close(sock) == -1) { + LOG(error) << "CCDBDownloader: Socket failed to close"; + } delete data; } @@ -216,7 +246,7 @@ void CCDBDownloader::curlPerform(uv_poll_t* handle, int status, int events) auto context = (CCDBDownloader::curl_context_t*)handle->data; - curl_multi_socket_action(context->CD->mCurlMultiHandle, context->sockfd, flags, &running_handles); + curlMultiErrorCheck(curl_multi_socket_action(context->CD->mCurlMultiHandle, context->sockfd, flags, &running_handles)); context->CD->checkMultiInfo(); } @@ -233,7 +263,7 @@ int CCDBDownloader::handleSocket(CURL* easy, curl_socket_t s, int action, void* case CURL_POLL_INOUT: curl_context = socketp ? (CCDBDownloader::curl_context_t*)socketp : CD->createCurlContext(s); - curl_multi_assign(socketData->curlm, s, (void*)curl_context); + curlMultiErrorCheck(curl_multi_assign(socketData->curlm, s, (void*)curl_context)); if (action != CURL_POLL_IN) { events |= UV_WRITABLE; @@ -243,19 +273,19 @@ int CCDBDownloader::handleSocket(CURL* easy, curl_socket_t s, int action, void* } if (CD->mSocketTimerMap.find(s) != CD->mSocketTimerMap.end()) { - uv_timer_stop(CD->mSocketTimerMap[s]); + uvErrorCheck(uv_timer_stop(CD->mSocketTimerMap[s])); } - uv_poll_start(curl_context->poll_handle, events, curlPerform); + uvErrorCheck(uv_poll_start(curl_context->poll_handle, events, curlPerform)); break; case CURL_POLL_REMOVE: if (socketp) { if (CD->mSocketTimerMap.find(s) != CD->mSocketTimerMap.end()) { - uv_timer_start(CD->mSocketTimerMap[s], closeSocketByTimer, CD->mKeepaliveTimeoutMS, 0); + uvErrorCheck(uv_timer_start(CD->mSocketTimerMap[s], closeSocketByTimer, CD->mKeepaliveTimeoutMS, 0)); } - uv_poll_stop(((CCDBDownloader::curl_context_t*)socketp)->poll_handle); + uvErrorCheck(uv_poll_stop(((CCDBDownloader::curl_context_t*)socketp)->poll_handle)); CD->destroyCurlContext((CCDBDownloader::curl_context_t*)socketp); - curl_multi_assign(socketData->curlm, s, nullptr); + curlMultiErrorCheck(curl_multi_assign(socketData->curlm, s, nullptr)); } break; default: @@ -325,7 +355,7 @@ CCDBDownloader::curl_context_t* CCDBDownloader::createCurlContext(curl_socket_t context->sockfd = sockfd; context->poll_handle = new uv_poll_t(); - uv_poll_init_socket(mUVLoop, context->poll_handle, sockfd); + uvErrorCheck(uv_poll_init_socket(mUVLoop, context->poll_handle, sockfd)); mHandleMap[(uv_handle_t*)(context->poll_handle)] = true; context->poll_handle->data = context; @@ -354,9 +384,9 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode) { mHandlesInUse--; PerformData* data; - curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &data); + curlEasyErrorCheck(curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &data)); - curl_multi_remove_handle(mCurlMultiHandle, easy_handle); + curlMultiErrorCheck(curl_multi_remove_handle(mCurlMultiHandle, easy_handle)); *data->codeDestination = curlCode; // If no requests left then signal finished based on type of operation @@ -383,7 +413,7 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode) // Calling timeout starts a new download if a new easy_handle was added. int running_handles; - curl_multi_socket_action(mCurlMultiHandle, CURL_SOCKET_TIMEOUT, 0, &running_handles); + curlMultiErrorCheck(curl_multi_socket_action(mCurlMultiHandle, CURL_SOCKET_TIMEOUT, 0, &running_handles)); checkMultiInfo(); } @@ -411,27 +441,27 @@ int CCDBDownloader::startTimeout(CURLM* multi, long timeout_ms, void* userp) auto timeout = (uv_timer_t*)userp; if (timeout_ms < 0) { - uv_timer_stop(timeout); + uvErrorCheck(uv_timer_stop(timeout)); } else { if (timeout_ms == 0) { timeout_ms = 1; // Calling curlTimeout when timeout = 0 could create an infinite loop } - uv_timer_start(timeout, curlTimeout, timeout_ms, 0); + uvErrorCheck(uv_timer_start(timeout, curlTimeout, timeout_ms, 0)); } return 0; } void CCDBDownloader::setHandleOptions(CURL* handle, PerformData* data) { - curl_easy_setopt(handle, CURLOPT_PRIVATE, data); - curl_easy_setopt(handle, CURLOPT_CLOSESOCKETFUNCTION, closesocketCallback); - curl_easy_setopt(handle, CURLOPT_CLOSESOCKETDATA, this); - curl_easy_setopt(handle, CURLOPT_OPENSOCKETFUNCTION, opensocketCallback); - curl_easy_setopt(handle, CURLOPT_OPENSOCKETDATA, this); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_PRIVATE, data)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_CLOSESOCKETFUNCTION, closesocketCallback)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_CLOSESOCKETDATA, this)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_OPENSOCKETFUNCTION, opensocketCallback)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_OPENSOCKETDATA, this)); - curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, mRequestTimeoutMS); - curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, mConnectionTimeoutMS); - curl_easy_setopt(handle, CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS, mHappyEyeballsHeadstartMS); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, mRequestTimeoutMS)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, mConnectionTimeoutMS)); + curlEasyErrorCheck(curl_easy_setopt(handle, CURLOPT_HAPPY_EYEBALLS_TIMEOUT_MS, mHappyEyeballsHeadstartMS)); } void CCDBDownloader::checkHandleQueue() @@ -441,7 +471,7 @@ void CCDBDownloader::checkHandleQueue() if (mHandlesToBeAdded.size() > 0) { // Add handles without going over the limit while (mHandlesToBeAdded.size() > 0 && mHandlesInUse < mMaxHandlesInUse) { - curl_multi_add_handle(mCurlMultiHandle, mHandlesToBeAdded.front()); + curlMultiErrorCheck(curl_multi_add_handle(mCurlMultiHandle, mHandlesToBeAdded.front())); mHandlesInUse++; mHandlesToBeAdded.erase(mHandlesToBeAdded.begin()); } @@ -451,7 +481,10 @@ void CCDBDownloader::checkHandleQueue() void CCDBDownloader::runLoop() { - uv_run(mUVLoop, UV_RUN_DEFAULT); + uvErrorCheck(uv_run(mUVLoop, UV_RUN_DEFAULT)); + if (!mIsClosing) { + LOG(error) << "CCDBDownloader: uvloop closed prematurely"; + } } CURLcode CCDBDownloader::perform(CURL* handle) @@ -545,8 +578,8 @@ void CCDBDownloader::makeLoopCheckQueueAsync() { auto asyncHandle = new uv_async_t(); asyncHandle->data = this; - uv_async_init(mUVLoop, asyncHandle, asyncUVHandleCheckQueue); - uv_async_send(asyncHandle); + uvErrorCheck(uv_async_init(mUVLoop, asyncHandle, asyncUVHandleCheckQueue)); + uvErrorCheck(uv_async_send(asyncHandle)); } } // namespace o2 From 8e57f4958f0605e3455e2cb62b9efcca965ebb60 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 23 Jun 2023 09:09:05 +0200 Subject: [PATCH 019/289] dpl-workflow.sh: fix missing spaces --- prodtests/full-system-test/dpl-workflow.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh index 2959bbe294054..3262e3d3509b4 100755 --- a/prodtests/full-system-test/dpl-workflow.sh +++ b/prodtests/full-system-test/dpl-workflow.sh @@ -247,7 +247,7 @@ elif [[ -z "$DISABLE_ROOT_OUTPUT" ]] && has_detector_reco TOF && ! has_detector_ fi if has_detector_calib PHS && workflow_has_parameter CALIB; then - PHS_CONFIG+="--fullclu-output" + PHS_CONFIG+=" --fullclu-output" fi ( workflow_has_parameter AOD || [[ -z "$DISABLE_ROOT_OUTPUT" ]] || needs_root_output o2-emcal-cell-writer-workflow ) && has_detector EMC && RAW_EMC_SUBSPEC=" --subspecification 1 " @@ -518,8 +518,8 @@ workflow_has_parameter GPU_DISPLAY && [[ $NUMAID == 0 ]] && add_W o2-gpu-display # AOD : ${AOD_INPUT:=$TRACK_SOURCES} : ${AODPROD_OPT:=} -( ! has_detector_matching SECVTX || ! has_detectors_reco ITS || [[ $BEAMTYPE == "cosmic" ]]) && AODPROD_OPT+="--disable-secondary-vertices" -( ! has_detector_matching STRK || ! has_detector_matching SECVTX || ! has_detectors_reco ITS || [[ $BEAMTYPE == "cosmic" ]]) && AODPROD_OPT+="--disable-strangeness-tracking" +( ! has_detector_matching SECVTX || ! has_detectors_reco ITS || [[ $BEAMTYPE == "cosmic" ]]) && AODPROD_OPT+=" --disable-secondary-vertices" +( ! has_detector_matching STRK || ! has_detector_matching SECVTX || ! has_detectors_reco ITS || [[ $BEAMTYPE == "cosmic" ]]) && AODPROD_OPT+=" --disable-strangeness-tracking" workflow_has_parameter AOD && [[ ! -z "$AOD_INPUT" ]] && add_W o2-aod-producer-workflow "$AODPROD_OPT --info-sources $AOD_INPUT $DISABLE_DIGIT_ROOT_INPUT --aod-writer-keep dangling --aod-writer-resfile \"AO2D\" --aod-writer-resmode UPDATE $DISABLE_MC --pipeline $(get_N aod-producer-workflow AOD REST 1 AODPROD)" From c1ef8fe06098cf5205af22f7d42ba33f4ca56336 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 22 Jun 2023 23:05:26 +0200 Subject: [PATCH 020/289] GPU TPC: Don't search cluster grid outside of current TPC sector --- GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx | 4 ++++ GPU/GPUTracking/SliceTracker/GPUTPCRow.h | 2 ++ GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx | 4 ++-- GPU/GPUTracking/Standalone/CMakeLists.txt | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx index 59eab8569c174..095316f6505a1 100644 --- a/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx +++ b/GPU/GPUTracking/Merger/GPUTPCGMTrackParam.cxx @@ -564,6 +564,10 @@ GPUd() void GPUTPCGMTrackParam::AttachClusters(const GPUTPCGMMerger* GPUrestrict } else { Merger->GetConstantMem()->calibObjects.fastTransformHelper->InverseTransformYZtoNominalYZ(slice, iRow, Y, Z, nY, nZ); } + + if (CAMath::Abs(nY) > row.getTPCMaxY()) { + return; + } row.Grid().GetBinArea(nY, nZ + zOffset, tubeY, tubeZ, bin, ny, nz); const int nBinsY = row.Grid().Ny(); diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCRow.h b/GPU/GPUTracking/SliceTracker/GPUTPCRow.h index 369ceabf4ee96..7a6f00cb800cd 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCRow.h +++ b/GPU/GPUTracking/SliceTracker/GPUTPCRow.h @@ -56,6 +56,8 @@ class GPUTPCRow GPUhd() float HstepZi() const { return mHstepZi; } GPUhd() int HitNumberOffset() const { return mHitNumberOffset; } GPUhd() unsigned int FirstHitInBinOffset() const { return mFirstHitInBinOffset; } + GPUhd() static float getTPCMaxY1X() { return 0.1763269f; } // 0.1763269 = tan(2Pi / (2 * 18)) + GPUhd() float getTPCMaxY() const { return getTPCMaxY1X() * mX; } private: friend class GPUTPCNeighboursFinder; diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx index 441799b6871b7..c3547a2ac4cb1 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx @@ -298,8 +298,8 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int /*nBlocks*/, int #endif calink best = CALINK_INVAL; - { // search for the closest hit - tracker.GetErrors2Seeding(iRow, *((MEM_LG2(GPUTPCTrackParam)*)&tParam), err2Y, err2Z); + tracker.GetErrors2Seeding(iRow, *((MEM_LG2(GPUTPCTrackParam)*)&tParam), err2Y, err2Z); + if (CAMath::Abs(yUncorrected) < x * MEM_GLOBAL(GPUTPCRow)::getTPCMaxY1X()) { // search for the closest hit const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 7.0f * 7.0f; const float maxWindow2 = tracker.Param().rec.tpc.hitSearchArea2; const float sy2 = CAMath::Min(maxWindow2, kFactor * (tParam.Err2Y() + err2Y)); diff --git a/GPU/GPUTracking/Standalone/CMakeLists.txt b/GPU/GPUTracking/Standalone/CMakeLists.txt index 0422bff1d8828..f4e4796cfc27e 100644 --- a/GPU/GPUTracking/Standalone/CMakeLists.txt +++ b/GPU/GPUTracking/Standalone/CMakeLists.txt @@ -49,7 +49,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED TRUE) set(CMAKE_POSITION_INDEPENDENT_CODE ON) if(BUILD_DEBUG) - set(CMAKE_CXX_FLAGS "-O0 -ggdb") + set(CMAKE_CXX_FLAGS "-O0 -ggdb") # -fsanitize=address,undefined -fno-sanitize=vptr set(CMAKE_BUILD_TYPE DEBUG) else() set(CMAKE_CXX_FLAGS "-O3 -march=native -ggdb -minline-all-stringops -funroll-loops -fno-stack-protector") From 25d115514712946d7e2a50c03cfd3462d9f3eac1 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Thu, 22 Jun 2023 23:06:01 +0200 Subject: [PATCH 021/289] GPU TPC: temporary fix to avoid querrying the splines at invalid places --- GPU/TPCFastTransformation/Spline1DSpec.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GPU/TPCFastTransformation/Spline1DSpec.h b/GPU/TPCFastTransformation/Spline1DSpec.h index c11fc2e782948..6073d0c36c3a3 100644 --- a/GPU/TPCFastTransformation/Spline1DSpec.h +++ b/GPU/TPCFastTransformation/Spline1DSpec.h @@ -230,7 +230,7 @@ GPUdi() int Spline1DContainer::getLeftKnotIndexForU(DataT u) const { /// Get i: u is in [knot_i, knot_{i+1}) segment /// when u is otside of [0, mUmax], return a corresponding edge segment - int iu = (int)u; + int iu = u < 0 ? 0 : (u > (float)mUmax ? mUmax : (int)u); if (SafeT == SafetyLevel::kSafe) { iu = (iu < 0) ? 0 : (iu > mUmax ? mUmax : iu); } From f162fbaccc4406780bb31c306611f444a265a1ab Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 23 Jun 2023 00:01:55 +0200 Subject: [PATCH 022/289] GPU: Fix sanitizer warning (though didn't lead to real invalid access) --- GPU/GPUTracking/Definitions/GPUDefGPUParameters.h | 2 +- GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/GPU/GPUTracking/Definitions/GPUDefGPUParameters.h b/GPU/GPUTracking/Definitions/GPUDefGPUParameters.h index e0122daf11e04..07f9035329847 100644 --- a/GPU/GPUTracking/Definitions/GPUDefGPUParameters.h +++ b/GPU/GPUTracking/Definitions/GPUDefGPUParameters.h @@ -558,7 +558,7 @@ #endif #define GPUCA_MAX_THREADS 1024 -#define GPUCA_MAX_STREAMS 32 +#define GPUCA_MAX_STREAMS 36 #define GPUCA_SORT_STARTHITS_GPU // Sort the start hits when running on GPU #define GPUCA_ROWALIGNMENT 16 // Align of Row Hits and Grid diff --git a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx index 3f49b1ea95595..6043dd4e2a471 100644 --- a/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx +++ b/GPU/GPUTracking/Global/GPUChainTrackingClusterizer.cxx @@ -602,6 +602,7 @@ int GPUChainTracking::RunTPCClusterizer(bool synchronizeOutput) for (unsigned int iSliceBase = 0; iSliceBase < NSLICES; iSliceBase += GetProcessingSettings().nTPCClustererLanes) { std::vector laneHasData(GetProcessingSettings().nTPCClustererLanes, false); + static_assert(NSLICES <= GPUCA_MAX_STREAMS, "Stream events must be able to hold all slices"); const int maxLane = std::min(GetProcessingSettings().nTPCClustererLanes, NSLICES - iSliceBase); for (CfFragment fragment = mCFContext->fragmentFirst; !fragment.isEnd(); fragment = fragment.next()) { if (GetProcessingSettings().debugLevel >= 3) { From 1d368797d73d1c480fcf191f76b00383c3db1dff Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 23 Jun 2023 00:19:10 +0200 Subject: [PATCH 023/289] GPU: Avoid false compiler warnings --- .../SliceTracker/GPUTPCTrackletConstructor.cxx | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx index c3547a2ac4cb1..6afe7d2557c8f 100644 --- a/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx +++ b/GPU/GPUTracking/SliceTracker/GPUTPCTrackletConstructor.cxx @@ -163,7 +163,6 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int /*nBlocks*/, int r.mLastY = y; CADEBUG(printf("Tracklet %5d: FIT INIT ROW %3d X %8.3f -", r.mISH, iRow, tParam.X()); for (int i = 0; i < 5; i++) { printf(" %8.3f", tParam.Par()[i]); } printf(" -"); for (int i = 0; i < 15; i++) { printf(" %8.3f", tParam.Cov()[i]); } printf("\n")); } else { - float err2Y, err2Z; float dx = x - tParam.X(); float dy, dz; if (r.mNHits >= 10) { @@ -177,13 +176,12 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int /*nBlocks*/, int r.mLastZ = z; float ri = 1.f / CAMath::Sqrt(dx * dx + dy * dy); - if (iRow == r.mStartRow + 2) { // SG!!! important - thanks to Matthias + if (iRow == r.mStartRow + 2) { tParam.SetSinPhi(dy * ri); tParam.SetSignCosPhi(dx); tParam.SetDzDs(dz * ri); - // std::cout << "Init. errors... " << r.mISH << std::endl; + float err2Y, err2Z; tracker.GetErrors2Seeding(iRow, tParam, err2Y, err2Z); - // std::cout << "Init. errors = " << err2Y << " " << err2Z << std::endl; tParam.SetCov(0, err2Y); tParam.SetCov(2, err2Z); } @@ -201,6 +199,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int /*nBlocks*/, int break; } CADEBUG(printf("%5s hits %3d: FIT PROP ROW %3d X %8.3f -", "", r.mNHits, iRow, tParam.X()); for (int i = 0; i < 5; i++) { printf(" %8.3f", tParam.Par()[i]); } printf(" -"); for (int i = 0; i < 15; i++) { printf(" %8.3f", tParam.Cov()[i]); } printf("\n")); + float err2Y, err2Z; tracker.GetErrors2Seeding(iRow, tParam.GetZ(), sinPhi, tParam.GetDzDs(), err2Y, err2Z); if (r.mNHits >= 10) { @@ -275,7 +274,6 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int /*nBlocks*/, int } #endif - float err2Y, err2Z; CADEBUG(printf("%14s: SEA TRACK ROW %3d X %8.3f -", "", iRow, tParam.X()); for (int i = 0; i < 5; i++) { printf(" %8.3f", tParam.Par()[i]); } printf(" -"); for (int i = 0; i < 15; i++) { printf(" %8.3f", tParam.Cov()[i]); } printf("\n")); if (!tParam.TransportToX(x, tParam.SinPhi(), tParam.GetCosPhi(), tracker.Param().constBz, GPUCA_MAX_SIN_PHI_LOW)) { r.mGo = 0; @@ -298,6 +296,7 @@ GPUdic(2, 1) void GPUTPCTrackletConstructor::UpdateTracklet(int /*nBlocks*/, int #endif calink best = CALINK_INVAL; + float err2Y, err2Z; tracker.GetErrors2Seeding(iRow, *((MEM_LG2(GPUTPCTrackParam)*)&tParam), err2Y, err2Z); if (CAMath::Abs(yUncorrected) < x * MEM_GLOBAL(GPUTPCRow)::getTPCMaxY1X()) { // search for the closest hit const float kFactor = tracker.Param().rec.tpc.hitPickUpFactor * tracker.Param().rec.tpc.hitPickUpFactor * 7.0f * 7.0f; From cc0d84c301854b4d043a19039e12297c7bf8fde5 Mon Sep 17 00:00:00 2001 From: Michal Date: Mon, 19 Jun 2023 10:59:18 +0200 Subject: [PATCH 024/289] CCDBDownloader: Additional error check --- CCDB/src/CCDBDownloader.cxx | 1 + 1 file changed, 1 insertion(+) diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx index f502b1550c76a..1a7bd7d09c2c0 100644 --- a/CCDB/src/CCDBDownloader.cxx +++ b/CCDB/src/CCDBDownloader.cxx @@ -235,6 +235,7 @@ void CCDBDownloader::curlTimeout(uv_timer_t* handle) void CCDBDownloader::curlPerform(uv_poll_t* handle, int status, int events) { + uvErrorCheck(status); int running_handles; int flags = 0; if (events & UV_READABLE) { From 316c4e1465991385e83a89e7ca30a020ffbdd296 Mon Sep 17 00:00:00 2001 From: Michal Date: Thu, 22 Jun 2023 11:24:17 +0200 Subject: [PATCH 025/289] Improved thread safety in construtor and destructor of Downloader --- CCDB/include/CCDB/CCDBDownloader.h | 43 +++++---- CCDB/src/CCDBDownloader.cxx | 131 ++++++++++++++-------------- CCDB/test/testCcdbApiDownloader.cxx | 28 ------ 3 files changed, 95 insertions(+), 107 deletions(-) diff --git a/CCDB/include/CCDB/CCDBDownloader.h b/CCDB/include/CCDB/CCDBDownloader.h index e4382dbdd3fa1..68bbf9f932972 100644 --- a/CCDB/include/CCDB/CCDBDownloader.h +++ b/CCDB/include/CCDB/CCDBDownloader.h @@ -114,8 +114,7 @@ class CCDBDownloader */ int mMaxHandlesInUse = 3; - // CCDBDownloader(uv_loop_t uv_loop); - CCDBDownloader(uv_loop_t* uv_loop = nullptr); + CCDBDownloader(); ~CCDBDownloader(); /** @@ -145,6 +144,9 @@ class CCDBDownloader */ std::vector batchAsynchPerform(std::vector const& handleVector, bool* completionFlag); + void sendCloseLoopHandle(); + static void closeLoop(uv_async_t* handle); + /** * Limits the number of parallel connections. Should be used only if no transfers are happening. */ @@ -181,12 +183,6 @@ class CCDBDownloader void setOnlineTimeoutSettings(); private: - /** - * Indicates whether the loop that the downloader is running on has been created by it or provided externally. - * In case of external loop, the loop will not be closed after downloader is deleted. - */ - bool mIsExternalLoop; - /** * Used in debug to detect whether uv loop closed prematurely. */ @@ -217,6 +213,16 @@ class CCDBDownloader */ std::mutex mHandlesQueueLock; + /** + * Blocks the constructor from returning before the uv_loop has started running + */ + std::condition_variable* mConstructorCV; + + /** + * Prevents the mConstructorCV from being notified after the constructor returned + */ + bool mLoopRunning = false; + /** * Thread on which the thread with uv_loop runs. */ @@ -227,11 +233,6 @@ class CCDBDownloader */ std::vector> mThreadFlagPairVector; - /** - * Flag used to signall the loop to close. - */ - bool mCloseLoop = false; - /** * Types of requests. */ @@ -293,11 +294,11 @@ class CCDBDownloader static void curlPerform(uv_poll_t* handle, int status, int events); /** - * Check if loop was signalled to close. The handle connected with this callbacks is always active as to prevent the uv_loop from stopping. + * This handle should be always running to keep the uv_loop from closing. * * @param handle uv_handle to which this callbacks is assigned */ - static void checkStopSignal(uv_timer_t* handle); + static void upkeepTimerFunction(uv_timer_t* handle); /** * Used by CURL to react to action happening on a socket. @@ -332,6 +333,13 @@ class CCDBDownloader */ static void curlCloseCB(uv_handle_t* handle); + /** + * Callback for the asynchronous handle that closes the uv_loop + * + * @param handle Handle assigned to this callback. + */ + static void uvCloseCallback(uv_async_t* handle); + /** * Close poll handle assigned to the socket contained in the context and free data within the handle. * @@ -348,6 +356,11 @@ class CCDBDownloader */ static int startTimeout(CURLM* multi, long timeout_ms, void* userp); + /** + * Sends an asynchronous handle that signalls the uvloop to close. + */ + void signalToClose(); + /** * Check if any of the callback threads have finished running and approprietly join them. */ diff --git a/CCDB/src/CCDBDownloader.cxx b/CCDB/src/CCDBDownloader.cxx index 1a7bd7d09c2c0..7a0fb60840714 100644 --- a/CCDB/src/CCDBDownloader.cxx +++ b/CCDB/src/CCDBDownloader.cxx @@ -52,35 +52,19 @@ void curlMultiErrorCheck(CURLMcode code) } } -CCDBDownloader::CCDBDownloader(uv_loop_t* uv_loop) +CCDBDownloader::CCDBDownloader() { - if (uv_loop) { - mUVLoop = uv_loop; - mIsExternalLoop = true; - } else { - mUVLoop = new uv_loop_t(); - mIsExternalLoop = false; - } - - // Preparing timer to be used by curl - mTimeoutTimer = new uv_timer_t(); - mTimeoutTimer->data = this; - uvErrorCheck(uv_loop_init(mUVLoop)); - uvErrorCheck(uv_timer_init(mUVLoop, mTimeoutTimer)); - mHandleMap[(uv_handle_t*)mTimeoutTimer] = true; - - // Preparing curl handle - initializeMultiHandle(); - - // Global timer - // uv_loop runs only when there are active handles, this handle guarantees the loop won't close immedietly after starting - auto timerCheckQueueHandle = new uv_timer_t(); - timerCheckQueueHandle->data = this; - uvErrorCheck(uv_timer_init(mUVLoop, timerCheckQueueHandle)); - mHandleMap[(uv_handle_t*)timerCheckQueueHandle] = true; - uvErrorCheck(uv_timer_start(timerCheckQueueHandle, checkStopSignal, 100, 100)); + mConstructorCV = new std::condition_variable(); + std::mutex cv_m; + std::unique_lock lk(cv_m); mLoopThread = new std::thread(&CCDBDownloader::runLoop, this); + + // Don't allow constructor to return unless the uv_loop started running. + // This protects the loop from receiving handles before it was initialized. + mConstructorCV->wait(lk); + mLoopRunning = true; + delete mConstructorCV; } void CCDBDownloader::initializeMultiHandle() @@ -98,49 +82,36 @@ void CCDBDownloader::initializeMultiHandle() CCDBDownloader::~CCDBDownloader() { + // Flag used for debug. Indicates that the uv_loop is supposed to close. mIsClosing = true; - // Cleanup and close all socket timers (curl_multi_cleanup will take care of the sockets) - for (auto socketTimerPair : mSocketTimerMap) { - auto timer = socketTimerPair.second; - if (timer->data) { - delete (DataForClosingSocket*)timer->data; - } - uvErrorCheck(uv_timer_stop(socketTimerPair.second)); - uv_close((uv_handle_t*)socketTimerPair.second, onUVClose); - } - // all timers have been closed --> so clear this map (otherwise it may get accessed in different callbacks again - // ... for instance when called from within curl_multi_cleanup) - mSocketTimerMap.clear(); - // Close loop thread - mCloseLoop = true; + // Send asynchronous signal to close the loop + signalToClose(); mLoopThread->join(); - delete mLoopThread; - // Close the loop and if any handles are running then signal to close, and run loop once to close them - // This may take more then one iteration of loop - hence the "while" - if (mIsExternalLoop) { - uv_walk(mUVLoop, closeHandles, this); - } else { - while (UV_EBUSY == uv_loop_close(mUVLoop)) { - mCloseLoop = false; - uv_walk(mUVLoop, closeHandles, this); - uvErrorCheck(uv_run(mUVLoop, UV_RUN_ONCE)); - } - delete mUVLoop; - } + delete mLoopThread; + delete mUVLoop; +} - // delete timer - // delete mTimeoutTimer; ---> not necessay (done elsewhere??) +void CCDBDownloader::signalToClose() +{ + auto asyncHandle = new uv_async_t(); + asyncHandle->data = this; + uvErrorCheck(uv_async_init(mUVLoop, asyncHandle, uvCloseCallback)); + uvErrorCheck(uv_async_send(asyncHandle)); +} - curlMultiErrorCheck(curl_multi_cleanup(mCurlMultiHandle)); +void CCDBDownloader::uvCloseCallback(uv_async_t* handle) +{ + auto CD = (CCDBDownloader*)handle->data; + uv_close((uv_handle_t*)handle, onUVClose); + uv_stop(CD->mUVLoop); } void closeHandles(uv_handle_t* handle, void* arg) { auto CD = (CCDBDownloader*)arg; - if (!uv_is_closing(handle) && CD->mHandleMap.find(handle) != CD->mHandleMap.end()) { - CD->mHandleMap.erase(handle); + if (!uv_is_closing(handle)) { uv_close(handle, onUVClose); } } @@ -152,13 +123,13 @@ void onUVClose(uv_handle_t* handle) } } -void CCDBDownloader::checkStopSignal(uv_timer_t* handle) +void CCDBDownloader::upkeepTimerFunction(uv_timer_t* handle) { - // Check for closing signal auto CD = (CCDBDownloader*)handle->data; - if (CD->mCloseLoop) { - uvErrorCheck(uv_timer_stop(handle)); - uv_stop(CD->mUVLoop); + if (!CD->mLoopRunning) { + // The fact that this callback is executed means the uv_loop is running. + // Notify the constructor if it hasn't been already. + CD->mConstructorCV->notify_all(); } CD->checkForThreadsToJoin(); } @@ -482,7 +453,39 @@ void CCDBDownloader::checkHandleQueue() void CCDBDownloader::runLoop() { - uvErrorCheck(uv_run(mUVLoop, UV_RUN_DEFAULT)); + // Only runLoop() function and functions called by it are allowed to interact with the uv_loop via non asynchronous handles. + mUVLoop = new uv_loop_t(); + uvErrorCheck(uv_loop_init(mUVLoop)); + + // Preparing timer to be used by curl + mTimeoutTimer = new uv_timer_t(); + mTimeoutTimer->data = this; + uvErrorCheck(uv_timer_init(mUVLoop, mTimeoutTimer)); + mHandleMap[(uv_handle_t*)mTimeoutTimer] = true; + + // Preparing curl handle + initializeMultiHandle(); + + // Global timer + // uv_loop runs only when there are active handles, this handle guarantees the loop won't close immedietly after starting + auto timerCheckQueueHandle = new uv_timer_t(); + timerCheckQueueHandle->data = this; + uvErrorCheck(uv_timer_init(mUVLoop, timerCheckQueueHandle)); + mHandleMap[(uv_handle_t*)timerCheckQueueHandle] = true; + uvErrorCheck(uv_timer_start(timerCheckQueueHandle, upkeepTimerFunction, 1, 100)); + + // Start the loop + uv_run(mUVLoop, UV_RUN_DEFAULT); + + // Loop has been ordered to stop via signalToClose() + curlMultiErrorCheck(curl_multi_cleanup(mCurlMultiHandle)); + + // Schedule all handles to close. Execute loop once to allow them to execute their destructors. + while (uv_loop_alive(mUVLoop) && uv_loop_close(mUVLoop) == UV_EBUSY) { + uv_walk(mUVLoop, closeHandles, this); + uv_run(mUVLoop, UV_RUN_ONCE); + } + if (!mIsClosing) { LOG(error) << "CCDBDownloader: uvloop closed prematurely"; } diff --git a/CCDB/test/testCcdbApiDownloader.cxx b/CCDB/test/testCcdbApiDownloader.cxx index 80e2826209bb4..caed41278004a 100644 --- a/CCDB/test/testCcdbApiDownloader.cxx +++ b/CCDB/test/testCcdbApiDownloader.cxx @@ -274,33 +274,5 @@ BOOST_AUTO_TEST_CASE(asynch_batch_callback) curl_global_cleanup(); } -BOOST_AUTO_TEST_CASE(external_loop_test) -{ - if (curl_global_init(CURL_GLOBAL_ALL)) { - fprintf(stderr, "Could not init curl\n"); - return; - } - - uv_loop_t loop; - - CCDBDownloader downloader(&loop); - std::string dst = ""; - CURL* handle = createTestHandle(&dst); - - CURLcode curlCode = downloader.perform(handle); - - BOOST_CHECK(curlCode == CURLE_OK); - std::cout << "CURL code: " << curlCode << "\n"; - - long httpCode; - curl_easy_getinfo(handle, CURLINFO_HTTP_CODE, &httpCode); - BOOST_CHECK(httpCode == 200); - std::cout << "HTTP code: " << httpCode << "\n"; - - curl_easy_cleanup(handle); - - curl_global_cleanup(); -} - } // namespace ccdb } // namespace o2 From 245df1dcbed7104346bc2fa4eb21800267d538a6 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 23 Jun 2023 14:58:16 +0200 Subject: [PATCH 026/289] DPL: Raw parser should check packetCounter and pageCounter, since in UL data with multiple links, the packetCounter is not guaranteed to be incremental --- Framework/Utils/include/DPLUtils/RawParser.h | 6 ++++-- Framework/Utils/test/RawPageTestData.cxx | 9 +++++++++ Framework/Utils/test/test_RawParser.cxx | 2 +- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Framework/Utils/include/DPLUtils/RawParser.h b/Framework/Utils/include/DPLUtils/RawParser.h index 49e64506768e5..a0374804b63ea 100644 --- a/Framework/Utils/include/DPLUtils/RawParser.h +++ b/Framework/Utils/include/DPLUtils/RawParser.h @@ -237,6 +237,7 @@ class ConcreteRawParser bool next() { int lastPacketCounter = -1; + int lastHBFPacketCounter = -1; unsigned int lastFEEID = -1; if (mPosition == nullptr) { mPosition = mRawBuffer; @@ -251,6 +252,7 @@ class ConcreteRawParser } if (RawParserHelper::sCheckIncompleteHBF) { lastPacketCounter = header().packetCounter; + lastHBFPacketCounter = header().pageCnt; lastFEEID = header().feeId; } mPosition += offset; @@ -278,13 +280,13 @@ class ConcreteRawParser mPosition = mRawBuffer + mSize; return false; } - if (lastPacketCounter != -1 && (unsigned char)(lastPacketCounter + 1) != header().packetCounter && lastFEEID == header().feeId) { + // FIXME: The && should be ||, since both packetCounter and pageCnt should be incremental. But currently, UL data with multiple links does not have incremental packetCounter. + if (lastPacketCounter != -1 && lastFEEID == header().feeId && ((unsigned char)(lastPacketCounter + 1) != header().packetCounter && (unsigned short)(lastHBFPacketCounter + 1) != header().pageCnt)) { if (RawParserHelper::sErrorMode >= 2 && RawParserHelper::sCheckIncompleteHBF >= 2) { throw std::runtime_error("Incomplete HBF - jump in packet counter"); } if (RawParserHelper::checkPrintError(mNErrors)) { LOG(error) << "RAWPARSER: Incomplete HBF - jump in packet counter " << lastPacketCounter << " to " << header().packetCounter << " (" << RawParserHelper::sErrors << " total RawParser errors)"; - ; } mPosition = mRawBuffer + mSize; return false; diff --git a/Framework/Utils/test/RawPageTestData.cxx b/Framework/Utils/test/RawPageTestData.cxx index 5c57695a98e9b..d05bf45246bbb 100644 --- a/Framework/Utils/test/RawPageTestData.cxx +++ b/Framework/Utils/test/RawPageTestData.cxx @@ -48,6 +48,9 @@ DataSet createData(std::vector const& inputspecs, std::vector(wrtptr); @@ -55,8 +58,14 @@ DataSet createData(std::vector const& inputspecs, std::vectorfeeId != lastFEEID || header->orbit != lastOrbit) { + pageCounter = 0; + } + lastFEEID = header->feeId; + lastOrbit = header->orbit; header->memorySize = PAGESIZE; header->offsetToNext = PAGESIZE; + header->pageCnt = pageCounter++; header->packetCounter = packetCounter++; *reinterpret_cast(wrtptr + header->headerSize) = value; wrtptr += PAGESIZE; diff --git a/Framework/Utils/test/test_RawParser.cxx b/Framework/Utils/test/test_RawParser.cxx index 82ee871f551cc..7b07a8ead38a9 100644 --- a/Framework/Utils/test/test_RawParser.cxx +++ b/Framework/Utils/test/test_RawParser.cxx @@ -33,7 +33,7 @@ void fillPages(Container& buffer) rdh->headerSize = sizeof(RDH); rdh->offsetToNext = PageSize; rdh->memorySize = PageSize; - rdh->pageCnt = NofPages; + rdh->pageCnt = pageNo; rdh->packetCounter = pageNo; rdh->stop = pageNo + 1 == NofPages; auto* data = reinterpret_cast(buffer.data() + pageNo * PageSize + rdh->headerSize); From c58f9d456e61c84b7728a21f77b973580a52b288 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Fri, 23 Jun 2023 15:39:54 +0200 Subject: [PATCH 027/289] GPU TPC: Use pageCounter instead of packetCounter to detect packet loss --- GPU/GPUTracking/Base/GPUReconstructionConvert.cxx | 7 ++++++- GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx b/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx index f02d11fa59bed..a605fd97803e9 100644 --- a/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx +++ b/GPU/GPUTracking/Base/GPUReconstructionConvert.cxx @@ -215,6 +215,7 @@ struct zsEncoder { float encodeBitsFactor = 0; bool needAnotherPage = false; unsigned int packetCounter = 0; + unsigned int pageCounter = 0; void ZSfillEmpty(void* ptr, int shift, unsigned int feeId, int orbit, int linkid); static void ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn, unsigned char* bufOut, unsigned int& lenOut, unsigned int nBits); long int getHbf(long int timestamp) { return (timestamp * LHCBCPERTIMEBIN + bcShiftInFirstHBF) / o2::constants::lhc::LHCMaxBunches; } @@ -231,6 +232,7 @@ inline void zsEncoder::ZSfillEmpty(void* ptr, int shift, unsigned int feeId, int o2::raw::RDHUtils::setDetectorField(*rdh, 2); o2::raw::RDHUtils::setLinkID(*rdh, linkid); o2::raw::RDHUtils::setPacketCounter(*rdh, packetCounter++); + o2::raw::RDHUtils::setPageCounter(*rdh, pageCounter++); } inline void zsEncoder::ZSstreamOut(unsigned short* bufIn, unsigned int& lenIn, unsigned char* bufOut, unsigned int& lenOut, unsigned int nBits) @@ -896,7 +898,7 @@ void zsEncoderDenseLinkBased::decodePage(std::vector& outputBuff const unsigned char* pageNext = ((const unsigned char*)decPage) + TPCZSHDR::TPC_ZS_PAGE_SIZE; const o2::header::RAWDataHeader* rdhNext = (const o2::header::RAWDataHeader*)pageNext; - if ((unsigned char)(o2::raw::RDHUtils::getPacketCounter(*rdh) + 1) != o2::raw::RDHUtils::getPacketCounter(*rdhNext)) { + if ((unsigned char)(o2::raw::RDHUtils::getPageCounter(*rdh) + 1) != o2::raw::RDHUtils::getPageCounter(*rdhNext)) { fprintf(stderr, "Incomplete HBF: Payload extended to next page, but next page missing in stream\n"); extendFailure = true; decPagePtr = payloadEnd; // Next 8kb page is missing in stream, cannot decode remaining data, skip it @@ -1037,6 +1039,7 @@ struct zsEncoderRun : public T { using T::packetCounter; using T::padding; using T::page; + using T::pageCounter; using T::pagePtr; using T::param; using T::raw; @@ -1151,6 +1154,7 @@ inline unsigned int zsEncoderRun::run(std::vector* buffer, std::vecto o2::raw::RDHUtils::setDetectorField(*rdh, 2); o2::raw::RDHUtils::setLinkID(*rdh, this->RAWLNK); o2::raw::RDHUtils::setPacketCounter(*rdh, packetCounter++); + o2::raw::RDHUtils::setPageCounter(*rdh, pageCounter++); } } if (k >= tmpBuffer.size() && !needAnotherPage) { @@ -1164,6 +1168,7 @@ inline unsigned int zsEncoderRun::run(std::vector* buffer, std::vecto hbf = nexthbf; lastTime = -1; lastEndpoint = endpoint; + pageCounter = 0; } if (raw) { page = &singleBuffer; diff --git a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx index 171935b10584b..a7f20cea170bb 100644 --- a/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx +++ b/GPU/GPUTracking/TPCClusterFinder/GPUTPCCFDecodeZS.cxx @@ -623,13 +623,13 @@ GPUd() uint32_t GPUTPCCFDecodeZSDenseLink::DecodePage(GPUSharedMemory& smem, pro if (i == decHeader->nTimebinHeaders - 1 && decHeader->flags & o2::tpc::TPCZSHDRV2::ZSFlags::payloadExtendsToNextPage) { assert(o2::raw::RDHUtils::getMemorySize(*rawDataHeader) == TPCZSHDR::TPC_ZS_PAGE_SIZE); // Disable check for dropped pages temporarily, decoding fails on large dataset when enabled... - if ((unsigned char)(raw::RDHUtils::getPacketCounter(rawDataHeader) + 1) == raw::RDHUtils::getPacketCounter(nextPage)) { + if ((unsigned char)(raw::RDHUtils::getPageCounter(rawDataHeader) + 1) == raw::RDHUtils::getPageCounter(nextPage)) { nSamplesWrittenTB = DecodeTB(clusterer, smem, iThread, page, pageDigitOffset, rawDataHeader, firstHBF, decHeader->cruID, payloadEnd, nextPage); } else { nSamplesWrittenTB = FillWithInvalid(clusterer, iThread, nThreads, pageDigitOffset, nSamplesInPage - nSamplesWritten); #ifdef GPUCA_CHECK_TPCZS_CORRUPTION if (iThread == 0) { - clusterer.raiseError(GPUErrors::ERROR_TPCZS_INCOMPLETE_HBF, clusterer.mISlice, raw::RDHUtils::getPacketCounter(rawDataHeader), raw::RDHUtils::getPacketCounter(nextPage)); + clusterer.raiseError(GPUErrors::ERROR_TPCZS_INCOMPLETE_HBF, clusterer.mISlice, raw::RDHUtils::getPageCounter(rawDataHeader), raw::RDHUtils::getPageCounter(nextPage)); } #endif } From f3e8935e9e39162277a75e604f49e4516e8cf99a Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sat, 24 Jun 2023 16:22:38 +0200 Subject: [PATCH 028/289] Revert "Suppress negative start orbit warning for SYNTHETIC runs" This reverts commit fdb78f6023aa6cd3d3d8e0dcb9a8bf55addae0b8. --- .../include/DetectorsCalibration/TimeSlotCalibration.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h index 2eb436ab357c8..48057547b5156 100644 --- a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h +++ b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h @@ -196,12 +196,7 @@ class TimeSlotCalibration { long orb = long(mCurrentTFInfo.firstTForbit) - long(o2::base::GRPGeomHelper::getNHBFPerTF() * mCurrentTFInfo.tfCounter); if (orb < 0) { - static bool suppressRunStartWarning = false; - if (mCurrentTFInfo.firstTForbit < 32 && mCurrentTFInfo.firstTForbit > 0) { - suppressRunStartWarning = true; - } else if (!suppressRunStartWarning && mCurrentTFInfo.firstTForbit >= 32) { // Otherwise this is most likely a SYNTHETIC run starting at orbit 0, so we don't need to print errors. - LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit); - } + LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit); orb = 0; } return uint32_t(orb); From 36637ad4506b9f361976a8f5a8aa72dc9aa40911 Mon Sep 17 00:00:00 2001 From: David Rohr Date: Sat, 17 Jun 2023 14:35:32 +0200 Subject: [PATCH 029/289] Suppress negative start orbit warning for SYNTHETIC runs --- .../DetectorsCalibration/TimeSlotCalibration.h | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h index 48057547b5156..08b8f1bd1fe56 100644 --- a/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h +++ b/Detectors/Calibration/include/DetectorsCalibration/TimeSlotCalibration.h @@ -195,8 +195,24 @@ class TimeSlotCalibration uint32_t getRunStartOrbit() const { long orb = long(mCurrentTFInfo.firstTForbit) - long(o2::base::GRPGeomHelper::getNHBFPerTF() * mCurrentTFInfo.tfCounter); + static unsigned int threshold = 512 * o2::base::GRPGeomHelper::getNHBFPerTF(); if (orb < 0) { - LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit); + // If we have a firstTForbit between 1 and 512 * tf len, we disable the warning for negative runStartOrbit permanently, since this is a SYNTHETIC run. + static bool suppressRunStartWarning = false; + if (!suppressRunStartWarning) { + const auto* grpecs = o2::base::GRPGeomHelper::instance().getGRPECS(); + if (grpecs) { + if (grpecs->getRunType() == o2::parameters::GRPECS::SYNTHETIC) { + suppressRunStartWarning = true; + } + } else if (mCurrentTFInfo.firstTForbit < threshold && mCurrentTFInfo.firstTForbit > 0) { + suppressRunStartWarning = true; + } + } + + if (!suppressRunStartWarning && mCurrentTFInfo.firstTForbit >= threshold) { + LOGP(alarm, "Negative runStartOrbit = {} deduced from tfCounter={} and firstTForbit={}, enforcing runStartOrbit to 0", orb, mCurrentTFInfo.tfCounter, mCurrentTFInfo.firstTForbit); + } orb = 0; } return uint32_t(orb); From 7cd76e8e5857c5c32c42b357775e1c3ac85e88b4 Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 27 Jun 2023 11:54:28 +0200 Subject: [PATCH 030/289] Lumi scaling options belong to workflow and not to device --- prodtests/full-system-test/dpl-workflow.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh index 77649eaaa2f9f..568317d030daa 100755 --- a/prodtests/full-system-test/dpl-workflow.sh +++ b/prodtests/full-system-test/dpl-workflow.sh @@ -235,7 +235,6 @@ if [[ $HOSTMEMSIZE != "0" ]]; then fi GPU_CONFIG_SELF="--severity $SEVERITY_TPC" -GPU_CONFIG_SELF+=" $TPC_CORR_SCALING " if [[ $GPUTYPE != "CPU" && $(ulimit -e) -ge 25 && ${O2_GPU_WORKFLOW_NICE:-} == 1 ]]; then GPU_CONFIG_SELF+=" --child-driver 'nice -n -5'" fi @@ -407,7 +406,7 @@ fi # --------------------------------------------------------------------------------------------------------------------- # Common reconstruction workflows -(has_detector_reco TPC || has_detector_ctf TPC) && ! has_detector_from_global_reader TPC && add_W o2-gpu-reco-workflow "--gpu-reconstruction \"$GPU_CONFIG_SELF\" --input-type=$GPU_INPUT $DISABLE_MC --output-type $GPU_OUTPUT --pipeline gpu-reconstruction:${N_TPCTRK:-1} $GPU_CONFIG" "GPU_global.deviceType=$GPUTYPE;GPU_proc.debugLevel=0;$GPU_CONFIG_KEY;$TRACKTUNETPCINNER" +(has_detector_reco TPC || has_detector_ctf TPC) && ! has_detector_from_global_reader TPC && add_W o2-gpu-reco-workflow "--gpu-reconstruction \"$GPU_CONFIG_SELF\" $TPC_CORR_SCALING --input-type=$GPU_INPUT $DISABLE_MC --output-type $GPU_OUTPUT --pipeline gpu-reconstruction:${N_TPCTRK:-1} $GPU_CONFIG" "GPU_global.deviceType=$GPUTYPE;GPU_proc.debugLevel=0;$GPU_CONFIG_KEY;$TRACKTUNETPCINNER" (has_detector_reco TOF || has_detector_ctf TOF) && ! has_detector_from_global_reader TOF && add_W o2-tof-reco-workflow "$TOF_CONFIG --input-type $TOF_INPUT --output-type $TOF_OUTPUT $DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N tof-compressed-decoder TOF RAW 1),$(get_N TOFClusterer TOF REST 1)" has_detector_reco ITS && ! has_detector_from_global_reader ITS && add_W o2-its-reco-workflow "--trackerCA $ITS_CONFIG $DISABLE_MC $DISABLE_DIGIT_CLUSTER_INPUT $DISABLE_ROOT_OUTPUT --pipeline $(get_N its-tracker ITS REST 1 ITSTRK)" "$ITS_CONFIG_KEY;$ITSMFT_STROBES;$ITSEXTRAERR" has_detector_reco FT0 && ! has_detector_from_global_reader FT0 && add_W o2-ft0-reco-workflow "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N ft0-reconstructor FT0 REST 1)" From aeb4ec76a722c08ce1ed942c21ae34f96f4b9bba Mon Sep 17 00:00:00 2001 From: shahoian Date: Tue, 27 Jun 2023 17:17:06 +0200 Subject: [PATCH 031/289] Use same binding everywhere to access CTPLumi input --- Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx | 4 ++-- Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx | 4 ++-- Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx | 2 +- .../TPC/calibration/SpacePoints/src/ResidualAggregator.cxx | 4 ++-- Detectors/TPC/calibration/src/CorrectionMapsLoader.cxx | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx index 8752cbad726c5..fe330347882db 100644 --- a/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx @@ -56,7 +56,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) auto buff = pc.inputs().get>("ctf_CTP"); auto& digits = pc.outputs().make>(OutputRef{"digits"}); - auto& lumi = pc.outputs().make(OutputRef{"lumi"}); + auto& lumi = pc.outputs().make(OutputRef{"CTPLumi"}); // since the buff is const, we cannot use EncodedBlocks::relocate directly, instead we wrap its data to another flat object if (buff.size()) { @@ -78,7 +78,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) { std::vector outputs{ OutputSpec{{"digits"}, "CTP", "DIGITS", 0, Lifetime::Timeframe}, - OutputSpec{{"lumi"}, "CTP", "LUMI", 0, Lifetime::Timeframe}, + OutputSpec{{"CTPLumi"}, "CTP", "LUMI", 0, Lifetime::Timeframe}, OutputSpec{{"ctfrep"}, "CTP", "CTFDECREP", 0, Lifetime::Timeframe}}; std::vector inputs; diff --git a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx index ba091891fbd5c..0f6236b0675cc 100644 --- a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx @@ -52,7 +52,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) auto digits = pc.inputs().get>("digits"); LumiInfo lumi{}; if (!mNoLumi) { - lumi = pc.inputs().get("lumi"); + lumi = pc.inputs().get("CTPLumi"); } auto& buffer = pc.outputs().make>(Output{"CTP", "CTFDATA", 0, Lifetime::Timeframe}); auto iosize = mCTFCoder.encode(buffer, digits, lumi); @@ -72,7 +72,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR, bool nolumi) std::vector inputs; inputs.emplace_back("digits", "CTP", "DIGITS", 0, Lifetime::Timeframe); if (!nolumi) { - inputs.emplace_back("lumi", "CTP", "LUMI", 0, Lifetime::Timeframe); + inputs.emplace_back("CTPLumi", "CTP", "LUMI", 0, Lifetime::Timeframe); } inputs.emplace_back("ctfdict", "CTP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CTP/Calib/CTFDictionaryTree")); if (selIR) { diff --git a/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx b/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx index 10318698309c7..a65b94d16f43c 100644 --- a/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx +++ b/Detectors/CTP/workflowIO/src/DigitWriterSpec.cxx @@ -37,7 +37,7 @@ framework::DataProcessorSpec getDigitWriterSpec(bool raw) return MakeRootTreeWriterSpec("ctp-digit-writer-dec", "ctpdigits.root", MakeRootTreeWriterSpec::TreeAttributes{"o2sim", "Tree with CTP digits/Lumi"}, BranchDefinition>{InputSpec{"digit", "CTP", "DIGITS", 0}, "CTPDigits", logger}, - BranchDefinition{InputSpec{"lumi", "CTP", "LUMI", 0}, "CTPLumi"})(); + BranchDefinition{InputSpec{"CTPLumi", "CTP", "LUMI", 0}, "CTPLumi"})(); } // MC digits case, no lumi available return MakeRootTreeWriterSpec("ctp-digit-writer", "ctpdigits.root", diff --git a/Detectors/TPC/calibration/SpacePoints/src/ResidualAggregator.cxx b/Detectors/TPC/calibration/SpacePoints/src/ResidualAggregator.cxx index 8212dd5782a63..8d0f63ae09d52 100644 --- a/Detectors/TPC/calibration/SpacePoints/src/ResidualAggregator.cxx +++ b/Detectors/TPC/calibration/SpacePoints/src/ResidualAggregator.cxx @@ -100,7 +100,7 @@ void ResidualsContainer::init(const TrackResiduals* residualsEngine, std::string treeOutResidualsUnbinned = std::make_unique("unbinnedResid", "TPC unbinned residuals"); treeOutResidualsUnbinned->Branch("res", &unbinnedResPtr); treeOutResidualsUnbinned->Branch("trackInfo", &trackInfoPtr); - treeOutResidualsUnbinned->Branch("lumi", &lumiTF); + treeOutResidualsUnbinned->Branch("CTPLumi", &lumiTF); treeOutResidualsUnbinned->Branch("timeMS", &timeMS); } if (writeTrackData) { @@ -131,7 +131,7 @@ void ResidualsContainer::init(const TrackResiduals* residualsEngine, std::string treeOutRecords->Branch("firstTForbit", &tfOrbitsPtr); treeOutRecords->Branch("sumOfBinnedResiduals", &sumBinnedResidPtr); treeOutRecords->Branch("sumOfUnbinnedResiduals", &sumUnbinnedResidPtr); - treeOutRecords->Branch("lumi", &lumiPtr); + treeOutRecords->Branch("CTPLumi", &lumiPtr); LOG(debug) << "Done initializing residuals container for file named " << fileName; } diff --git a/Detectors/TPC/calibration/src/CorrectionMapsLoader.cxx b/Detectors/TPC/calibration/src/CorrectionMapsLoader.cxx index 855b93fddc2fd..2e557aac67dbf 100644 --- a/Detectors/TPC/calibration/src/CorrectionMapsLoader.cxx +++ b/Detectors/TPC/calibration/src/CorrectionMapsLoader.cxx @@ -54,7 +54,7 @@ void CorrectionMapsLoader::requestCCDBInputs(std::vector& inputs, std addInput(inputs, {"tpcCorrMap", "TPC", "CorrMap", 0, Lifetime::Condition, ccdbParamSpec(CDBTypeMap.at(CDBType::CalCorrMap), {}, 1)}); // time-dependent addInput(inputs, {"tpcCorrMapRef", "TPC", "CorrMapRef", 0, Lifetime::Condition, ccdbParamSpec(CDBTypeMap.at(CDBType::CalCorrMapRef), {}, 0)}); // load once if (requestCTPLumi) { - addInput(inputs, {"lumi", "CTP", "LUMI", 0, Lifetime::Timeframe}); + addInput(inputs, {"CTPLumi", "CTP", "LUMI", 0, Lifetime::Timeframe}); } addOptions(options); } From de4f9d38745f18c6c8989f3e03efd57588855c88 Mon Sep 17 00:00:00 2001 From: shahoian Date: Wed, 28 Jun 2023 09:26:33 +0200 Subject: [PATCH 032/289] Less invasive application of TPC_CORR_SCALING local/global options --- prodtests/full-system-test/dpl-workflow.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/prodtests/full-system-test/dpl-workflow.sh b/prodtests/full-system-test/dpl-workflow.sh index 568317d030daa..bc4c869fd62a3 100755 --- a/prodtests/full-system-test/dpl-workflow.sh +++ b/prodtests/full-system-test/dpl-workflow.sh @@ -235,6 +235,11 @@ if [[ $HOSTMEMSIZE != "0" ]]; then fi GPU_CONFIG_SELF="--severity $SEVERITY_TPC" +ASK_CTP_LUMI_GPU="--require-ctp-lumi" +: ${TPC_CORR_SCALING_GPU:=""} +[[ "$TPC_CORR_SCALING" == *"$ASK_CTP_LUMI_GPU"* ]] && TPC_CORR_SCALING_GPU=${TPC_CORR_SCALING//$ASK_CTP_LUMI_GPU/} || ASK_CTP_LUMI_GPU="" +GPU_CONFIG_SELF+=" $TPC_CORR_SCALING_GPU " + if [[ $GPUTYPE != "CPU" && $(ulimit -e) -ge 25 && ${O2_GPU_WORKFLOW_NICE:-} == 1 ]]; then GPU_CONFIG_SELF+=" --child-driver 'nice -n -5'" fi @@ -406,7 +411,7 @@ fi # --------------------------------------------------------------------------------------------------------------------- # Common reconstruction workflows -(has_detector_reco TPC || has_detector_ctf TPC) && ! has_detector_from_global_reader TPC && add_W o2-gpu-reco-workflow "--gpu-reconstruction \"$GPU_CONFIG_SELF\" $TPC_CORR_SCALING --input-type=$GPU_INPUT $DISABLE_MC --output-type $GPU_OUTPUT --pipeline gpu-reconstruction:${N_TPCTRK:-1} $GPU_CONFIG" "GPU_global.deviceType=$GPUTYPE;GPU_proc.debugLevel=0;$GPU_CONFIG_KEY;$TRACKTUNETPCINNER" +(has_detector_reco TPC || has_detector_ctf TPC) && ! has_detector_from_global_reader TPC && add_W o2-gpu-reco-workflow "--gpu-reconstruction \"$GPU_CONFIG_SELF\" $ASK_CTP_LUMI_GPU --input-type=$GPU_INPUT $DISABLE_MC --output-type $GPU_OUTPUT --pipeline gpu-reconstruction:${N_TPCTRK:-1} $GPU_CONFIG" "GPU_global.deviceType=$GPUTYPE;GPU_proc.debugLevel=0;$GPU_CONFIG_KEY;$TRACKTUNETPCINNER" (has_detector_reco TOF || has_detector_ctf TOF) && ! has_detector_from_global_reader TOF && add_W o2-tof-reco-workflow "$TOF_CONFIG --input-type $TOF_INPUT --output-type $TOF_OUTPUT $DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N tof-compressed-decoder TOF RAW 1),$(get_N TOFClusterer TOF REST 1)" has_detector_reco ITS && ! has_detector_from_global_reader ITS && add_W o2-its-reco-workflow "--trackerCA $ITS_CONFIG $DISABLE_MC $DISABLE_DIGIT_CLUSTER_INPUT $DISABLE_ROOT_OUTPUT --pipeline $(get_N its-tracker ITS REST 1 ITSTRK)" "$ITS_CONFIG_KEY;$ITSMFT_STROBES;$ITSEXTRAERR" has_detector_reco FT0 && ! has_detector_from_global_reader FT0 && add_W o2-ft0-reco-workflow "$DISABLE_DIGIT_ROOT_INPUT $DISABLE_ROOT_OUTPUT $DISABLE_MC --pipeline $(get_N ft0-reconstructor FT0 REST 1)" From 201ae2ec3ffa358d3504efde2976dfcf01153236 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 27 Jun 2023 20:23:27 +0200 Subject: [PATCH 033/289] TRD raw reader can optionally skip parsing every nth TF --- .../reconstruction/include/TRDReconstruction/DataReaderTask.h | 2 ++ Detectors/TRD/reconstruction/src/DataReader.cxx | 3 ++- Detectors/TRD/reconstruction/src/DataReaderTask.cxx | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Detectors/TRD/reconstruction/include/TRDReconstruction/DataReaderTask.h b/Detectors/TRD/reconstruction/include/TRDReconstruction/DataReaderTask.h index c7d72854fb152..7608dbd8992b9 100644 --- a/Detectors/TRD/reconstruction/include/TRDReconstruction/DataReaderTask.h +++ b/Detectors/TRD/reconstruction/include/TRDReconstruction/DataReaderTask.h @@ -51,6 +51,7 @@ class DataReaderTask : public Task bool mDataVerbose{false}; // verbose output of data unpacking bool mHeaderVerbose{false}; // verbose output of headers bool mCompressedData{false}; // are we dealing with the compressed data from the flp (send via option) + int mProcessEveryNthTF{1}; // to parse only every n-th TF and send empty output for the rest bool mInitOnceDone{false}; // flag for requesting new CCDB object upon global run number change std::bitset<16> mOptions; // stores the incoming of the above bools, useful to be able to send this on instead of the individual ones above // the above bools make the code more readable hence still here. @@ -64,6 +65,7 @@ class DataReaderTask : public Task uint64_t mWordsRejectedTotal{0}; // accumulate the total number of words rejected uint64_t mDigitsTotal{0}; // accumulate the total number of digits read uint64_t mTrackletsTotal{0}; // accumulate the total number os tracklets read + size_t mNTFsProcessed{0}; // keep track of the total number of TFs processed }; } // namespace o2::trd diff --git a/Detectors/TRD/reconstruction/src/DataReader.cxx b/Detectors/TRD/reconstruction/src/DataReader.cxx index 02324375a167b..11cbb8fff4a75 100644 --- a/Detectors/TRD/reconstruction/src/DataReader.cxx +++ b/Detectors/TRD/reconstruction/src/DataReader.cxx @@ -94,7 +94,8 @@ WorkflowSpec defineDataProcessing(ConfigContext const& cfgc) outputs, algoSpec, Options{{"log-max-errors", VariantType::Int, 20, {"maximum number of errors to log"}}, - {"log-max-warnings", VariantType::Int, 20, {"maximum number of warnings to log"}}}}); + {"log-max-warnings", VariantType::Int, 20, {"maximum number of warnings to log"}}, + {"every-nth-tf", VariantType::Int, 1, {"process only every n-th TF"}}}}); if (!cfgc.options().get("disable-root-output")) { workflow.emplace_back(o2::trd::getTRDDigitWriterSpec(false, false)); diff --git a/Detectors/TRD/reconstruction/src/DataReaderTask.cxx b/Detectors/TRD/reconstruction/src/DataReaderTask.cxx index ac5dfb84d7e97..7bba5d2b085d4 100644 --- a/Detectors/TRD/reconstruction/src/DataReaderTask.cxx +++ b/Detectors/TRD/reconstruction/src/DataReaderTask.cxx @@ -33,6 +33,7 @@ void DataReaderTask::init(InitContext& ic) { mReader.setMaxErrWarnPrinted(ic.options().get("log-max-errors"), ic.options().get("log-max-warnings")); mReader.configure(mTrackletHCHeaderState, mHalfChamberWords, mHalfChamberMajor, mOptions); + mProcessEveryNthTF = ic.options().get("every-nth-tf"); } void DataReaderTask::endOfStream(o2::framework::EndOfStreamContext& ec) @@ -98,7 +99,7 @@ void DataReaderTask::run(ProcessingContext& pc) updateTimeDependentParams(pc); auto dataReadStart = std::chrono::high_resolution_clock::now(); - if (isTimeFrameEmpty(pc)) { + if ((mNTFsProcessed++ % mProcessEveryNthTF != 0) || isTimeFrameEmpty(pc)) { mReader.buildDPLOutputs(pc); mReader.reset(); return; From b20e6466976b9a1466b166253fc8329f6157a85e Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Tue, 27 Jun 2023 20:23:49 +0200 Subject: [PATCH 034/289] TRD noise calibration can run as dummy without CCDB upload --- .../include/TRDWorkflow/NoiseCalibSpec.h | 17 ++++++++++++----- .../TRD/workflow/src/trd-calib-workflow.cxx | 17 ++++++++++++++++- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Detectors/TRD/workflow/include/TRDWorkflow/NoiseCalibSpec.h b/Detectors/TRD/workflow/include/TRDWorkflow/NoiseCalibSpec.h index 322aa91d9fc18..61a8e5df3afa4 100644 --- a/Detectors/TRD/workflow/include/TRDWorkflow/NoiseCalibSpec.h +++ b/Detectors/TRD/workflow/include/TRDWorkflow/NoiseCalibSpec.h @@ -33,7 +33,7 @@ namespace trd class TRDNoiseCalibSpec : public o2::framework::Task { public: - TRDNoiseCalibSpec() = default; + TRDNoiseCalibSpec(bool dummy) : mIsDummy(dummy) {} void init(o2::framework::InitContext& ic) final { // Do we need to initialize something? @@ -44,6 +44,10 @@ class TRDNoiseCalibSpec : public o2::framework::Task auto digits = pc.inputs().get>("trddigits"); auto trigRecs = pc.inputs().get>("trdtriggerrec"); + if (mIsDummy) { + return; + } + // Obtain rough time from the data header (done only once) if (mStartTime == 0) { o2::dataformats::TFIDInfo ti; @@ -63,7 +67,7 @@ class TRDNoiseCalibSpec : public o2::framework::Task sendOutput(pc.outputs()); mHaveSentOutput = true; } else { - if ((mNTFsProcessed % 50) == 0) { + if ((mNTFsProcessed % 200) == 0) { LOGP(important, "Not processing anymore. Seen {} TFs in total. Run can be stopped", mNTFsProcessed); } } @@ -96,7 +100,9 @@ class TRDNoiseCalibSpec : public o2::framework::Task if (mHaveSentOutput) { LOGP(important, "Received EoS after sending calibration object. All OK"); } else { - LOGP(alarm, "Received EoS before sending calibration object. Not enough digits received"); + if (!mIsDummy) { + LOGP(alarm, "Received EoS before sending calibration object. Not enough digits received"); + } } } @@ -104,10 +110,11 @@ class TRDNoiseCalibSpec : public o2::framework::Task CalibratorNoise mCalibrator{}; size_t mNTFsProcessed{0}; bool mHaveSentOutput{false}; + bool mIsDummy{false}; uint64_t mStartTime{0}; }; -o2::framework::DataProcessorSpec getTRDNoiseCalibSpec() +o2::framework::DataProcessorSpec getTRDNoiseCalibSpec(bool dummy) { std::vector inputs; inputs.emplace_back("trddigits", o2::header::gDataOriginTRD, "DIGITS", 0, Lifetime::Timeframe); @@ -121,7 +128,7 @@ o2::framework::DataProcessorSpec getTRDNoiseCalibSpec() "trd-noise-calib", inputs, outputs, - AlgorithmSpec{adaptFromTask()}, + AlgorithmSpec{adaptFromTask(dummy)}, Options{}}; } diff --git a/Detectors/TRD/workflow/src/trd-calib-workflow.cxx b/Detectors/TRD/workflow/src/trd-calib-workflow.cxx index 05ca517e5b953..f3eb520873dbd 100644 --- a/Detectors/TRD/workflow/src/trd-calib-workflow.cxx +++ b/Detectors/TRD/workflow/src/trd-calib-workflow.cxx @@ -28,6 +28,7 @@ void customize(std::vector& workflowOptions) {"vDriftAndExB", o2::framework::VariantType::Bool, false, {"enable vDrift and ExB calibration"}}, {"noise", o2::framework::VariantType::Bool, false, {"enable noise and pad status calibration"}}, {"gain", o2::framework::VariantType::Bool, false, {"enable gain calibration"}}, + {"calib-dds-collection-index", VariantType::Int, -1, {"allow only single collection to produce calibration objects (use -1 for no limit)"}}, {"configKeyValues", VariantType::String, "", {"Semicolon separated key=value strings"}}}; std::swap(workflowOptions, options); @@ -55,7 +56,21 @@ WorkflowSpec defineDataProcessing(ConfigContext const& configcontext) if (enableRootInp) { specs.emplace_back(o2::trd::getTRDDigitReaderSpec(false)); } - specs.emplace_back(o2::trd::getTRDNoiseCalibSpec()); + int ddsCollectionIdx = configcontext.options().get("calib-dds-collection-index"); + bool noiseCalibIsDummy = true; + if (ddsCollectionIdx != -1) { + char* colIdx = getenv("DDS_COLLECTION_INDEX"); + int myIdx = colIdx ? atoi(colIdx) : -1; + if (myIdx == ddsCollectionIdx) { + LOG(info) << "TRD noise calib is enabled for this collection, my index " << myIdx; + noiseCalibIsDummy = false; + } else { + LOG(info) << "TRD noise calib is disabled for this collection, my index " << myIdx; + } + } else { + noiseCalibIsDummy = false; + } + specs.emplace_back(o2::trd::getTRDNoiseCalibSpec(noiseCalibIsDummy)); } if (configcontext.options().get("gain")) { From 8f49a6a37d5cc8517ea369b6f116ca600283b603 Mon Sep 17 00:00:00 2001 From: cortesep <57937610+cortesep@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:47:10 +0200 Subject: [PATCH 035/289] ZDC Fix TDC calibration (#11568) * Fix TDC calibration * Please consider the following formatting changes (#73) --------- Co-authored-by: ALICE Builder --- .../ZDC/calib/include/ZDCCalib/TDCCalibEPN.h | 2 +- .../ZDC/calib/include/ZDCCalib/TDCCalibSpec.h | 8 +++++--- Detectors/ZDC/calib/src/TDCCalib.cxx | 6 +++--- Detectors/ZDC/calib/src/TDCCalibData.cxx | 4 ++-- Detectors/ZDC/calib/src/TDCCalibEPN.cxx | 10 +++++----- Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx | 1 + Detectors/ZDC/calib/src/TDCCalibSpec.cxx | 18 +++++++++--------- 7 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibEPN.h b/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibEPN.h index 7bf13a37926c5..49535fc4630be 100644 --- a/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibEPN.h +++ b/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibEPN.h @@ -48,7 +48,7 @@ class TDCCalibEPN static constexpr int HtdcZPCC = 8; static constexpr int HtdcZPCS = 9; static constexpr int NTDC = TDCCalibData::NTDC; - void clear(int ih = -1); + void clear(); int process(const gsl::span& bcrec, const gsl::span& energy, const gsl::span& tdc, diff --git a/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibSpec.h b/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibSpec.h index d84b381322b33..58df5ac2e974e 100644 --- a/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibSpec.h +++ b/Detectors/ZDC/calib/include/ZDCCalib/TDCCalibSpec.h @@ -55,9 +55,11 @@ class TDCCalibSpec : public o2::framework::Task bool mInitialized = false; // Connect once to CCDB during initialization TDCCalib mWorker; // TDC calibration object TStopwatch mTimer; - long mRunStartTime = 0; /// start time of the run (ms) - std::string mOutputDir; /// where to write calibration digits - std::string mHistoFileName; /// file name of output calib digits + long mRunStartTime = 0; /// start time of the run (ms) + o2::framework::DataAllocator* mOutput = nullptr; /// Pointer to output object + std::unique_ptr mHistoFileMetaData = nullptr; /// Pointer to metadata file + std::string mOutputDir; /// where to write calibration digits + std::string mHistoFileName; /// file name of output calib digits std::string mLHCPeriod; int mRunNumber = -1; }; diff --git a/Detectors/ZDC/calib/src/TDCCalib.cxx b/Detectors/ZDC/calib/src/TDCCalib.cxx index 27cb016ba6d6c..0522e1c863402 100644 --- a/Detectors/ZDC/calib/src/TDCCalib.cxx +++ b/Detectors/ZDC/calib/src/TDCCalib.cxx @@ -62,7 +62,7 @@ int TDCCalib::endOfRun() LOGF(info, "Computing TDC Calibration coefficients"); } for (int ih = 0; ih < TDCCalibData::NTDC; ih++) { - LOGF(info, "%s %i events and cuts (%g:%g)", TDCCalibData::CTDC[ih], mData.entries[ih], mTDCCalibConfig->cutLow[ih], mTDCCalibConfig->cutHigh[ih]); + LOGF(info, "%s %d events and cuts (%g:%g)", TDCCalibData::CTDC[ih], mData.entries[ih], mTDCCalibConfig->cutLow[ih], mTDCCalibConfig->cutHigh[ih]); if (!mTDCCalibConfig->enabled[ih]) { LOGF(info, "DISABLED processing of RUN3 data for ih = %d: %s", ih, TDCCalibData::CTDC[ih]); @@ -73,7 +73,7 @@ int TDCCalib::endOfRun() LOGF(info, "Processed RUN3 data for ih = %d: %s", ih, TDCCalibData::CTDC[ih]); assign(ih, true); } else { - LOGF(info, "FAILED processing RUN3 data for ih = %d: %s: TOO FEW EVENTS: %d", ih, TDCCalibData::CTDC[ih], 5); // instead of 5 put number of events + LOGF(info, "FAILED processing RUN3 data for ih = %d: %s: TOO FEW EVENTS: %d", ih, TDCCalibData::CTDC[ih], mData.entries[ih]); assign(ih, false); } } @@ -183,7 +183,7 @@ double TDCCalib::extractShift(int ih) auto h1 = mCTDC[ih]->createTH1F(TDCCalibData::CTDC[ih]); // createTH1F(histo_name) //h1->Draw("HISTO"); int nEntries = h1->GetEntries(); - std::cout << nEntries << std::endl; + // std::cout << nEntries << std::endl; if ((ih >= 0 && ih <= 9) && (nEntries >= mTDCCalibConfig->min_e[ih])) { //TDC number is ok and more than minimum entries double avgShift = h1->GetMean(); return avgShift; diff --git a/Detectors/ZDC/calib/src/TDCCalibData.cxx b/Detectors/ZDC/calib/src/TDCCalibData.cxx index 0fe929a95b129..047583c5b15f4 100644 --- a/Detectors/ZDC/calib/src/TDCCalibData.cxx +++ b/Detectors/ZDC/calib/src/TDCCalibData.cxx @@ -17,7 +17,7 @@ using namespace o2::zdc; void TDCCalibData::print() const { for (int i = 0; i < NTDC; i++) { - LOGF(info, "%s", CTDC[i]); + LOGF(info, "%s entries: %d", CTDC[i], entries[i]); } } @@ -25,7 +25,7 @@ TDCCalibData& TDCCalibData::operator+=(const TDCCalibData& other) { for (int32_t ih = 0; ih < NTDC; ih++) { - entries[ih] += other.entries[ih]; + entries[ih] = entries[ih] + other.entries[ih]; } if (mCTimeBeg == 0 || other.mCTimeBeg < mCTimeBeg) { diff --git a/Detectors/ZDC/calib/src/TDCCalibEPN.cxx b/Detectors/ZDC/calib/src/TDCCalibEPN.cxx index 91e7096150ca2..cad4d2acadd9e 100644 --- a/Detectors/ZDC/calib/src/TDCCalibEPN.cxx +++ b/Detectors/ZDC/calib/src/TDCCalibEPN.cxx @@ -121,16 +121,16 @@ int TDCCalibEPN::endOfRun() //----// -void TDCCalibEPN::clear(int ih) +void TDCCalibEPN::clear() { - int ihstart = 0; - int ihstop = NTDC; - - for (int32_t ii = ihstart; ii < ihstop; ii++) { + for (int32_t ii = 0; ii < NTDC; ii++) { if (mTDC[ii]) { mTDC[ii]->clear(); } + mData.entries[ii] = 0; } + mData.mCTimeBeg = 0; + mData.mCTimeEnd = 0; } //----// diff --git a/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx b/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx index bd2be23a4bd81..0d987633b8eb7 100644 --- a/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx +++ b/Detectors/ZDC/calib/src/TDCCalibEPNSpec.cxx @@ -109,6 +109,7 @@ void TDCCalibEPNSpec::run(ProcessingContext& pc) o2::framework::Output output("ZDC", "TDC_1DH", ih, Lifetime::Timeframe); pc.outputs().snapshot(output, mWorker.mTDC[ih]->getBase()); } + mWorker.clear(); } void TDCCalibEPNSpec::endOfStream(EndOfStreamContext& ec) diff --git a/Detectors/ZDC/calib/src/TDCCalibSpec.cxx b/Detectors/ZDC/calib/src/TDCCalibSpec.cxx index e21b28c1300ae..e89b8c4161f1c 100644 --- a/Detectors/ZDC/calib/src/TDCCalibSpec.cxx +++ b/Detectors/ZDC/calib/src/TDCCalibSpec.cxx @@ -102,6 +102,9 @@ void TDCCalibSpec::run(ProcessingContext& pc) if (!mInitialized) { mInitialized = true; updateTimeDependentParams(pc); + mOutput = &(pc.outputs()); + mHistoFileMetaData = std::make_unique(); + mHistoFileMetaData->setDataTakingContext(pc.services().get()); mTimer.Stop(); mTimer.Reset(); mTimer.Start(false); @@ -149,8 +152,8 @@ void TDCCalibSpec::sendOutput(o2::framework::EndOfStreamContext& ec) if (mVerbosity > DbgMinimal) { payload.print(); } - output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "ZDC_TDCcalib", 0}, *image.get()); // vector - output.snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "ZDC_TDCcalib", 0}, info); // root-serlized + mOutput->snapshot(Output{o2::calibration::Utils::gDataOriginCDBPayload, "ZDC_TDCcalib", 0}, *image.get()); // vector + mOutput->snapshot(Output{o2::calibration::Utils::gDataOriginCDBWrapper, "ZDC_TDCcalib", 0}, info); // root-serlized // TODO: reset the outputs once they are already sent (is it necessary?) // mWorker.init(); @@ -165,17 +168,14 @@ void TDCCalibSpec::sendOutput(o2::framework::EndOfStreamContext& ec) } std::string metaFileDir = opt.metaFileDir; if (metaFileDir.compare("/dev/null")) { - std::unique_ptr histoFileMetaData; - histoFileMetaData = std::make_unique(); - histoFileMetaData->setDataTakingContext(ec.services().get()); - histoFileMetaData->fillFileData(mHistoFileName); - histoFileMetaData->type = "calib"; - histoFileMetaData->priority = "high"; + mHistoFileMetaData->fillFileData(mHistoFileName); + mHistoFileMetaData->type = "calib"; + mHistoFileMetaData->priority = "high"; std::string metaFileNameTmp = metaFileDir + (metaFileDir.back() == '/' ? "" : "/") + fmt::format("{}_{}.tmp", fn, mRunNumber); std::string metaFileName = metaFileDir + (metaFileDir.back() == '/' ? "" : "/") + fmt::format("{}_{}.done", fn, mRunNumber); try { std::ofstream metaFileOut(metaFileNameTmp); - metaFileOut << *histoFileMetaData.get(); + metaFileOut << *mHistoFileMetaData.get(); metaFileOut.close(); std::filesystem::rename(metaFileNameTmp, metaFileName); } catch (std::exception const& e) { From ac2d355458ed4128b387427e48ec75b7d84d1ca2 Mon Sep 17 00:00:00 2001 From: shahoian Date: Sat, 1 Jul 2023 19:44:32 +0200 Subject: [PATCH 036/289] Fix CorrMapLoader isUpdated... methods, improve reporting --- .../CorrectionMapsHelper.h | 22 +++++++++++-------- GPU/Workflow/src/GPUWorkflowSpec.cxx | 5 +++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/GPU/TPCFastTransformation/CorrectionMapsHelper.h b/GPU/TPCFastTransformation/CorrectionMapsHelper.h index a4bf4377cee44..52b1e9056a308 100644 --- a/GPU/TPCFastTransformation/CorrectionMapsHelper.h +++ b/GPU/TPCFastTransformation/CorrectionMapsHelper.h @@ -64,23 +64,23 @@ class CorrectionMapsHelper void setCorrMap(GPUCA_NAMESPACE::gpu::TPCFastTransform* m); void setCorrMapRef(GPUCA_NAMESPACE::gpu::TPCFastTransform* m); void reportScaling(); - void setInstLumi(float v) + void setInstLumi(float v, bool report = true) { if (v != mInstLumi) { mInstLumi = v; - updateLumiScale(); + updateLumiScale(report); } } - void setMeanLumi(float v) + void setMeanLumi(float v, bool report = true) { if (v != mMeanLumi) { mMeanLumi = v; - updateLumiScale(); + updateLumiScale(report); } } - void updateLumiScale() + void updateLumiScale(bool report = true) { if (mMeanLumi < 0.f || mInstLumi < 0.f) { mLumiScale = -1.f; @@ -88,7 +88,9 @@ class CorrectionMapsHelper mLumiScale = mMeanLumi ? mInstLumi / mMeanLumi : 0.f; } setUpdatedLumi(); - reportScaling(); + if (report) { + reportScaling(); + } } GPUd() float getInstLumi() const { return mInstLumi; } @@ -96,9 +98,9 @@ class CorrectionMapsHelper GPUd() float getLumiScale() const { return mLumiScale; } bool isUpdated() const { return mUpdatedFlags != 0; } - bool isUpdatedMap() const { return (mUpdatedFlags & UpdateFlags::MapBit) == 0; } - bool isUpdatedMapRef() const { return (mUpdatedFlags & UpdateFlags::MapRefBit) == 0; } - bool isUpdatedLumi() const { return (mUpdatedFlags & UpdateFlags::LumiBit) == 0; } + bool isUpdatedMap() const { return (mUpdatedFlags & UpdateFlags::MapBit) != 0; } + bool isUpdatedMapRef() const { return (mUpdatedFlags & UpdateFlags::MapRefBit) != 0; } + bool isUpdatedLumi() const { return (mUpdatedFlags & UpdateFlags::LumiBit) != 0; } void setUpdatedMap() { mUpdatedFlags |= UpdateFlags::MapBit; } void setUpdatedMapRef() { mUpdatedFlags |= UpdateFlags::MapRefBit; } void setUpdatedLumi() { mUpdatedFlags |= UpdateFlags::LumiBit; } @@ -119,6 +121,8 @@ class CorrectionMapsHelper void setInstLumiOverride(float f) { mInstLumiOverride = f; } float getInstLumiOverride() const { return mInstLumiOverride; } + int getUpdateFlags() const { return mUpdatedFlags; } + protected: enum UpdateFlags { MapBit = 0x1, MapRefBit = 0x2, diff --git a/GPU/Workflow/src/GPUWorkflowSpec.cxx b/GPU/Workflow/src/GPUWorkflowSpec.cxx index e9beda3fe9fc9..044e48d3481a7 100644 --- a/GPU/Workflow/src/GPUWorkflowSpec.cxx +++ b/GPU/Workflow/src/GPUWorkflowSpec.cxx @@ -1606,13 +1606,14 @@ bool GPURecoWorkflowSpec::fetchCalibsCCDBTPC(ProcessingContext& pc, T& newCalibO } if (mFastTransformNew || mFastTransformRefNew || mFastTransformHelper->isUpdatedLumi()) { mFastTransformHelperNew.reset(new o2::tpc::CorrectionMapsLoader); - mFastTransformHelperNew->setInstLumi(mFastTransformHelper->getInstLumi()); - mFastTransformHelperNew->setMeanLumi(mFastTransformHelper->getMeanLumi()); + mFastTransformHelperNew->setInstLumi(mFastTransformHelper->getInstLumi(), false); + mFastTransformHelperNew->setMeanLumi(mFastTransformHelper->getMeanLumi(), false); mFastTransformHelperNew->setUseCTPLumi(mFastTransformHelper->getUseCTPLumi()); mFastTransformHelperNew->setMeanLumiOverride(mFastTransformHelper->getMeanLumiOverride()); mFastTransformHelperNew->setInstLumiOverride(mFastTransformHelper->getInstLumiOverride()); mFastTransformHelperNew->setCorrMap(mFastTransformNew ? mFastTransformNew.get() : mFastTransform.get()); mFastTransformHelperNew->setCorrMapRef(mFastTransformRefNew ? mFastTransformRefNew.get() : mFastTransformRef.get()); + mFastTransformHelperNew->acknowledgeUpdate(); newCalibObjects.fastTransformHelper = mFastTransformHelperNew.get(); } mMustUpdateFastTransform = true; From 65c2f211048e81b886d1a71d9a66112aee6ef342 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Fri, 11 Aug 2023 15:26:13 +0200 Subject: [PATCH 037/289] TRD raw reader update QC statistics --- DataFormats/Detectors/TRD/CMakeLists.txt | 3 +- .../TRD/include/DataFormatsTRD/RawDataStats.h | 53 +++++++------------ .../Detectors/TRD/src/DataFormatsTRDLinkDef.h | 1 + .../include/TRDReconstruction/EventRecord.h | 37 +++++++------ .../TRD/reconstruction/src/CruRawReader.cxx | 26 +++------ .../TRD/reconstruction/src/EventRecord.cxx | 31 ++++------- 6 files changed, 57 insertions(+), 94 deletions(-) diff --git a/DataFormats/Detectors/TRD/CMakeLists.txt b/DataFormats/Detectors/TRD/CMakeLists.txt index 4689fa6a40fc9..35712caf5bfb9 100644 --- a/DataFormats/Detectors/TRD/CMakeLists.txt +++ b/DataFormats/Detectors/TRD/CMakeLists.txt @@ -45,7 +45,8 @@ o2_target_root_dictionary(DataFormatsTRD include/DataFormatsTRD/KrCluster.h include/DataFormatsTRD/KrClusterTriggerRecord.h include/DataFormatsTRD/NoiseCalibration.h - include/DataFormatsTRD/PHData.h + include/DataFormatsTRD/PHData.h + include/DataFormatsTRD/RawDataStats.h include/DataFormatsTRD/CTF.h include/DataFormatsTRD/CalVdriftExB.h include/DataFormatsTRD/CalGain.h diff --git a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h index d91a38ad87451..3a06116ef5fd9 100644 --- a/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h +++ b/DataFormats/Detectors/TRD/include/DataFormatsTRD/RawDataStats.h @@ -16,13 +16,11 @@ #ifndef O2_TRD_RAWDATASTATS #define O2_TRD_RAWDATASTATS -#include "TObject.h" +#include "Rtypes.h" #include #include #include -#include #include -#include #include "DataFormatsTRD/Constants.h" namespace o2::trd @@ -123,59 +121,44 @@ enum OptionBits { TRDSortDigits }; // this is currently 16 options, the array is 16, if you add here you need to change the 16; -//Data to be stored and accumulated on an event basis. -//events are spread out with in the data coming in with a halfcruheader per event, per ... half cru. -//this is looked up via the interaction record (orbit and bunchcrossing). -//this permits averaging in the data that gets senton per timeframe -struct TRDDataCountersPerEvent { - TRDDataCountersPerEvent() = default; - double mTimeTaken = 0.; // time take to process an event (summed trackletparsing and digitparsing) parts not accounted for. - double mTimeTakenForDigits = 0.; // time take to process tracklet data blocks [us]. - double mTimeTakenForTracklets = 0.; // time take to process digit data blocks [us]. - uint64_t mWordsRead = 0; // words read in - uint64_t mWordsRejected = 0; // words skipped for various reasons. - uint16_t mTrackletsFound = 0; // tracklets found in the event - uint16_t mDigitsFound = 0; // digits found in the event -}; - //Data to be stored on a timeframe basis to then be sent as a message to be ultimately picked up by qc. //Some countes include a average over the numbers stored on a per event basis, e.g. digits per event. class TRDDataCountersPerTimeFrame { public: - std::array mLinkErrorFlag{}; // status of the error flags for this timeframe, 8bit values from cru halfchamber header. - std::array mLinkNoData; // Link had no data or was not present. - std::array mLinkWords{}; // units of 256bits, read from the cru half chamber header - std::array mLinkWordsRead{}; // units of 32 bits the data words read before dumping or finishing - std::array mLinkWordsRejected{}; // units of 32 bits the data dumped due to some or other error + std::array mLinkErrorFlag{}; // status of the error flags for this timeframe, 8bit values from cru halfchamber header. + std::array mLinkNoData; // Link had no data or was not present. + std::array mLinkWords{}; // units of 256bits, read from the cru half chamber header + std::array mLinkWordsRead{}; // units of 32 bits the data words read before dumping or finishing + std::array mLinkWordsRejected{}; // units of 32 bits the data dumped due to some or other error + std::array mParsingOK{}; // count how often given link could be parsed without any errors std::array mParsingErrors{}; // errors in parsing, indexed by enum above of ParsingErrors - std::array mParsingErrorsByLink{}; // errors in parsing, indexed by enum above of ParsingErrors - uint16_t mDigitsPerEvent; // average digits found per event in this timeframe, ignoring the no digit events where there is no calibration trigger. - uint16_t mTrackletsPerEvent; // average tracklets found per event in this timeframe - double mTimeTaken; // time taken to process the entire timeframe [ms]. - double mTimeTakenForDigits; // time take to process tracklet data blocks [ms]. - double mTimeTakenForTracklets; // time take to process digit data blocks [ms]. + std::vector mParsingErrorsByLink{}; // each entry is for a single parsing error on a given link (HCID * number of Errors + error index) + float mTimeTaken; // time taken to process all half-CRU data blocks combined [us]. + float mTimeTakenForDigits; // time take to process tracklet data blocks [us]. + float mTimeTakenForTracklets; // time take to process digit data blocks [us]. uint32_t mDigitsFound; // digits found in the time frame. uint32_t mTrackletsFound; // tracklets found in the time frame. - std::array mDataFormatRead{}; // We just keep the major version number + uint16_t mNTriggersCalib; // number of triggers with digit readout + uint16_t mNTriggersTotal; // total number of triggers + std::array mDataFormatRead{}; // We just keep the major version number void clear() { mLinkNoData.fill(0); mLinkWords.fill(0); mLinkWordsRead.fill(0); mLinkWordsRejected.fill(0); + mParsingOK.fill(0); mParsingErrors.fill(0); - mParsingErrorsByLink.fill(0); - mDigitsPerEvent = 0; - mTrackletsPerEvent = 0; + mParsingErrorsByLink.clear(); mTimeTaken = 0; mTimeTakenForDigits = 0; mTimeTakenForTracklets = 0; mDigitsFound = 0; - mTrackletsFound = 0; //tracklets found in timeframe. + mTrackletsFound = 0; mDataFormatRead.fill(0); }; - ClassDefNV(TRDDataCountersPerTimeFrame, 1); // primarily for serialisation so we can send this as a message in o2 + ClassDefNV(TRDDataCountersPerTimeFrame, 2); // primarily for serialisation so we can send this as a message in o2 }; } // namespace o2::trd diff --git a/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h b/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h index 3c3b93e58ce40..35ce803924485 100644 --- a/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h +++ b/DataFormats/Detectors/TRD/src/DataFormatsTRDLinkDef.h @@ -43,6 +43,7 @@ #pragma link C++ class o2::trd::ChannelInfo + ; #pragma link C++ class o2::trd::ChannelInfoContainer + ; #pragma link C++ struct o2::trd::PHData + ; +#pragma link C++ class o2::trd::TRDDataCountersPerTimeFrame + ; #pragma link C++ class std::vector < o2::trd::Tracklet64> + ; #pragma link C++ class std::vector < o2::trd::CalibratedTracklet> + ; #pragma link C++ class std::vector < o2::trd::TrackTriggerRecord> + ; diff --git a/Detectors/TRD/reconstruction/include/TRDReconstruction/EventRecord.h b/Detectors/TRD/reconstruction/include/TRDReconstruction/EventRecord.h index b33966d78309c..0b1549f675cce 100644 --- a/Detectors/TRD/reconstruction/include/TRDReconstruction/EventRecord.h +++ b/Detectors/TRD/reconstruction/include/TRDReconstruction/EventRecord.h @@ -46,6 +46,10 @@ class EventRecord const std::vector& getDigits() const { return mDigits; } const std::vector& getTracklets() const { return mTracklets; } + bool getIsCalibTrigger() const { return mIsCalibTrigger; } + float getTotalTime() const { return mTimeTaken; } + float getDigitTime() const { return mTimeTakenForDigits; } + float getTrackletTime() const { return mTimeTakenForTracklets; } // needed, in order to check if a trigger already exist for this bunch crossing bool operator==(const EventRecord& o) const { return mBCData == o.mBCData; } @@ -53,24 +57,19 @@ class EventRecord // sort the tracklets (and optionally digits) by detector, pad row, pad column void sortData(bool sortDigits); - //statistics stuff these get passed to the per tf data at the end of the timeframe, - //but as we read in per link, events are seperated hence these counters - const TRDDataCountersPerEvent& getEventStats() const { return mEventStats; } - void incTrackletTime(double timeadd) { mEventStats.mTimeTakenForTracklets += timeadd; } - void incDigitTime(double timeadd) { mEventStats.mTimeTakenForDigits += timeadd; } - void incTime(double duration) { mEventStats.mTimeTaken += duration; } - void incWordsRead(int count) { mEventStats.mWordsRead += count; } // words read in - void incWordsRejected(int count) { mEventStats.mWordsRejected += count; } // words read in - void incTrackletsFound(int count) { mEventStats.mTrackletsFound += count; } - void incDigitsFound(int count) { mEventStats.mDigitsFound += count; } - // OS: Do we need to keep event statistics at all? Are they not anyhow accumulated for a whole TF? - // or are they also used on a per event basis somewhere? + void incTrackletTime(float timeadd) { mTimeTakenForTracklets += timeadd; } + void incDigitTime(float timeadd) { mTimeTakenForDigits += timeadd; } + void incTime(float duration) { mTimeTaken += duration; } + void setIsCalibTrigger() { mIsCalibTrigger = true; } private: BCData mBCData; /// orbit and Bunch crossing data of the physics trigger std::vector mDigits{}; /// digit data, for this event std::vector mTracklets{}; /// tracklet data, for this event - TRDDataCountersPerEvent mEventStats{}; /// statistics, for this trigger + float mTimeTaken = 0.; // total parsing time [us] (including digit and tracklet parsing time) + float mTimeTakenForDigits = 0.; // time take to process tracklet data blocks [us]. + float mTimeTakenForTracklets = 0.; // time take to process digit data blocks [us]. + bool mIsCalibTrigger = false; // flag calibration trigger }; /// \class EventRecordContainer @@ -87,11 +86,7 @@ class EventRecordContainer void setCurrentEventRecord(const InteractionRecord& ir); EventRecord& getCurrentEventRecord() { return mEventRecords.at(mCurrEventRecord); } - //statistics to keep - void incTrackletTime(double timeadd) { mTFStats.mTimeTakenForTracklets += timeadd; } - void incDigitTime(double timeadd) { mTFStats.mTimeTakenForDigits += timeadd; } - void incTrackletsFound(int count) { mTFStats.mTrackletsFound += count; } - void incDigitsFound(int count) { mTFStats.mDigitsFound += count; } + // statistics to keep void incLinkErrorFlags(int hcid, unsigned int flag) { mTFStats.mLinkErrorFlag[hcid] |= flag; } void incLinkNoData(int hcid) { mTFStats.mLinkNoData[hcid]++; } void incLinkWords(int hcid, int count) { mTFStats.mLinkWords[hcid] += count; } @@ -103,7 +98,11 @@ class EventRecordContainer { mTFStats.mParsingErrors[error]++; if (hcid >= 0) { // hcid==-1 is reserved for those errors where we don't have the corresponding link ID - mTFStats.mParsingErrorsByLink[hcid * TRDLastParsingError + error]++; + if (error == NoError) { + mTFStats.mParsingOK[hcid]++; + } else { + mTFStats.mParsingErrorsByLink.push_back(hcid * TRDLastParsingError + error); + } } } void reset(); diff --git a/Detectors/TRD/reconstruction/src/CruRawReader.cxx b/Detectors/TRD/reconstruction/src/CruRawReader.cxx index 26647e79a1a30..5a97ad281d9f0 100644 --- a/Detectors/TRD/reconstruction/src/CruRawReader.cxx +++ b/Detectors/TRD/reconstruction/src/CruRawReader.cxx @@ -480,6 +480,9 @@ bool CruRawReader::processHalfCRU(int iteration) mIR.bc -= o2::ctp::TriggerOffsetsParam::Instance().LM_L0; } mEventRecords.setCurrentEventRecord(mIR); + if (mCurrentHalfCRUHeader.EventType == ETYPECALIBRATIONTRIGGER) { + mEventRecords.getCurrentEventRecord().setIsCalibTrigger(); + } //loop over links uint32_t linksizeAccum32 = 0; // accumulated size of all links in 32-bit words @@ -516,7 +519,7 @@ bool CruRawReader::processHalfCRU(int iteration) } int trackletWordsRejected = 0; int trackletWordsRead = parseTrackletLinkData(currentlinksize32, halfChamberId, trackletWordsRejected); - std::chrono::duration trackletparsingtime = std::chrono::high_resolution_clock::now() - trackletparsingstart; + std::chrono::duration trackletparsingtime = std::chrono::high_resolution_clock::now() - trackletparsingstart; if (trackletWordsRead == -1) { // something went wrong bailout of here. mHBFoffset32 = hbfOffsetTmp + linksizeAccum32; @@ -529,24 +532,19 @@ bool CruRawReader::processHalfCRU(int iteration) mHBFoffset32 += trackletWordsRead; if (mCurrentHalfCRUHeader.EventType == ETYPEPHYSICSTRIGGER && endOfCurrentLink - mHBFoffset32 >= 8) { - /* - // disabled for the same reason as for the warning after the digits parsing below if (mMaxWarnPrinted > 0) { LOGF(warn, "After successfully parsing the tracklet data for link %i there are %u words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32); checkNoWarn(); } - */ incrementErrors(UnparsedTrackletDataRemaining, halfChamberId, fmt::format("On link {} there are {} words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32)); linkOK = false; } - mEventRecords.getCurrentEventRecord().incTrackletTime((double)std::chrono::duration_cast(trackletparsingtime).count()); + mEventRecords.getCurrentEventRecord().incTrackletTime(trackletparsingtime.count()); if (mOptions[TRDVerboseBit]) { LOGF(info, "Read %i tracklet words and rejected %i words", trackletWordsRead, trackletWordsRejected); } mTrackletWordsRejected += trackletWordsRejected; mTrackletWordsRead += trackletWordsRead; - mEventRecords.getCurrentEventRecord().incWordsRead(trackletWordsRead); - mEventRecords.getCurrentEventRecord().incWordsRejected(trackletWordsRejected); mEventRecords.incLinkWordsRead(halfChamberId, trackletWordsRead); mEventRecords.incLinkWordsRejected(halfChamberId, trackletWordsRejected); @@ -587,7 +585,7 @@ bool CruRawReader::processHalfCRU(int iteration) auto digitsparsingstart = std::chrono::high_resolution_clock::now(); int digitWordsRejected = 0; int digitWordsRead = parseDigitLinkData(endOfCurrentLink - mHBFoffset32, halfChamberId, digitWordsRejected); - std::chrono::duration digitsparsingtime = std::chrono::high_resolution_clock::now() - digitsparsingstart; + std::chrono::duration digitsparsingtime = std::chrono::high_resolution_clock::now() - digitsparsingstart; if (digitWordsRead == -1) { // something went wrong bailout of here. mHBFoffset32 = hbfOffsetTmp + linksizeAccum32; @@ -597,23 +595,17 @@ bool CruRawReader::processHalfCRU(int iteration) if (endOfCurrentLink - mHBFoffset32 >= 8) { // check if some data is lost (probably due to bug in CRU user logic) // we should have max 7 padding words to align the link to 256 bits - /* - // due to the current CRU bug this is almost always the case - // TODO enable warning again when CRU UL is fixed if (mMaxWarnPrinted > 0) { LOGF(warn, "After successfully parsing the digit data for link %i there are %u words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32); checkNoWarn(); } - */ incrementErrors(UnparsedDigitDataRemaining, halfChamberId, fmt::format("On link {} there are {} words remaining which did not get parsed", currentlinkindex, endOfCurrentLink - mHBFoffset32)); linkOK = false; } if (digitWordsRejected > 0) { linkOK = false; } - mEventRecords.getCurrentEventRecord().incDigitTime((double)std::chrono::duration_cast(digitsparsingtime).count()); - mEventRecords.getCurrentEventRecord().incWordsRead(digitWordsRead); - mEventRecords.getCurrentEventRecord().incWordsRejected(digitWordsRejected); + mEventRecords.getCurrentEventRecord().incDigitTime(digitsparsingtime.count()); mEventRecords.incLinkWordsRead(halfChamberId, digitWordsRead); mEventRecords.incLinkWordsRejected(halfChamberId, digitWordsRejected); @@ -648,7 +640,7 @@ bool CruRawReader::processHalfCRU(int iteration) //extract the vectors and copy them to tracklets and digits here, building the indexing(triggerrecords) //as this is for a single cru half chamber header all the tracklets and digits are for the same trigger defined by the bc and orbit in the rdh which we hold in mIR - std::chrono::duration cruparsingtime = std::chrono::high_resolution_clock::now() - crustart; + std::chrono::duration cruparsingtime = std::chrono::high_resolution_clock::now() - crustart; mEventRecords.getCurrentEventRecord().incTime(cruparsingtime.count()); //if we get here all is ok. @@ -801,7 +793,6 @@ int CruRawReader::parseDigitLinkData(int maxWords32, int hcid, int& wordsRejecte break; } mEventRecords.getCurrentEventRecord().addDigit(Digit(hcid / 2, (int)mcmHeader.rob, (int)mcmHeader.mcm, iChannel, adcValues, mPreTriggerPhase)); - mEventRecords.getCurrentEventRecord().incDigitsFound(1); ++mDigitsFound; } // end active channel } // end channel loop @@ -939,7 +930,6 @@ int CruRawReader::parseTrackletLinkData(int linkSize32, int& hcid, int& wordsRej TrackletMCMData mcmData; mcmData.word = currWord; mEventRecords.getCurrentEventRecord().addTracklet(assembleTracklet64(hcHeader.format, mcmHeader, mcmData, iCpu, hcid)); - mEventRecords.getCurrentEventRecord().incTrackletsFound(1); ++numberOfTrackletsFound; ++mTrackletsFound; addedTracklet = true; diff --git a/Detectors/TRD/reconstruction/src/EventRecord.cxx b/Detectors/TRD/reconstruction/src/EventRecord.cxx index 9312890ef95e5..6fd9ab99d8f7c 100644 --- a/Detectors/TRD/reconstruction/src/EventRecord.cxx +++ b/Detectors/TRD/reconstruction/src/EventRecord.cxx @@ -88,27 +88,16 @@ void EventRecordContainer::sendData(o2::framework::ProcessingContext& pc, bool g void EventRecordContainer::accumulateStats() { - int eventcount = mEventRecords.size(); - int sumtracklets = 0; - int sumdigits = 0; - double sumdigittime = 0; - double sumtracklettime = 0; - double sumtime = 0; - uint64_t sumwordsrejected = 0; - uint64_t sumwordsread = 0; - for (auto event : mEventRecords) { - sumtracklets += event.getEventStats().mTrackletsFound; - sumdigits += event.getEventStats().mDigitsFound; - sumtracklettime += event.getEventStats().mTimeTakenForTracklets; - sumdigittime += event.getEventStats().mTimeTakenForDigits; - sumtime += event.getEventStats().mTimeTaken; - } - if (eventcount != 0) { - mTFStats.mTrackletsPerEvent = sumtracklets / eventcount; - mTFStats.mDigitsPerEvent = sumdigits / eventcount; - mTFStats.mTimeTakenForTracklets = sumtracklettime; - mTFStats.mTimeTakenForDigits = sumdigittime; - mTFStats.mTimeTaken = sumtime; + mTFStats.mNTriggersTotal = mEventRecords.size(); + for (const auto& event : mEventRecords) { + mTFStats.mTrackletsFound += event.getTracklets().size(); + mTFStats.mDigitsFound += event.getDigits().size(); + mTFStats.mTimeTakenForTracklets += event.getTrackletTime(); + mTFStats.mTimeTakenForDigits += event.getDigitTime(); + mTFStats.mTimeTaken += event.getTotalTime(); + if (event.getIsCalibTrigger()) { + ++mTFStats.mNTriggersCalib; + } } } From 138cd545a7173093aa991a535eb3983909a60b81 Mon Sep 17 00:00:00 2001 From: Ole Schmidt Date: Mon, 4 Sep 2023 14:45:37 +0200 Subject: [PATCH 038/289] Reverts all commits from PR#11215 (CTF) --- DataFormats/Detectors/Common/CMakeLists.txt | 11 - .../DetectorsCommonDataFormats/ANSHeader.h | 70 -- .../EncodedBlocks.h | 922 +++++------------- .../DetectorsCommonDataFormats/Metadata.h | 193 ---- .../internal/ExternalEntropyCoder.h | 120 --- .../internal/InplaceEntropyCoder.h | 313 ------ .../internal/Packer.h | 112 --- .../Common/test/testCTFEntropyCoder.cxx | 350 ------- .../FIT/FDD/include/DataFormatsFDD/CTF.h | 4 +- .../FIT/FT0/include/DataFormatsFT0/CTF.h | 4 +- .../FIT/FV0/include/DataFormatsFV0/CTF.h | 4 +- .../common/include/DataFormatsITSMFT/CTF.h | 6 +- .../TOF/include/DataFormatsTOF/CTF.h | 8 +- .../Base/include/DetectorsBase/CTFCoderBase.h | 112 +-- .../include/CPVReconstruction/CTFCoder.h | 29 +- .../include/CPVReconstruction/CTFHelper.h | 92 +- Detectors/CPV/reconstruction/src/CTFCoder.cxx | 7 +- .../CPV/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../CPV/workflow/src/EntropyEncoderSpec.cxx | 7 +- Detectors/CTF/test/test_ctf_io_cpv.cxx | 12 +- Detectors/CTF/test/test_ctf_io_ctp.cxx | 12 +- Detectors/CTF/test/test_ctf_io_emcal.cxx | 12 +- Detectors/CTF/test/test_ctf_io_fdd.cxx | 12 +- Detectors/CTF/test/test_ctf_io_ft0.cxx | 12 +- Detectors/CTF/test/test_ctf_io_fv0.cxx | 12 +- Detectors/CTF/test/test_ctf_io_hmpid.cxx | 12 +- Detectors/CTF/test/test_ctf_io_itsmft.cxx | 12 +- Detectors/CTF/test/test_ctf_io_mch.cxx | 12 +- Detectors/CTF/test/test_ctf_io_mid.cxx | 12 +- Detectors/CTF/test/test_ctf_io_phos.cxx | 12 +- Detectors/CTF/test/test_ctf_io_tof.cxx | 12 +- Detectors/CTF/test/test_ctf_io_tpc.cxx | 17 +- Detectors/CTF/test/test_ctf_io_trd.cxx | 12 +- Detectors/CTF/test/test_ctf_io_zdc.cxx | 12 +- Detectors/CTF/utils/CMakeLists.txt | 4 - Detectors/CTF/utils/convCTFDict.C | 121 --- Detectors/CTF/workflow/src/CTFWriterSpec.cxx | 50 +- .../include/CTPReconstruction/CTFCoder.h | 20 +- .../include/CTPReconstruction/CTFHelper.h | 91 +- Detectors/CTP/reconstruction/src/CTFCoder.cxx | 6 +- .../CTP/workflow/src/EntropyDecoderSpec.cxx | 3 +- .../CTP/workflow/src/EntropyEncoderSpec.cxx | 3 +- .../include/EMCALReconstruction/CTFCoder.h | 29 +- .../include/EMCALReconstruction/CTFHelper.h | 89 +- .../EMCAL/reconstruction/src/CTFCoder.cxx | 7 +- .../EMCAL/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../EMCAL/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/FDDReconstruction/CTFCoder.h | 25 +- .../FIT/FDD/reconstruction/src/CTFCoder.cxx | 23 +- .../FDD/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../FDD/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/FT0Reconstruction/CTFCoder.h | 26 +- .../FIT/FT0/reconstruction/src/CTFCoder.cxx | 26 +- .../FT0/workflow/src/EntropyDecoderSpec.cxx | 6 +- .../FT0/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/FV0Reconstruction/CTFCoder.h | 23 +- .../FIT/FV0/reconstruction/src/CTFCoder.cxx | 25 +- .../FV0/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../FV0/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/HMPIDReconstruction/CTFCoder.h | 30 +- .../include/HMPIDReconstruction/CTFHelper.h | 82 +- .../HMPID/reconstruction/src/CTFCoder.cxx | 8 +- .../HMPID/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../HMPID/workflow/src/EntropyEncoderSpec.cxx | 8 +- .../include/ITSMFTReconstruction/CTFCoder.h | 34 +- .../common/reconstruction/src/CTFCoder.cxx | 39 +- .../workflow/src/EntropyDecoderSpec.cxx | 7 +- .../workflow/src/EntropyEncoderSpec.cxx | 7 +- .../MUON/MCH/CTF/include/MCHCTF/CTFCoder.h | 33 +- .../MUON/MCH/CTF/include/MCHCTF/CTFHelper.h | 88 +- Detectors/MUON/MCH/CTF/src/CTFCoder.cxx | 9 +- .../MUON/MCH/CTF/src/EntropyDecoderSpec.cxx | 7 +- .../Workflow/src/entropy-encoder-workflow.cxx | 7 +- .../MUON/MID/CTF/include/MIDCTF/CTFCoder.h | 27 +- .../MUON/MID/CTF/include/MIDCTF/CTFHelper.h | 89 +- Detectors/MUON/MID/CTF/src/CTFCoder.cxx | 7 +- .../MID/Workflow/src/EntropyDecoderSpec.cxx | 8 +- .../MID/Workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/PHOSReconstruction/CTFCoder.h | 27 +- .../include/PHOSReconstruction/CTFHelper.h | 88 +- .../PHOS/reconstruction/src/CTFCoder.cxx | 7 +- .../PHOS/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../PHOS/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/TOFReconstruction/CTFCoder.h | 30 +- Detectors/TOF/reconstruction/src/CTFCoder.cxx | 31 +- .../TOF/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../TOF/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/TPCReconstruction/CTFCoder.h | 65 +- Detectors/TPC/reconstruction/src/CTFCoder.cxx | 77 +- .../TPC/workflow/src/EntropyDecoderSpec.cxx | 3 +- .../TPC/workflow/src/EntropyEncoderSpec.cxx | 3 +- .../include/TRDReconstruction/CTFCoder.h | 46 +- .../include/TRDReconstruction/CTFHelper.h | 84 +- Detectors/TRD/reconstruction/src/CTFCoder.cxx | 8 +- .../TRD/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../TRD/workflow/src/EntropyEncoderSpec.cxx | 7 +- .../include/ZDCReconstruction/CTFCoder.h | 37 +- .../include/ZDCReconstruction/CTFHelper.h | 104 +- Detectors/ZDC/reconstruction/src/CTFCoder.cxx | 7 +- .../ZDC/workflow/src/EntropyDecoderSpec.cxx | 7 +- .../ZDC/workflow/src/EntropyEncoderSpec.cxx | 7 +- Utilities/rANS/CMakeLists.txt | 249 +---- .../benchmarks/bench_ransCombinedIterator.cxx | 15 +- .../rANS/benchmarks/bench_ransDecode.cxx | 162 --- .../benchmarks/bench_ransDecodeScaling.cxx | 121 --- .../rANS/benchmarks/bench_ransEncode.cxx | 351 ------- .../rANS/benchmarks/bench_ransEncodeImpl.cxx | 379 ------- .../rANS/benchmarks/bench_ransHistogram.cxx | 226 ----- Utilities/rANS/benchmarks/bench_ransPack.cxx | 136 --- .../rANS/benchmarks/bench_ransStreaming.cxx | 323 ------ Utilities/rANS/benchmarks/bench_ransTPC.cxx | 390 -------- .../rANS/benchmarks/bench_ransUnpack.cxx | 100 -- Utilities/rANS/benchmarks/helpers.h | 316 ------ Utilities/rANS/include/rANS/Decoder.h | 101 ++ Utilities/rANS/include/rANS/DedupDecoder.h | 106 ++ Utilities/rANS/include/rANS/DedupEncoder.h | 136 +++ Utilities/rANS/include/rANS/Encoder.h | 122 +++ Utilities/rANS/include/rANS/FrequencyTable.h | 363 +++++++ Utilities/rANS/include/rANS/LiteralDecoder.h | 125 +++ Utilities/rANS/include/rANS/LiteralEncoder.h | 138 +++ .../include/rANS/RenormedFrequencyTable.h | 94 ++ Utilities/rANS/include/rANS/compat.h | 295 ------ Utilities/rANS/include/rANS/decode.h | 23 - Utilities/rANS/include/rANS/definitions.h | 36 + Utilities/rANS/include/rANS/encode.h | 23 - Utilities/rANS/include/rANS/factory.h | 238 ----- Utilities/rANS/include/rANS/histogram.h | 25 - .../rANS/include/rANS/internal/Decoder.h | 166 ++++ .../rANS/include/rANS/internal/DecoderBase.h | 72 ++ .../include/rANS/internal/DecoderSymbol.h | 58 ++ .../rANS/include/rANS/internal/Encoder.h | 156 +++ .../rANS/include/rANS/internal/EncoderBase.h | 70 ++ .../include/rANS/internal/EncoderSymbol.h | 145 +++ .../rANS/internal/ReverseSymbolLookupTable.h | 98 ++ .../rANS/include/rANS/internal/SymbolTable.h | 129 +++ .../rANS/internal/common/codertraits.h | 116 --- .../rANS/internal/common/containertraits.h | 289 ------ .../include/rANS/internal/common/defaults.h | 62 -- .../include/rANS/internal/common/defines.h | 69 -- .../include/rANS/internal/common/exceptions.h | 97 -- .../include/rANS/internal/common/simdops.h | 469 --------- .../include/rANS/internal/common/simdtypes.h | 123 --- .../include/rANS/internal/common/typetraits.h | 121 --- .../rANS/include/rANS/internal/common/utils.h | 322 ------ .../internal/containers/AdaptiveHistogram.h | 133 --- .../internal/containers/AdaptiveSymbolTable.h | 139 --- .../rANS/internal/containers/AlignedArray.h | 314 ------ .../include/rANS/internal/containers/BitPtr.h | 167 ---- .../rANS/internal/containers/Container.h | 216 ---- .../rANS/internal/containers/DenseHistogram.h | 621 ------------ .../internal/containers/DenseSymbolTable.h | 158 --- .../rANS/internal/containers/HashTable.h | 102 -- .../containers/HighRangeDecoderTable.h | 104 -- .../internal/containers/HistogramConcept.h | 89 -- .../internal/containers/HistogramInterface.h | 88 -- .../rANS/internal/containers/HistogramView.h | 240 ----- .../containers/LowRangeDecoderTable.h | 77 -- .../rANS/internal/containers/OrderedSet.h | 290 ------ .../internal/containers/RenormedHistogram.h | 98 -- .../containers/ReverseSymbolLookupTable.h | 82 -- .../internal/containers/ShiftableVector.h | 202 ---- .../internal/containers/SparseHistogram.h | 147 --- .../internal/containers/SparseSymbolTable.h | 132 --- .../rANS/internal/containers/SparseVector.h | 366 ------- .../include/rANS/internal/containers/Symbol.h | 215 ---- .../include/rANS/internal/decode/Decoder.h | 101 -- .../rANS/internal/decode/DecoderConcept.h | 133 --- .../rANS/internal/decode/DecoderImpl.h | 123 --- .../include/rANS/internal/encode/Encoder.h | 187 ---- .../rANS/internal/encode/EncoderImpl.h | 81 -- .../internal/encode/EncoderSymbolMapper.h | 359 ------- .../rANS/internal/encode/SIMDEncoderImpl.h | 223 ----- .../internal/encode/SingleStreamEncoderImpl.h | 163 ---- .../include/rANS/internal/encode/simdKernel.h | 562 ----------- Utilities/rANS/include/rANS/internal/helper.h | 166 ++++ .../rANS/internal/metrics/DictSizeEstimate.h | 102 -- .../include/rANS/internal/metrics/Metrics.h | 183 ---- .../rANS/internal/metrics/SizeEstimate.h | 139 --- .../rANS/internal/metrics/properties.h | 58 -- .../include/rANS/internal/metrics/utils.h | 92 -- .../internal/pack/DictionaryStreamReader.h | 159 --- .../include/rANS/internal/pack/eliasDelta.h | 75 -- .../rANS/include/rANS/internal/pack/pack.h | 364 ------- .../rANS/include/rANS/internal/pack/utils.h | 86 -- .../rANS/internal/transform/SourceProxy.h | 151 --- .../rANS/internal/transform/algorithm.h | 185 ---- .../rANS/internal/transform/algorithmImpl.h | 256 ----- .../include/rANS/internal/transform/renorm.h | 230 ----- Utilities/rANS/include/rANS/iterator.h | 21 - Utilities/rANS/include/rANS/metrics.h | 24 - Utilities/rANS/include/rANS/pack.h | 23 - Utilities/rANS/include/rANS/rans.h | 75 ++ Utilities/rANS/include/rANS/serialize.h | 239 ----- Utilities/rANS/include/rANS/utils.h | 28 +- .../rANS/include/rANS/utils/HistogramView.h | 159 +++ .../iterator.h => utils/iterators.h} | 78 +- Utilities/rANS/run/bin-encode-decode.cxx | 114 +-- Utilities/rANS/src/FrequencyTable.cxx | 100 ++ Utilities/rANS/src/RenormedFrequencyTable.cxx | 230 +++++ Utilities/rANS/test/test_ransAlignedArray.cxx | 69 -- Utilities/rANS/test/test_ransBitPtr.cxx | 158 --- Utilities/rANS/test/test_ransEncodeDecode.cxx | 250 ++--- .../rANS/test/test_ransFrequencyTable.cxx | 400 ++++++++ .../rANS/test/test_ransHistogramView.cxx | 133 +-- Utilities/rANS/test/test_ransHistograms.cxx | 495 ---------- Utilities/rANS/test/test_ransIterators.cxx | 28 +- Utilities/rANS/test/test_ransMetrics.cxx | 364 ------- Utilities/rANS/test/test_ransPack.cxx | 169 ---- .../test_ransReverseSymbolLookupTable.cxx | 37 +- Utilities/rANS/test/test_ransSIMD.cxx | 206 ---- .../rANS/test/test_ransSIMDEncoderKernels.cxx | 412 -------- Utilities/rANS/test/test_ransSerialize.cxx | 387 -------- Utilities/rANS/test/test_ransSparseVector.cxx | 94 -- Utilities/rANS/test/test_ransSymbolTable.cxx | 121 +++ Utilities/rANS/test/test_ransSymbolTables.cxx | 145 --- Utilities/rANS/test/test_ransUtils.cxx | 49 - dependencies/FindVtune.cmake | 17 - dependencies/O2Dependencies.cmake | 6 - 218 files changed, 4936 insertions(+), 19367 deletions(-) delete mode 100644 DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h delete mode 100644 DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h delete mode 100644 DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h delete mode 100644 DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h delete mode 100644 DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h delete mode 100644 DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx delete mode 100644 Detectors/CTF/utils/convCTFDict.C delete mode 100644 Utilities/rANS/benchmarks/bench_ransDecode.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransDecodeScaling.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransEncode.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransEncodeImpl.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransHistogram.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransPack.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransStreaming.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransTPC.cxx delete mode 100644 Utilities/rANS/benchmarks/bench_ransUnpack.cxx delete mode 100644 Utilities/rANS/benchmarks/helpers.h create mode 100644 Utilities/rANS/include/rANS/Decoder.h create mode 100644 Utilities/rANS/include/rANS/DedupDecoder.h create mode 100644 Utilities/rANS/include/rANS/DedupEncoder.h create mode 100644 Utilities/rANS/include/rANS/Encoder.h create mode 100644 Utilities/rANS/include/rANS/FrequencyTable.h create mode 100644 Utilities/rANS/include/rANS/LiteralDecoder.h create mode 100644 Utilities/rANS/include/rANS/LiteralEncoder.h create mode 100644 Utilities/rANS/include/rANS/RenormedFrequencyTable.h delete mode 100644 Utilities/rANS/include/rANS/compat.h delete mode 100644 Utilities/rANS/include/rANS/decode.h create mode 100644 Utilities/rANS/include/rANS/definitions.h delete mode 100644 Utilities/rANS/include/rANS/encode.h delete mode 100644 Utilities/rANS/include/rANS/factory.h delete mode 100644 Utilities/rANS/include/rANS/histogram.h create mode 100644 Utilities/rANS/include/rANS/internal/Decoder.h create mode 100644 Utilities/rANS/include/rANS/internal/DecoderBase.h create mode 100644 Utilities/rANS/include/rANS/internal/DecoderSymbol.h create mode 100644 Utilities/rANS/include/rANS/internal/Encoder.h create mode 100644 Utilities/rANS/include/rANS/internal/EncoderBase.h create mode 100644 Utilities/rANS/include/rANS/internal/EncoderSymbol.h create mode 100644 Utilities/rANS/include/rANS/internal/ReverseSymbolLookupTable.h create mode 100644 Utilities/rANS/include/rANS/internal/SymbolTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/codertraits.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/containertraits.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/defaults.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/defines.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/exceptions.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/simdops.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/simdtypes.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/typetraits.h delete mode 100644 Utilities/rANS/include/rANS/internal/common/utils.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/AdaptiveHistogram.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/AdaptiveSymbolTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/AlignedArray.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/BitPtr.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/Container.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/DenseHistogram.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/DenseSymbolTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/HashTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/HighRangeDecoderTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/HistogramConcept.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/HistogramInterface.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/HistogramView.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/LowRangeDecoderTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/OrderedSet.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/RenormedHistogram.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/ReverseSymbolLookupTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/ShiftableVector.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/SparseHistogram.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/SparseSymbolTable.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/SparseVector.h delete mode 100644 Utilities/rANS/include/rANS/internal/containers/Symbol.h delete mode 100644 Utilities/rANS/include/rANS/internal/decode/Decoder.h delete mode 100644 Utilities/rANS/include/rANS/internal/decode/DecoderConcept.h delete mode 100644 Utilities/rANS/include/rANS/internal/decode/DecoderImpl.h delete mode 100644 Utilities/rANS/include/rANS/internal/encode/Encoder.h delete mode 100644 Utilities/rANS/include/rANS/internal/encode/EncoderImpl.h delete mode 100644 Utilities/rANS/include/rANS/internal/encode/EncoderSymbolMapper.h delete mode 100644 Utilities/rANS/include/rANS/internal/encode/SIMDEncoderImpl.h delete mode 100644 Utilities/rANS/include/rANS/internal/encode/SingleStreamEncoderImpl.h delete mode 100644 Utilities/rANS/include/rANS/internal/encode/simdKernel.h create mode 100644 Utilities/rANS/include/rANS/internal/helper.h delete mode 100644 Utilities/rANS/include/rANS/internal/metrics/DictSizeEstimate.h delete mode 100644 Utilities/rANS/include/rANS/internal/metrics/Metrics.h delete mode 100644 Utilities/rANS/include/rANS/internal/metrics/SizeEstimate.h delete mode 100644 Utilities/rANS/include/rANS/internal/metrics/properties.h delete mode 100644 Utilities/rANS/include/rANS/internal/metrics/utils.h delete mode 100644 Utilities/rANS/include/rANS/internal/pack/DictionaryStreamReader.h delete mode 100644 Utilities/rANS/include/rANS/internal/pack/eliasDelta.h delete mode 100644 Utilities/rANS/include/rANS/internal/pack/pack.h delete mode 100644 Utilities/rANS/include/rANS/internal/pack/utils.h delete mode 100644 Utilities/rANS/include/rANS/internal/transform/SourceProxy.h delete mode 100644 Utilities/rANS/include/rANS/internal/transform/algorithm.h delete mode 100644 Utilities/rANS/include/rANS/internal/transform/algorithmImpl.h delete mode 100644 Utilities/rANS/include/rANS/internal/transform/renorm.h delete mode 100644 Utilities/rANS/include/rANS/iterator.h delete mode 100644 Utilities/rANS/include/rANS/metrics.h delete mode 100644 Utilities/rANS/include/rANS/pack.h create mode 100644 Utilities/rANS/include/rANS/rans.h delete mode 100644 Utilities/rANS/include/rANS/serialize.h create mode 100644 Utilities/rANS/include/rANS/utils/HistogramView.h rename Utilities/rANS/include/rANS/{internal/transform/iterator.h => utils/iterators.h} (81%) create mode 100644 Utilities/rANS/src/FrequencyTable.cxx create mode 100644 Utilities/rANS/src/RenormedFrequencyTable.cxx delete mode 100644 Utilities/rANS/test/test_ransAlignedArray.cxx delete mode 100644 Utilities/rANS/test/test_ransBitPtr.cxx create mode 100644 Utilities/rANS/test/test_ransFrequencyTable.cxx delete mode 100644 Utilities/rANS/test/test_ransHistograms.cxx delete mode 100644 Utilities/rANS/test/test_ransMetrics.cxx delete mode 100644 Utilities/rANS/test/test_ransPack.cxx delete mode 100644 Utilities/rANS/test/test_ransSIMD.cxx delete mode 100644 Utilities/rANS/test/test_ransSIMDEncoderKernels.cxx delete mode 100644 Utilities/rANS/test/test_ransSerialize.cxx delete mode 100644 Utilities/rANS/test/test_ransSparseVector.cxx create mode 100644 Utilities/rANS/test/test_ransSymbolTable.cxx delete mode 100644 Utilities/rANS/test/test_ransSymbolTables.cxx delete mode 100644 Utilities/rANS/test/test_ransUtils.cxx delete mode 100644 dependencies/FindVtune.cmake diff --git a/DataFormats/Detectors/Common/CMakeLists.txt b/DataFormats/Detectors/Common/CMakeLists.txt index bbc0aaf9e8320..6b090771f292a 100644 --- a/DataFormats/Detectors/Common/CMakeLists.txt +++ b/DataFormats/Detectors/Common/CMakeLists.txt @@ -52,14 +52,3 @@ o2_add_test(DetID PUBLIC_LINK_LIBRARIES O2::DetectorsCommonDataFormats COMPONENT_NAME DetectorsCommonDataFormats LABELS dataformats) - -o2_add_test(CTFEntropyCoder - NAME CTFEntropyCoder - SOURCES test/testCTFEntropyCoder.cxx - PUBLIC_LINK_LIBRARIES O2::DetectorsCommonDataFormats O2::rANS - COMPONENT_NAME DetectorsCommonDataFormats - TARGETVARNAME TEST_CTF_ENTROPY_CODER - LABELS detectorsbase) -if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64") - target_compile_options(${TEST_CTF_ENTROPY_CODER} PRIVATE -march=native) -endif() diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h deleted file mode 100644 index 110af41cddd49..0000000000000 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/ANSHeader.h +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file ANSHeader.h -/// \author michael.lettrich@cern.ch -/// \brief representation of ANS Version number in a comparable way - -#ifndef _ALICEO2_ANSHEADER_H_ -#define _ALICEO2_ANSHEADER_H_ - -#include -#include -#include -#include - -namespace o2::ctf -{ - -struct ANSHeader { - uint8_t majorVersion; - uint8_t minorVersion; - - void clear() { majorVersion = minorVersion = 0; } - inline constexpr operator uint16_t() const noexcept - { - uint16_t major = majorVersion; - uint16_t minor = minorVersion; - return (major << 8) | minor; - }; - - inline operator std::string() const - { - return fmt::format("{}.{}", majorVersion, minorVersion); - }; - inline constexpr uint32_t version() const noexcept { return static_cast(*this); }; - inline constexpr bool operator==(const ANSHeader& other) const noexcept { return static_cast(*this) == static_cast(other); }; - inline constexpr bool operator!=(const ANSHeader& other) const noexcept { return static_cast(*this) != static_cast(other); }; - inline constexpr bool operator<(const ANSHeader& other) const noexcept { return static_cast(*this) < static_cast(other); }; - inline constexpr bool operator>(const ANSHeader& other) const noexcept { return static_cast(*this) > static_cast(other); }; - inline constexpr bool operator>=(const ANSHeader& other) const noexcept { return static_cast(*this) >= static_cast(other); }; - inline constexpr bool operator<=(const ANSHeader& other) const noexcept { return static_cast(*this) <= static_cast(other); }; - ClassDefNV(ANSHeader, 2); -}; - -inline constexpr ANSHeader ANSVersionUnspecified{0, 0}; -inline constexpr ANSHeader ANSVersionCompat{0, 1}; -inline constexpr ANSHeader ANSVersion1{1, 0}; - -inline ANSHeader ansVersionFromString(const std::string& ansVersionString) -{ - if (ansVersionString == "0.1" || ansVersionString == "compat") { - return ctf::ANSVersionCompat; - } else if (ansVersionString == "1.0") { - return ctf::ANSVersion1; - } else { - return ctf::ANSVersionUnspecified; - } -} - -} // namespace o2::ctf - -#endif /* _ALICEO2_ANSHEADER_H_ */ diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h index 4e3dd1f9f4004..e29067e6d6030 100644 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h +++ b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/EncodedBlocks.h @@ -16,30 +16,18 @@ #ifndef ALICEO2_ENCODED_BLOCKS_H #define ALICEO2_ENCODED_BLOCKS_H -// #undef NDEBUG -// #include +#undef NDEBUG +#include #include #include #include -#include - +#include "rANS/rans.h" +#include "rANS/utils.h" #include "TTree.h" #include "CommonUtils/StringUtils.h" #include "Framework/Logger.h" #include "DetectorsCommonDataFormats/CTFDictHeader.h" #include "DetectorsCommonDataFormats/CTFIOSize.h" -#include "DetectorsCommonDataFormats/ANSHeader.h" -#include "DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h" -#include "DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h" -#include "DetectorsCommonDataFormats/internal/Packer.h" -#include "DetectorsCommonDataFormats/Metadata.h" -#include "rANS/compat.h" -#include "rANS/histogram.h" -#include "rANS/serialize.h" -#include "rANS/factory.h" -#include "rANS/metrics.h" -#include "rANS/serialize.h" -#include "rANS/utils.h" namespace o2 { @@ -62,20 +50,9 @@ struct is_iterator inline constexpr bool is_iterator_v = is_iterator::value; - -inline constexpr bool mayEEncode(Metadata::OptStore opt) noexcept -{ - return (opt == Metadata::OptStore::EENCODE) || (opt == Metadata::OptStore::EENCODE_OR_PACK); -} - -inline constexpr bool mayPack(Metadata::OptStore opt) noexcept -{ - return (opt == Metadata::OptStore::PACK) || (opt == Metadata::OptStore::EENCODE_OR_PACK); -} - } // namespace detail -constexpr size_t PackingThreshold = 512; +using namespace o2::rans; constexpr size_t Alignment = 16; constexpr int WrappersSplitLevel = 99; @@ -99,10 +76,10 @@ inline T* relocatePointer(const char* oldBase, char* newBase, const T* ptr) } template = sizeof(source_T)), bool> = true> -inline constexpr size_t calculateNDestTElements(size_t nElems) noexcept +inline size_t calculateNDestTElements(size_t sourceElems) noexcept { - const size_t srcBufferSize = nElems * sizeof(source_T); - return srcBufferSize / sizeof(dest_T) + (srcBufferSize % sizeof(dest_T) != 0); + const size_t sizeOfSourceArray = sourceElems * sizeof(source_T); + return sizeOfSourceArray / sizeof(dest_T) + (sizeOfSourceArray % sizeof(dest_T) != 0); }; template = sizeof(source_T)), bool> = true> @@ -117,6 +94,52 @@ inline size_t calculatePaddedSize(size_t nElems) noexcept ///>>======================== Auxiliary classes =======================>> +struct ANSHeader { + uint8_t majorVersion; + uint8_t minorVersion; + + void clear() { majorVersion = minorVersion = 0; } + ClassDefNV(ANSHeader, 1); +}; + +struct Metadata { + enum class OptStore : uint8_t { // describe how the store the data described by this metadata + EENCODE, // entropy encoding applied + ROOTCompression, // original data repacked to array with slot-size = streamSize and saved with root compression + NONE, // original data repacked to array with slot-size = streamSize and saved w/o compression + NODATA // no data was provided + }; + size_t messageLength = 0; + size_t nLiterals = 0; + uint8_t messageWordSize = 0; + uint8_t coderType = 0; + uint8_t streamSize = 0; + uint8_t probabilityBits = 0; + OptStore opt = OptStore::EENCODE; + int32_t min = 0; + int32_t max = 0; + int nDictWords = 0; + int nDataWords = 0; + int nLiteralWords = 0; + + size_t getUncompressedSize() const { return messageLength * messageWordSize; } + size_t getCompressedSize() const { return (nDictWords + nDataWords + nLiteralWords) * streamSize; } + void clear() + { + min = max = 0; + messageLength = 0; + messageWordSize = 0; + nLiterals = 0; + coderType = 0; + streamSize = 0; + probabilityBits = 0; + nDictWords = 0; + nDataWords = 0; + nLiteralWords = 0; + } + ClassDefNV(Metadata, 2); +}; + /// registry struct for the buffer start and offsets of writable space struct Registry { char* head = nullptr; //! pointer on the head of the CTF @@ -137,12 +160,6 @@ struct Registry { return size - offsFreeStart; } - char* getFreeBlockEnd() const - { - assert(offsFreeStart <= size); - return getFreeBlockStart() + getFreeSize(); - } - ClassDefNV(Registry, 1); }; @@ -161,21 +178,11 @@ struct Block { inline const W* getData() const { return nData ? (payload + nDict) : nullptr; } inline const W* getDataPointer() const { return payload ? (payload + nDict) : nullptr; } // needed when nData is not set yet inline const W* getLiterals() const { return nLiterals ? (payload + nDict + nData) : nullptr; } - inline const W* getEndOfBlock() const - { - if (!registry) { - return nullptr; - } - // get last legal W*, since unaligned data is undefined behavior! - const size_t delta = reinterpret_cast(registry->getFreeBlockEnd()) % sizeof(W); - return reinterpret_cast(registry->getFreeBlockEnd() - delta); - } inline W* getCreatePayload() { return payload ? payload : (registry ? (payload = reinterpret_cast(registry->getFreeBlockStart())) : nullptr); } inline W* getCreateDict() { return payload ? payload : getCreatePayload(); } inline W* getCreateData() { return payload ? (payload + nDict) : getCreatePayload(); } inline W* getCreateLiterals() { return payload ? payload + (nDict + nData) : getCreatePayload(); } - inline W* getEndOfBlock() { return const_cast(static_cast(*this).getEndOfBlock()); }; inline auto getOffsDict() { return reinterpret_cast(getCreateDict()) - reinterpret_cast(registry->head); } inline auto getOffsData() { return reinterpret_cast(getCreateData()) - reinterpret_cast(registry->head); } @@ -333,9 +340,6 @@ class EncodedBlocks public: typedef EncodedBlocks base; - template - using dictionaryType = std::variant, rans::RenormedDenseHistogram>; - void setHeader(const H& h) { mHeader = h; } const H& getHeader() const { return mHeader; } H& getHeader() { return mHeader; } @@ -357,47 +361,15 @@ class EncodedBlocks return mBlocks[i]; } - template - dictionaryType getDictionary(int i, ANSHeader ansVersion = ANSVersionUnspecified) const + o2::rans::RenormedFrequencyTable getFrequencyTable(int i) const { const auto& block = getBlock(i); const auto& metadata = getMetadata(i); - ansVersion = checkANSVersion(ansVersion); - - assert(static_cast(std::numeric_limits::min()) <= static_cast(metadata.max)); - assert(static_cast(std::numeric_limits::max()) >= static_cast(metadata.min)); - - if (ansVersion == ANSVersionCompat) { - rans::DenseHistogram histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min}; - return rans::compat::renorm(std::move(histogram), metadata.probabilityBits); - } else if (ansVersion == ANSVersion1) { - // dictionary is loaded from an explicit dict file and is stored densly - if (getANSHeader() == ANSVersionUnspecified) { - rans::DenseHistogram histogram{block.getDict(), block.getDict() + block.getNDict(), metadata.min}; - size_t renormingBits = rans::utils::sanitizeRenormingBitRange(metadata.probabilityBits); - LOG_IF(debug, renormingBits != metadata.probabilityBits) << fmt::format("While reading metadata from external dictionary, rANSV1 is rounding renorming precision from {} to {}"); - return rans::renorm(std::move(histogram), renormingBits, rans::RenormingPolicy::ForceIncompressible); - } else { - // dictionary is elias-delta coded inside the block - if constexpr (sizeof(source_T) > 2) { - return rans::readRenormedSetDictionary(block.getDict(), block.getDict() + block.getNDict(), - static_cast(metadata.min), static_cast(metadata.max), - metadata.probabilityBits); - } else { - return rans::readRenormedDictionary(block.getDict(), block.getDict() + block.getNDict(), - static_cast(metadata.min), static_cast(metadata.max), - metadata.probabilityBits); - } - } - } else { - throw std::runtime_error(fmt::format("Failed to load serialized Dictionary. Unsupported ANS Version: {}", static_cast(ansVersion))); - } - }; - - void setANSHeader(const ANSHeader& h) - { - mANSHeader = h; + rans::FrequencyTable frequencyTable{block.getDict(), block.getDict() + block.getNDict(), metadata.min}; + return rans::renorm(std::move(frequencyTable), metadata.probabilityBits); } + + void setANSHeader(const ANSHeader& h) { mANSHeader = h; } const ANSHeader& getANSHeader() const { return mANSHeader; } ANSHeader& getANSHeader() { return mANSHeader; } @@ -463,25 +435,25 @@ class EncodedBlocks /// encode vector src to bloc at provided slot template - inline o2::ctf::CTFIOSize encode(const VE& src, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f) + inline o2::ctf::CTFIOSize encode(const VE& src, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const void* encoderExt = nullptr, float memfc = 1.f) { return encode(std::begin(src), std::end(src), slot, symbolTablePrecision, opt, buffer, encoderExt, memfc); } /// encode vector src to bloc at provided slot template - o2::ctf::CTFIOSize encode(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f); + o2::ctf::CTFIOSize encode(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, Metadata::OptStore opt, buffer_T* buffer = nullptr, const void* encoderExt = nullptr, float memfc = 1.f); /// decode block at provided slot to destination vector (will be resized as needed) template - o2::ctf::CTFIOSize decode(container_T& dest, int slot, const std::any& decoderExt = {}) const; + o2::ctf::CTFIOSize decode(container_T& dest, int slot, const void* decoderExt = nullptr) const; /// decode block at provided slot to destination pointer, the needed space assumed to be available template , bool> = true> - o2::ctf::CTFIOSize decode(D_IT dest, int slot, const std::any& decoderExt = {}) const; + o2::ctf::CTFIOSize decode(D_IT dest, int slot, const void* decoderExt = nullptr) const; /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables - static std::vector createDictionaryBlocks(const std::vector>& vfreq, const std::vector& prbits); + static std::vector createDictionaryBlocks(const std::vector& vfreq, const std::vector& prbits); /// print itself void print(const std::string& prefix = "", int verbosity = 1) const; @@ -496,8 +468,6 @@ class EncodedBlocks std::array mMetadata; // compressed block's details std::array, N> mBlocks; //! this is in fact stored, but to overcome TBuffer limits we have to define the branches per block!!! - inline static constexpr Metadata::OptStore FallbackStorageType{Metadata::OptStore::NONE}; - /// setup internal structure and registry for given buffer size (in bytes!!!) void init(size_t sz); @@ -525,86 +495,7 @@ class EncodedBlocks template static bool readTreeBranch(TTree& tree, const std::string& brname, D& dt, int ev = 0); - template - auto expandStorage(size_t slot, size_t nElemets, T* buffer = nullptr) -> decltype(auto); - - inline ANSHeader checkANSVersion(ANSHeader ansVersion) const - { - auto ctfANSHeader = getANSHeader(); - ANSHeader ret{ANSVersionUnspecified}; - - const bool isEqual{ansVersion == ctfANSHeader}; - const bool isHeaderUnspecified{ctfANSHeader == ANSVersionUnspecified}; - - if (isEqual) { - if (isHeaderUnspecified) { - throw std::runtime_error{fmt::format("Missmatch of ANSVersions, trying to encode/decode CTF with ANS Version Header {} with ANS Version {}", - static_cast(ctfANSHeader), - static_cast(ansVersion))}; - } else { - ret = ctfANSHeader; - } - } else { - if (isHeaderUnspecified) { - ret = ansVersion; - } else { - ret = ctfANSHeader; - } - } - - return ret; - }; - - template - o2::ctf::CTFIOSize entropyCodeRANSCompat(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f); - - template - o2::ctf::CTFIOSize entropyCodeRANSV1(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer = nullptr, const std::any& encoderExt = {}, float memfc = 1.f); - - template - o2::ctf::CTFIOSize encodeRANSV1External(const input_IT srcBegin, const input_IT srcEnd, int slot, const std::any& encoderExt, buffer_T* buffer = nullptr, double_t sizeEstimateSafetyFactor = 1); - - template - o2::ctf::CTFIOSize encodeRANSV1Inplace(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer = nullptr, double_t sizeEstimateSafetyFactor = 1); - - template - o2::ctf::CTFIOSize pack(const input_IT srcBegin, const input_IT srcEnd, int slot, rans::Metrics::value_type> metrics, buffer_T* buffer = nullptr); - - template - inline o2::ctf::CTFIOSize pack(const input_IT srcBegin, const input_IT srcEnd, int slot, buffer_T* buffer = nullptr) - { - using source_type = typename std::iterator_traits::value_type; - - rans::Metrics metrics{}; - - const auto [minIter, maxIter] = std::minmax_element(srcBegin, srcEnd); - if (minIter != maxIter) { - metrics.getDatasetProperties().min = *minIter; - metrics.getDatasetProperties().max = *maxIter; - metrics.getDatasetProperties().alphabetRangeBits = rans::utils::getRangeBits(metrics.getDatasetProperties().min, - metrics.getDatasetProperties().max); - } - - return pack(srcBegin, srcEnd, slot, metrics, buffer); - } - - template - o2::ctf::CTFIOSize store(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer = nullptr); - - // decode - template - CTFIOSize decodeCompatImpl(dst_IT dest, int slot, const std::any& decoderExt) const; - - template - CTFIOSize decodeRansV1Impl(dst_IT dest, int slot, const std::any& decoderExt) const; - - template - CTFIOSize decodeUnpackImpl(dst_IT dest, int slot) const; - - template - CTFIOSize decodeCopyImpl(dst_IT dest, int slot) const; - - ClassDefNV(EncodedBlocks, 3); + ClassDefNV(EncodedBlocks, 2); }; ///_____________________________________________________________________________ @@ -844,7 +735,6 @@ inline auto EncodedBlocks::create(VD& v) template void EncodedBlocks::print(const std::string& prefix, int verbosity) const { - verbosity = 5; if (verbosity > 0) { LOG(info) << prefix << "Container of " << N << " blocks, size: " << size() << " bytes, unused: " << getFreeSize(); for (int i = 0; i < N; i++) { @@ -869,9 +759,9 @@ void EncodedBlocks::print(const std::string& prefix, int verbosity) con ///_____________________________________________________________________________ template template -inline o2::ctf::CTFIOSize EncodedBlocks::decode(container_T& dest, // destination container - int slot, // slot of the block to decode - const std::any& decoderExt) const // optional externally provided decoder +inline o2::ctf::CTFIOSize EncodedBlocks::decode(container_T& dest, // destination container + int slot, // slot of the block to decode + const void* decoderExt) const // optional externally provided decoder { dest.resize(mMetadata[slot].messageLength); // allocate output buffer return decode(std::begin(dest), slot, decoderExt); @@ -880,166 +770,54 @@ inline o2::ctf::CTFIOSize EncodedBlocks::decode(container_T& dest, ///_____________________________________________________________________________ template template , bool>> -CTFIOSize EncodedBlocks::decode(D_IT dest, // iterator to destination - int slot, // slot of the block to decode - const std::any& decoderExt) const // optional externally provided decoder +o2::ctf::CTFIOSize EncodedBlocks::decode(D_IT dest, // iterator to destination + int slot, // slot of the block to decode + const void* decoderExt) const // optional externally provided decoder { - // get references to the right data - const auto& ansVersion = getANSHeader(); const auto& block = mBlocks[slot]; const auto& md = mMetadata[slot]; - if (!block.getNStored()) { - return {0, md.getUncompressedSize(), md.getCompressedSize()}; - } + using dest_t = typename std::iterator_traits::value_type; - if (ansVersion == ANSVersionCompat) { - if (md.opt == Metadata::OptStore::EENCODE) { - return decodeCompatImpl(dest, slot, decoderExt); - } else { - return decodeCopyImpl(dest, slot); - } - } else if (ansVersion == ANSVersion1) { + // decode + if (block.getNStored()) { if (md.opt == Metadata::OptStore::EENCODE) { - return decodeRansV1Impl(dest, slot, decoderExt); - } else if (md.opt == Metadata::OptStore::PACK) { - return decodeUnpackImpl(dest, slot); - } else { - return decodeCopyImpl(dest, slot); - } - } else { - throw std::runtime_error("unsupported ANS Version"); - } -}; - -template -template -CTFIOSize EncodedBlocks::decodeCompatImpl(dst_IT dstBegin, int slot, const std::any& decoderExt) const -{ - - // get references to the right data - const auto& ansVersion = getANSHeader(); - const auto& block = mBlocks[slot]; - const auto& md = mMetadata[slot]; - - using dst_type = typename std::iterator_traits::value_type; - using decoder_type = typename rans::compat::decoder_type; - - std::optional inplaceDecoder{}; - if (md.nDictWords > 0) { - inplaceDecoder = decoder_type{std::get>(this->getDictionary(slot))}; - } else if (!decoderExt.has_value()) { - throw std::runtime_error("neither dictionary nor external decoder provided"); - } - - auto getDecoder = [&]() -> const decoder_type& { - if (inplaceDecoder.has_value()) { - return inplaceDecoder.value(); - } else { - return std::any_cast(decoderExt); - } - }; - - const size_t NDecoderStreams = rans::compat::defaults::CoderPreset::nStreams; - - if (block.getNLiterals()) { - auto* literalsEnd = reinterpret_cast(block.getLiterals()) + md.nLiterals; - getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams, literalsEnd); - } else { - getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, NDecoderStreams); - } - return {0, md.getUncompressedSize(), md.getCompressedSize()}; -}; - -template -template -CTFIOSize EncodedBlocks::decodeRansV1Impl(dst_IT dstBegin, int slot, const std::any& decoderExt) const -{ - - // get references to the right data - const auto& ansVersion = getANSHeader(); - const auto& block = mBlocks[slot]; - const auto& md = mMetadata[slot]; - - using dst_type = typename std::iterator_traits::value_type; - using decoder_type = typename rans::defaultDecoder_type; - - std::optional inplaceDecoder{}; - if (md.nDictWords > 0) { - std::visit([&](auto&& arg) { inplaceDecoder = decoder_type{arg}; }, this->getDictionary(slot)); - } else if (!decoderExt.has_value()) { - throw std::runtime_error("no dictionary nor external decoder provided"); - } - - auto getDecoder = [&]() -> const decoder_type& { - if (inplaceDecoder.has_value()) { - return inplaceDecoder.value(); - } else { - return std::any_cast(decoderExt); - } - }; - - // verify decoders - [&]() { - const decoder_type& decoder = getDecoder(); - const size_t decoderSymbolTablePrecision = decoder.getSymbolTablePrecision(); - - if (md.probabilityBits != decoderSymbolTablePrecision) { - throw std::runtime_error(fmt::format( - "Missmatch in decoder renorming precision vs metadata:{} Bits vs {} Bits.", - md.probabilityBits, decoderSymbolTablePrecision)); - } - - if (md.streamSize != rans::utils::getStreamingLowerBound_v) { - throw std::runtime_error("Streaming lower bound of dataset and decoder do not match"); + if (!decoderExt && !block.getNDict()) { + LOG(error) << "Dictionaty is not saved for slot " << slot << " and no external decoder is provided"; + throw std::runtime_error("Dictionary is not saved and no external decoder provided"); + } + const o2::rans::LiteralDecoder64* decoder = reinterpret_cast*>(decoderExt); + std::unique_ptr> decoderLoc; + if (block.getNDict()) { // if dictionaty is saved, prefer it + decoderLoc = std::make_unique>(this->getFrequencyTable(slot)); + decoder = decoderLoc.get(); + } else { // verify that decoded corresponds to stored metadata + if (md.min != decoder->getMinSymbol()) { + LOG(error) << "Mismatch between min=" << md.min << " symbol in metadata and those in external decoder " + << decoder->getMinSymbol() << " for slot " << slot; + throw std::runtime_error("Mismatch between min symbol in metadata and the one in external decoder"); + } + } + // load incompressible symbols if they existed + std::vector literals; + if (block.getNLiterals()) { + // note: here we have to use md.nLiterals (original number of literal words) rather than md.nLiteralWords == block.getNLiterals() + // (number of W-words in the EncodedBlock occupied by literals) as we cast literals stored in W-word array + // to D-word array + literals = std::vector{reinterpret_cast(block.getLiterals()), reinterpret_cast(block.getLiterals()) + md.nLiterals}; + } + decoder->process(block.getData() + block.getNData(), dest, md.messageLength, literals); + } else { // data was stored as is + using destPtr_t = typename std::iterator_traits::pointer; + destPtr_t srcBegin = reinterpret_cast(block.payload); + destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t); + std::copy(srcBegin, srcEnd, dest); + // std::memcpy(dest, block.payload, md.messageLength * sizeof(dest_t)); } - }(); - - // do the actual decoding - if (block.getNLiterals()) { - std::vector literals(md.nLiterals); - rans::unpack(block.getLiterals(), md.nLiterals, literals.data(), md.literalsPackingWidth, md.literalsPackingOffset); - getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams, literals.end()); - } else { - getDecoder().process(block.getData() + block.getNData(), dstBegin, md.messageLength, md.nStreams); } return {0, md.getUncompressedSize(), md.getCompressedSize()}; -}; - -template -template -CTFIOSize EncodedBlocks::decodeUnpackImpl(dst_IT dest, int slot) const -{ - using dest_t = typename std::iterator_traits::value_type; - - const auto& block = mBlocks[slot]; - const auto& md = mMetadata[slot]; - - const size_t packingWidth = md.probabilityBits; - const dest_t offset = md.min; - rans::unpack(block.getData(), md.messageLength, dest, packingWidth, offset); - return {0, md.getUncompressedSize(), md.getCompressedSize()}; -}; - -template -template -CTFIOSize EncodedBlocks::decodeCopyImpl(dst_IT dest, int slot) const -{ - // get references to the right data - const auto& block = mBlocks[slot]; - const auto& md = mMetadata[slot]; - - using dest_t = typename std::iterator_traits::value_type; - using decoder_t = typename rans::compat::decoder_type; - using destPtr_t = typename std::iterator_traits::pointer; - - destPtr_t srcBegin = reinterpret_cast(block.payload); - destPtr_t srcEnd = srcBegin + md.messageLength * sizeof(dest_t); - std::copy(srcBegin, srcEnd, dest); - - return {0, md.getUncompressedSize(), md.getCompressedSize()}; -}; +} ///_____________________________________________________________________________ template @@ -1050,408 +828,147 @@ o2::ctf::CTFIOSize EncodedBlocks::encode(const input_IT srcBegin, uint8_t symbolTablePrecision, // encoding into Metadata::OptStore opt, // option for data compression buffer_T* buffer, // optional buffer (vector) providing memory for encoded blocks - const std::any& encoderExt, // optional external encoder + const void* encoderExt, // optional external encoder float memfc) // memory allocation margin factor { + + using storageBuffer_t = W; + using input_t = typename std::iterator_traits::value_type; + using ransEncoder_t = typename rans::LiteralEncoder64; + using ransState_t = typename ransEncoder_t::coder_t; + using ransStream_t = typename ransEncoder_t::stream_t; + + // assert at compile time that output types align so that padding is not necessary. + static_assert(std::is_same_v); + static_assert(std::is_same_v); + // fill a new block assert(slot == mRegistry.nFilledBlocks); mRegistry.nFilledBlocks++; const size_t messageLength = std::distance(srcBegin, srcEnd); // cover three cases: - // * empty source message: no co + // * empty source message: no entropy coding // * source message to pass through without any entropy coding // * source message where entropy coding should be applied // case 1: empty source message if (messageLength == 0) { - mMetadata[slot] = Metadata{}; - mMetadata[slot].opt = Metadata::OptStore::NODATA; + mMetadata[slot] = Metadata{0, 0, sizeof(input_t), sizeof(ransState_t), sizeof(ransStream_t), symbolTablePrecision, Metadata::OptStore::NODATA, 0, 0, 0, 0, 0}; return {}; } - if (detail::mayEEncode(opt)) { - const ANSHeader& ansVersion = getANSHeader(); - if (ansVersion == ANSVersionCompat) { - return entropyCodeRANSCompat(srcBegin, srcEnd, slot, symbolTablePrecision, buffer, encoderExt, memfc); - } else if (ansVersion == ANSVersion1) { - return entropyCodeRANSV1(srcBegin, srcEnd, slot, opt, buffer, encoderExt, memfc); - } else { - throw std::runtime_error(fmt::format("Unsupported ANS Coder Version: {}.{}", ansVersion.majorVersion, ansVersion.minorVersion)); - } - } else if (detail::mayPack(opt)) { - return pack(srcBegin, srcEnd, slot, buffer); - } else { - return store(srcBegin, srcEnd, slot, opt, buffer); - } -}; - -template -template -[[nodiscard]] auto EncodedBlocks::expandStorage(size_t slot, size_t nElements, T* buffer) -> decltype(auto) -{ - // after previous relocation this (hence its data members) are not guaranteed to be valid - auto* old = get(buffer->data()); - auto* thisBlock = &(old->mBlocks[slot]); - auto* thisMetadata = &(old->mMetadata[slot]); - - // resize underlying buffer of block if necessary and update all pointers. - auto* const blockHead = get(thisBlock->registry->head); // extract pointer from the block, as "this" might be invalid - const size_t additionalSize = blockHead->estimateBlockSize(nElements); // additionalSize is in bytes!!! - if (additionalSize >= thisBlock->registry->getFreeSize()) { - LOGP(debug, "Slot {} with {} available words needs to allocate {} bytes for a total of {} words.", slot, thisBlock->registry->getFreeSize(), additionalSize, nElements); - if (buffer) { - blockHead->expand(*buffer, blockHead->size() + (additionalSize - blockHead->getFreeSize())); - thisMetadata = &(get(buffer->data())->mMetadata[slot]); - thisBlock = &(get(buffer->data())->mBlocks[slot]); // in case of resizing this and any this.xxx becomes invalid - } else { - throw std::runtime_error("failed to allocate additional space in provided external buffer"); - } - } - return std::make_pair(thisBlock, thisMetadata); -}; - -template -template -o2::ctf::CTFIOSize EncodedBlocks::entropyCodeRANSCompat(const input_IT srcBegin, const input_IT srcEnd, int slot, uint8_t symbolTablePrecision, buffer_T* buffer, const std::any& encoderExt, float memfc) -{ - using storageBuffer_t = W; - using input_t = typename std::iterator_traits::value_type; - using ransEncoder_t = typename rans::compat::encoder_type; - using ransState_t = typename ransEncoder_t::coder_type::state_type; - using ransStream_t = typename ransEncoder_t::stream_type; - - // assert at compile time that output types align so that padding is not necessary. - static_assert(std::is_same_v); - static_assert(std::is_same_v); auto* thisBlock = &mBlocks[slot]; auto* thisMetadata = &mMetadata[slot]; - // build symbol statistics - constexpr size_t SizeEstMarginAbs = 10 * 1024; - const float SizeEstMarginRel = 1.5 * memfc; - - const size_t messageLength = std::distance(srcBegin, srcEnd); - rans::DenseHistogram frequencyTable{}; - rans::compat::encoder_type inplaceEncoder{}; - - try { - std::tie(inplaceEncoder, frequencyTable) = [&]() { - if (encoderExt.has_value()) { - return std::make_tuple(ransEncoder_t{}, rans::DenseHistogram{}); + // resize underlying buffer of block if necessary and update all pointers. + auto expandStorage = [&](int additionalElements) { + auto* const blockHead = get(thisBlock->registry->head); // extract pointer from the block, as "this" might be invalid + const size_t additionalSize = blockHead->estimateBlockSize(additionalElements); // size in bytes!!! + if (additionalSize >= thisBlock->registry->getFreeSize()) { + LOG(debug) << "Slot " << slot << ": free size: " << thisBlock->registry->getFreeSize() << ", need " << additionalSize << " for " << additionalElements << " words"; + if (buffer) { + blockHead->expand(*buffer, blockHead->size() + (additionalSize - blockHead->getFreeSize())); + thisMetadata = &(get(buffer->data())->mMetadata[slot]); + thisBlock = &(get(buffer->data())->mBlocks[slot]); // in case of resizing this and any this.xxx becomes invalid } else { - auto histogram = rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd); - auto encoder = rans::compat::makeEncoder::fromHistogram(histogram, symbolTablePrecision); - return std::make_tuple(std::move(encoder), std::move(histogram)); + throw std::runtime_error("no room for encoded block in provided container"); } - }(); - } catch (const rans::HistogramError& error) { - LOGP(warning, "Failed to build Dictionary for rANS encoding, using fallback option"); - return store(srcBegin, srcEnd, slot, this->FallbackStorageType, buffer); - } - const ransEncoder_t& encoder = encoderExt.has_value() ? std::any_cast(encoderExt) : inplaceEncoder; - - // estimate size of encode buffer - int dataSize = rans::compat::calculateMaxBufferSizeB(messageLength, rans::compat::getAlphabetRangeBits(encoder.getSymbolTable())); // size in bytes - // preliminary expansion of storage based on dict size + estimated size of encode buffer - dataSize = SizeEstMarginAbs + int(SizeEstMarginRel * (dataSize / sizeof(storageBuffer_t))) + (sizeof(input_t) < sizeof(storageBuffer_t)); // size in words of output stream - - const auto view = rans::trim(rans::makeHistogramView(frequencyTable)); - std::tie(thisBlock, thisMetadata) = expandStorage(slot, view.size() + dataSize, buffer); - - // store dictionary first - - if (!view.empty()) { - thisBlock->storeDict(view.size(), view.data()); - LOGP(info, "StoreDict {} bytes, offs: {}:{}", view.size() * sizeof(W), thisBlock->getOffsDict(), thisBlock->getOffsDict() + view.size() * sizeof(W)); - } - // vector of incompressible literal symbols - std::vector literals; - // directly encode source message into block buffer. - storageBuffer_t* const blockBufferBegin = thisBlock->getCreateData(); - const size_t maxBufferSize = thisBlock->registry->getFreeSize(); // note: "this" might be not valid after expandStorage call!!! - const auto [encodedMessageEnd, literalsEnd] = encoder.process(srcBegin, srcEnd, blockBufferBegin, std::back_inserter(literals)); - rans::utils::checkBounds(encodedMessageEnd, blockBufferBegin + maxBufferSize / sizeof(W)); - dataSize = encodedMessageEnd - thisBlock->getDataPointer(); - thisBlock->setNData(dataSize); - thisBlock->realignBlock(); - LOGP(info, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(W), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(W)); - // update the size claimed by encode message directly inside the block - - // store incompressible symbols if any - const size_t nLiteralSymbols = literals.size(); - const size_t nLiteralWords = [&]() { - if (!literals.empty()) { - const size_t nSymbols = literals.size(); - // introduce padding in case literals don't align; - const size_t nLiteralSymbolsPadded = calculatePaddedSize(nSymbols); - literals.resize(nLiteralSymbolsPadded, {}); - - const size_t nLiteralStorageElems = calculateNDestTElements(nSymbols); - std::tie(thisBlock, thisMetadata) = expandStorage(slot, nLiteralStorageElems, buffer); - thisBlock->storeLiterals(nLiteralStorageElems, reinterpret_cast(literals.data())); - LOGP(info, "StoreLiterals {} bytes, offs: {}:{}", nLiteralStorageElems * sizeof(W), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + nLiteralStorageElems * sizeof(W)); - return nLiteralStorageElems; } - return size_t(0); - }(); - - LOGP(info, "Min, {} Max, {}, size, {}, nSamples {}", view.getMin(), view.getMax(), view.size(), frequencyTable.getNumSamples()); - - *thisMetadata = detail::makeMetadataRansCompat(encoder.getNStreams(), - messageLength, - nLiteralSymbols, - encoder.getSymbolTable().getPrecision(), - view.getMin(), - view.getMax(), - view.size(), - dataSize, - nLiteralWords); - - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; -} - -template -template -o2::ctf::CTFIOSize EncodedBlocks::entropyCodeRANSV1(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer, const std::any& encoderExt, float memfc) -{ - CTFIOSize encoderStatistics{}; + }; - const size_t nSamples = std::distance(srcBegin, srcEnd); - if (detail::mayPack(opt) && nSamples < PackingThreshold) { - encoderStatistics = pack(srcBegin, srcEnd, slot, buffer); - } else { + // case 3: message where entropy coding should be applied + if (opt == Metadata::OptStore::EENCODE) { + // build symbol statistics + constexpr size_t SizeEstMarginAbs = 10 * 1024; + const float SizeEstMarginRel = 1.5 * memfc; - if (encoderExt.has_value()) { - encoderStatistics = encodeRANSV1External(srcBegin, srcEnd, slot, encoderExt, buffer, memfc); - } else { - encoderStatistics = encodeRANSV1Inplace(srcBegin, srcEnd, slot, opt, buffer, memfc); + const auto [inplaceEncoder, frequencyTable] = [&]() { + if (encoderExt) { + return std::make_tuple(ransEncoder_t{}, rans::FrequencyTable{}); + } else { + rans::FrequencyTable frequencyTable = rans::makeFrequencyTableFromSamples(srcBegin, srcEnd); + RenormedFrequencyTable renormedFrequencyTable = rans::renorm(frequencyTable, symbolTablePrecision); + return std::make_tuple(ransEncoder_t{renormedFrequencyTable}, frequencyTable); + } + }(); + ransEncoder_t const* const encoder = encoderExt ? reinterpret_cast(encoderExt) : &inplaceEncoder; + + // estimate size of encode buffer + int dataSize = rans::calculateMaxBufferSize(messageLength, encoder->getAlphabetRangeBits(), sizeof(input_t)); // size in bytes + // preliminary expansion of storage based on dict size + estimated size of encode buffer + dataSize = SizeEstMarginAbs + int(SizeEstMarginRel * (dataSize / sizeof(storageBuffer_t))) + (sizeof(input_t) < sizeof(storageBuffer_t)); // size in words of output stream + expandStorage(frequencyTable.size() + dataSize); + // store dictionary first + if (!frequencyTable.empty()) { + thisBlock->storeDict(frequencyTable.size(), frequencyTable.data()); + LOGP(debug, "StoreDict {} bytes, offs: {}:{}", frequencyTable.size() * sizeof(W), thisBlock->getOffsDict(), thisBlock->getOffsDict() + frequencyTable.size() * sizeof(W)); } - } - return encoderStatistics; -} - -template -template -CTFIOSize EncodedBlocks::encodeRANSV1External(const input_IT srcBegin, const input_IT srcEnd, int slot, const std::any& encoderExt, buffer_T* buffer, double_t sizeEstimateSafetyFactor) -{ - using storageBuffer_t = W; - using input_t = typename std::iterator_traits::value_type; - using ransEncoder_t = typename internal::ExternalEntropyCoder::encoder_type; - using ransState_t = typename ransEncoder_t::coder_type::state_type; - using ransStream_t = typename ransEncoder_t::stream_type; - - // assert at compile time that output types align so that padding is not necessary. - static_assert(std::is_same_v); - static_assert(std::is_same_v); - - auto* thisBlock = &mBlocks[slot]; - auto* thisMetadata = &mMetadata[slot]; - - const size_t messageLength = std::distance(srcBegin, srcEnd); - internal::ExternalEntropyCoder encoder{std::any_cast(encoderExt)}; - - const size_t payloadSizeWords = encoder.template computePayloadSizeEstimate(messageLength); - std::tie(thisBlock, thisMetadata) = expandStorage(slot, payloadSizeWords, buffer); - - // encode payload - auto encodedMessageEnd = encoder.encode(srcBegin, srcEnd, thisBlock->getCreateData(), thisBlock->getEndOfBlock()); - const size_t dataSize = std::distance(thisBlock->getCreateData(), encodedMessageEnd); - thisBlock->setNData(dataSize); - thisBlock->realignBlock(); - LOGP(info, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(storageBuffer_t)); - // update the size claimed by encoded message directly inside the block - - // encode literals - size_t literalsSize = 0; - if (encoder.getNIncompressibleSamples() > 0) { - const size_t literalsBufferSizeWords = encoder.template computePackedIncompressibleSize(); - std::tie(thisBlock, thisMetadata) = expandStorage(slot, literalsBufferSizeWords, buffer); - auto literalsEnd = encoder.writeIncompressible(thisBlock->getCreateLiterals(), thisBlock->getEndOfBlock()); - literalsSize = std::distance(thisBlock->getCreateLiterals(), literalsEnd); - thisBlock->setNLiterals(literalsSize); + // vector of incompressible literal symbols + std::vector literals; + // directly encode source message into block buffer. + storageBuffer_t* const blockBufferBegin = thisBlock->getCreateData(); + const size_t maxBufferSize = thisBlock->registry->getFreeSize(); // note: "this" might be not valid after expandStorage call!!! + const auto encodedMessageEnd = encoder->process(srcBegin, srcEnd, blockBufferBegin, literals); + rans::utils::checkBounds(encodedMessageEnd, blockBufferBegin + maxBufferSize / sizeof(W)); + dataSize = encodedMessageEnd - thisBlock->getDataPointer(); + thisBlock->setNData(dataSize); thisBlock->realignBlock(); - LOGP(info, "StoreLiterals {} bytes, offs: {}:{}", literalsSize * sizeof(storageBuffer_t), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + literalsSize * sizeof(storageBuffer_t)); - } - - // write metadata - const auto& symbolTable = encoder.getEncoder().getSymbolTable(); - *thisMetadata = detail::makeMetadataRansV1(encoder.getEncoder().getNStreams(), - rans::utils::getStreamingLowerBound_v, - messageLength, - encoder.getNIncompressibleSamples(), - symbolTable.getPrecision(), - symbolTable.getOffset(), - symbolTable.getOffset() + symbolTable.size(), - encoder.getIncompressibleSymbolOffset(), - encoder.getIncompressibleSymbolPackingBits(), - 0, - dataSize, - literalsSize); - - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; -}; - -template -template -CTFIOSize EncodedBlocks::encodeRANSV1Inplace(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer, double_t sizeEstimateSafetyFactor) -{ - using storageBuffer_t = W; - using input_t = typename std::iterator_traits::value_type; - using ransEncoder_t = typename rans::denseEncoder_type; - using ransState_t = typename ransEncoder_t::coder_type::state_type; - using ransStream_t = typename ransEncoder_t::stream_type; - - // assert at compile time that output types align so that padding is not necessary. - static_assert(std::is_same_v); - static_assert(std::is_same_v); - - auto* thisBlock = &mBlocks[slot]; - auto* thisMetadata = &mMetadata[slot]; - - internal::InplaceEntropyCoder encoder{}; - rans::SourceProxy proxy{srcBegin, srcEnd, [](input_IT begin, input_IT end) { - const size_t nSamples = std::distance(begin, end); - return (!std::is_pointer_v && (nSamples < rans::utils::pow2(23))); - }}; - - try { - if (proxy.isCached()) { - encoder = internal::InplaceEntropyCoder{proxy.beginCache(), proxy.endCache()}; - } else { - encoder = internal::InplaceEntropyCoder{proxy.beginIter(), proxy.endIter()}; - } - } catch (const rans::HistogramError& error) { - LOGP(warning, "Failed to build Dictionary for rANS encoding, using fallback option"); - if (proxy.isCached()) { - return store(proxy.beginCache(), proxy.endCache(), slot, this->FallbackStorageType, buffer); - } else { - return store(proxy.beginIter(), proxy.endIter(), slot, this->FallbackStorageType, buffer); - } - } - - const rans::Metrics& metrics = encoder.getMetrics(); - - if constexpr (sizeof(input_t) > 2) { - const auto& dp = metrics.getDatasetProperties(); - LOGP(info, "Metrics:{{slot: {}, numSamples: {}, min: {}, max: {}, alphabetRangeBits: {}, nUsedAlphabetSymbols: {}, preferPacking: {}}}", slot, dp.numSamples, dp.min, dp.max, dp.alphabetRangeBits, dp.nUsedAlphabetSymbols, metrics.getSizeEstimate().preferPacking()); - } - - if (detail::mayPack(opt) && metrics.getSizeEstimate().preferPacking()) { - if (proxy.isCached()) { - return pack(proxy.beginCache(), proxy.endCache(), slot, metrics, buffer); - } else { - return pack(proxy.beginIter(), proxy.endIter(), slot, metrics, buffer); - }; - } + LOGP(debug, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(W), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(W)); + // update the size claimed by encode message directly inside the block + + // store incompressible symbols if any + const size_t nLiteralSymbols = literals.size(); + const size_t nLiteralWords = [&]() { + if (!literals.empty()) { + const size_t nSymbols = literals.size(); + // introduce padding in case literals don't align; + const size_t nLiteralSymbolsPadded = calculatePaddedSize(nSymbols); + literals.resize(nLiteralSymbolsPadded, {}); + + const size_t nLiteralStorageElems = calculateNDestTElements(nSymbols); + expandStorage(nLiteralStorageElems); + thisBlock->storeLiterals(nLiteralStorageElems, reinterpret_cast(literals.data())); + LOGP(debug, "StoreLiterals {} bytes, offs: {}:{}", nLiteralStorageElems * sizeof(W), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + nLiteralStorageElems * sizeof(W)); + return nLiteralStorageElems; + } + return size_t(0); + }(); - encoder.makeEncoder(); - - const rans::SizeEstimate sizeEstimate = metrics.getSizeEstimate(); - const size_t bufferSizeWords = rans::utils::nBytesTo((sizeEstimate.getCompressedDictionarySize() + - sizeEstimate.getCompressedDatasetSize() + - sizeEstimate.getIncompressibleSize()) * - sizeEstimateSafetyFactor); - std::tie(thisBlock, thisMetadata) = expandStorage(slot, bufferSizeWords, buffer); - - // encode dict - auto encodedDictEnd = encoder.writeDictionary(thisBlock->getCreateDict(), thisBlock->getEndOfBlock()); - const size_t dictSize = std::distance(thisBlock->getCreateDict(), encodedDictEnd); - thisBlock->setNDict(dictSize); - thisBlock->realignBlock(); - LOGP(info, "StoreDict {} bytes, offs: {}:{}", dictSize * sizeof(storageBuffer_t), thisBlock->getOffsDict(), thisBlock->getOffsDict() + dictSize * sizeof(storageBuffer_t)); - - // encode payload - auto encodedMessageEnd = thisBlock->getCreateData(); - if (proxy.isCached()) { - encodedMessageEnd = encoder.encode(proxy.beginCache(), proxy.endCache(), thisBlock->getCreateData(), thisBlock->getEndOfBlock()); - } else { - encodedMessageEnd = encoder.encode(proxy.beginIter(), proxy.endIter(), thisBlock->getCreateData(), thisBlock->getEndOfBlock()); - } - const size_t dataSize = std::distance(thisBlock->getCreateData(), encodedMessageEnd); - thisBlock->setNData(dataSize); - thisBlock->realignBlock(); - LOGP(info, "StoreData {} bytes, offs: {}:{}", dataSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + dataSize * sizeof(storageBuffer_t)); - // update the size claimed by encoded message directly inside the block - - // encode literals - size_t literalsSize{}; - if (encoder.getNIncompressibleSamples() > 0) { - auto literalsEnd = encoder.writeIncompressible(thisBlock->getCreateLiterals(), thisBlock->getEndOfBlock()); - literalsSize = std::distance(thisBlock->getCreateLiterals(), literalsEnd); - thisBlock->setNLiterals(literalsSize); - thisBlock->realignBlock(); - LOGP(info, "StoreLiterals {} bytes, offs: {}:{}", literalsSize * sizeof(storageBuffer_t), thisBlock->getOffsLiterals(), thisBlock->getOffsLiterals() + literalsSize * sizeof(storageBuffer_t)); + *thisMetadata = Metadata{messageLength, + nLiteralSymbols, + sizeof(input_t), + sizeof(ransState_t), + sizeof(ransStream_t), + static_cast(encoder->getSymbolTablePrecision()), + opt, + encoder->getMinSymbol(), + encoder->getMaxSymbol(), + static_cast(frequencyTable.size()), + dataSize, + static_cast(nLiteralWords)}; + } else { // store original data w/o EEncoding + // FIXME(milettri): we should be able to do without an intermediate vector; + // provided iterator is not necessarily pointer, need to use intermediate vector!!! + + // introduce padding in case literals don't align; + const size_t nSourceElemsPadded = calculatePaddedSize(messageLength); + std::vector tmp(nSourceElemsPadded, {}); + std::copy(srcBegin, srcEnd, std::begin(tmp)); + + const size_t nBufferElems = calculateNDestTElements(messageLength); + expandStorage(nBufferElems); + thisBlock->storeData(nBufferElems, reinterpret_cast(tmp.data())); + + *thisMetadata = Metadata{messageLength, 0, sizeof(input_t), sizeof(ransState_t), sizeof(storageBuffer_t), symbolTablePrecision, opt, 0, 0, 0, static_cast(nBufferElems), 0}; } - - // write metadata - *thisMetadata = detail::makeMetadataRansV1(encoder.getNStreams(), - rans::utils::getStreamingLowerBound_v, - std::distance(srcBegin, srcEnd), - encoder.getNIncompressibleSamples(), - encoder.getSymbolTablePrecision(), - *metrics.getCoderProperties().min, - *metrics.getCoderProperties().max, - metrics.getDatasetProperties().min, - metrics.getDatasetProperties().alphabetRangeBits, - dictSize, - dataSize, - literalsSize); - - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; -}; // namespace ctf - -template -template -o2::ctf::CTFIOSize EncodedBlocks::pack(const input_IT srcBegin, const input_IT srcEnd, int slot, rans::Metrics::value_type> metrics, buffer_T* buffer) -{ - using storageBuffer_t = W; - using input_t = typename std::iterator_traits::value_type; - - const size_t messageLength = std::distance(srcBegin, srcEnd); - - internal::Packer packer{metrics}; - size_t packingBufferWords = packer.template getPackingBufferSize(messageLength); - auto [thisBlock, thisMetadata] = expandStorage(slot, packingBufferWords, buffer); - - auto packedMessageEnd = packer.pack(srcBegin, srcEnd, thisBlock->getCreateData(), thisBlock->getEndOfBlock()); - const size_t packeSize = std::distance(thisBlock->getCreateData(), packedMessageEnd); - thisBlock->setNData(packeSize); - thisBlock->realignBlock(); - - LOGP(info, "StoreData {} bytes, offs: {}:{}", packeSize * sizeof(storageBuffer_t), thisBlock->getOffsData(), thisBlock->getOffsData() + packeSize * sizeof(storageBuffer_t)); - - *thisMetadata = detail::makeMetadataPack(messageLength, packer.getPackingWidth(), packer.getOffset(), packeSize); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; -}; - -template -template -o2::ctf::CTFIOSize EncodedBlocks::store(const input_IT srcBegin, const input_IT srcEnd, int slot, Metadata::OptStore opt, buffer_T* buffer) -{ - using storageBuffer_t = W; - using input_t = typename std::iterator_traits::value_type; - - const size_t messageLength = std::distance(srcBegin, srcEnd); - // introduce padding in case literals don't align; - const size_t nSourceElemsPadded = calculatePaddedSize(messageLength); - std::vector tmp(nSourceElemsPadded, {}); - std::copy(srcBegin, srcEnd, std::begin(tmp)); - - const size_t nBufferElems = calculateNDestTElements(messageLength); - auto [thisBlock, thisMetadata] = expandStorage(slot, nBufferElems, buffer); - thisBlock->storeData(nBufferElems, reinterpret_cast(tmp.data())); - - *thisMetadata = detail::makeMetadataStore(messageLength, opt, nBufferElems); - return {0, thisMetadata->getUncompressedSize(), thisMetadata->getCompressedSize()}; -}; +} /// create a special EncodedBlocks containing only dictionaries made from provided vector of frequency tables template -std::vector EncodedBlocks::createDictionaryBlocks(const std::vector>& vfreq, const std::vector& vmd) +std::vector EncodedBlocks::createDictionaryBlocks(const std::vector& vfreq, const std::vector& vmd) { - if (vfreq.size() != N) { throw std::runtime_error(fmt::format("mismatch between the size of frequencies vector {} and number of blocks {}", vfreq.size(), N)); } @@ -1462,12 +979,9 @@ std::vector EncodedBlocks::createDictionaryBlocks(const std::vect std::vector vdict(sz); // memory space for dictionary auto dictBlocks = create(vdict.data(), sz); for (int ib = 0; ib < N; ib++) { - const auto& thisHistogram = vfreq[ib]; - const auto view = rans::trim(rans::makeHistogramView(thisHistogram)); - - if (!view.empty()) { - LOG(info) << "adding dictionary of " << view.size() << " words for block " << ib << ", min/max= " << view.getMin() << "/" << view.getMax(); - dictBlocks->mBlocks[ib].storeDict(view.size(), view.data()); + if (vfreq[ib].size()) { + LOG(info) << "adding dictionary of " << vfreq[ib].size() << " words for block " << ib << ", min/max= " << vfreq[ib].getMinSymbol() << "/" << vfreq[ib].getMaxSymbol(); + dictBlocks->mBlocks[ib].storeDict(vfreq[ib].size(), vfreq[ib].data()); dictBlocks = get(vdict.data()); // !!! rellocation might have invalidated dictBlocks pointer dictBlocks->mMetadata[ib] = vmd[ib]; dictBlocks->mMetadata[ib].opt = Metadata::OptStore::ROOTCompression; // we will compress the dictionary with root! @@ -1477,7 +991,7 @@ std::vector EncodedBlocks::createDictionaryBlocks(const std::vect } dictBlocks->mRegistry.nFilledBlocks++; } - return vdict; + return std::move(vdict); } template diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h deleted file mode 100644 index abf7561eb25a9..0000000000000 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/Metadata.h +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file Metadata.h -/// \brief Metadata required to decode a Block - -/// The Metadata required to decoded the CTF of particular detector. - -#ifndef ALICEO2_METADATA_H -#define ALICEO2_METADATA_H - -#include -#include - -namespace o2::ctf -{ - -struct Metadata { - enum class OptStore : uint8_t { // describe how the store the data described by this metadata - EENCODE, // entropy encoding applied - ROOTCompression, // original data repacked to array with slot-size = streamSize and saved with root compression - NONE, // original data repacked to array with slot-size = streamSize and saved w/o compression - NODATA, // no data was provided - PACK, // use Bitpacking - EENCODE_OR_PACK // decide at runtime if to encode or pack - }; - uint8_t nStreams = 0; // Amount of concurrent Streams used by the encoder. only used by rANS version >=1. - size_t messageLength = 0; // Message length (multiply with messageWordSize to get size in Bytes). - size_t nLiterals = 0; // Number of samples that were stored as literals. - uint8_t messageWordSize = 0; // size in Bytes of a symbol in the encoded message. - uint8_t coderType = 0; // what type of CTF Coder is used? (32 vs 64 bit coders). - uint8_t streamSize = 0; // how many Bytes is the rANS encoder emmiting during a stream-out step. - uint8_t probabilityBits = 0; // The encoder renormed the distribution of source symbols to sum up to 2^probabilityBits. - OptStore opt = OptStore::EENCODE; // The type of storage operation that was conducted. - int32_t min = 0; // min symbol of the source dataset. - int32_t max = 0; // max symbol of the source dataset. - int32_t literalsPackingOffset = 0; // Offset from 0 used for bit packing of literal (incompressible) symbols. only used by rANS version >=1. - uint8_t literalsPackingWidth = 0; // Amount of bits used to pack literal (incompressible) symbols. only used by rANS version >=1. - int nDictWords = 0; // Amount of words used to store the encoding dictionary. - int nDataWords = 0; // Amount of words used to store the actual data. - int nLiteralWords = 0; // Amount of words used to store literal (incompressible) samples. - - size_t getUncompressedSize() const { return messageLength * messageWordSize; } - size_t getCompressedSize() const { return (nDictWords + nDataWords + nLiteralWords) * streamSize; } - void clear() - { - nStreams = 0; - messageLength = 0; - nLiterals = 0; - messageWordSize = 0; - coderType = 0; - streamSize = 0; - probabilityBits = 0; - min = 0; - max = 0; - literalsPackingOffset = 0; - literalsPackingWidth = 0; - nDictWords = 0; - nDataWords = 0; - nLiteralWords = 0; - } - ClassDefNV(Metadata, 3); -}; - -namespace detail -{ - -template -[[nodiscard]] inline constexpr Metadata makeMetadataRansCompat(size_t nStreams, size_t messageLength, - size_t nLiterals, size_t symbolTablePrecision, - source_T min, source_T max, size_t dictWords, - size_t dataWords, size_t literalWords) noexcept -{ - return Metadata{ - static_cast(nStreams), - messageLength, - nLiterals, - static_cast(sizeof(source_T)), - static_cast(sizeof(state_T)), - static_cast(sizeof(stream_T)), - static_cast(symbolTablePrecision), - Metadata::OptStore::EENCODE, - static_cast(min), - static_cast(max), - static_cast(0), - static_cast(sizeof(source_T)), - static_cast(dictWords), - static_cast(dataWords), - static_cast(literalWords)}; -}; - -template -[[nodiscard]] inline constexpr Metadata makeMetadataRansDict(size_t symbolTablePrecision, source_T min, - source_T max, size_t dictWords, ctf::Metadata::OptStore optStore) noexcept -{ - return Metadata{ - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(sizeof(source_T)), - static_cast(0), - static_cast(0), - static_cast(symbolTablePrecision), - optStore, - static_cast(min), - static_cast(max), - static_cast(0), - static_cast(0), - static_cast(dictWords), - static_cast(0), - static_cast(0)}; -}; - -template -[[nodiscard]] inline constexpr Metadata makeMetadataRansV1(size_t nStreams, size_t streamingLowerBound, size_t messageLength, - size_t nLiterals, size_t symbolTablePrecision, - source_T dictMin, source_T dictMax, - source_T literalsOffset, size_t literalsPackingWidth, size_t dictWords, - size_t dataWords, size_t literalWords) noexcept -{ - return Metadata{ - static_cast(nStreams), - messageLength, - nLiterals, - static_cast(sizeof(source_T)), - static_cast(sizeof(state_T)), - static_cast(streamingLowerBound), - static_cast(symbolTablePrecision), - Metadata::OptStore::EENCODE, - static_cast(dictMin), - static_cast(dictMax), - static_cast(literalsOffset), - static_cast(literalsPackingWidth), - static_cast(dictWords), - static_cast(dataWords), - static_cast(literalWords)}; -}; - -template -[[nodiscard]] inline constexpr Metadata makeMetadataPack(size_t messageLength, size_t packingWidth, - source_T packingOffset, size_t dataWords) noexcept -{ - return Metadata{ - static_cast(1), - messageLength, - static_cast(0), - static_cast(sizeof(source_T)), - static_cast(0), - static_cast(0), - static_cast(packingWidth), - Metadata::OptStore::PACK, - static_cast(packingOffset), - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(dataWords), - static_cast(0)}; -}; - -template -[[nodiscard]] inline constexpr Metadata makeMetadataStore(size_t messageLength, Metadata::OptStore opStore, size_t dataWords) noexcept -{ - return Metadata{ - static_cast(0), - messageLength, - static_cast(0), - static_cast(sizeof(source_T)), - static_cast(0), - static_cast(sizeof(buffer_T)), - static_cast(0), - opStore, - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(0), - static_cast(dataWords), - static_cast(0)}; -}; - -} // namespace detail -} // namespace o2::ctf - -#endif \ No newline at end of file diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h deleted file mode 100644 index b2cbed1cd27f9..0000000000000 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h +++ /dev/null @@ -1,120 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file CTFEntropyCoder.h -/// \author michael.lettrich@cern.ch -/// \brief Interface for externally provided rANS entropy coders - -#ifndef ALICEO2_EXTERNALENTROPYCODER_H_ -#define ALICEO2_EXTERNALENTROPYCODER_H_ - -#include - -#include "DetectorsCommonDataFormats/internal/Packer.h" - -#include "rANS/encode.h" -#include "rANS/factory.h" -#include "rANS/histogram.h" -#include "rANS/metrics.h" -#include "rANS/serialize.h" - -namespace o2::ctf::internal -{ - -template -class ExternalEntropyCoder -{ - public: - using source_type = source_T; - using encoder_type = typename rans::denseEncoder_type; - using metrics_type = rans::Metrics; - - ExternalEntropyCoder(const encoder_type& encoder); - - [[nodiscard]] inline const encoder_type& getEncoder() const noexcept { return *mEncoder; }; - - template - [[nodiscard]] inline size_t computePayloadSizeEstimate(size_t nElements, double_t safetyFactor = 1); - - template - [[nodiscard]] dst_IT encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd); - - [[nodiscard]] inline size_t getNIncompressibleSamples() const noexcept { return mIncompressibleBuffer.size(); }; - - [[nodiscard]] inline source_type getIncompressibleSymbolOffset() const noexcept { return mIncompressiblePacker.getOffset(); }; - - [[nodiscard]] inline size_t getIncompressibleSymbolPackingBits() const noexcept { return mIncompressiblePacker.getPackingWidth(); }; - - template - [[nodiscard]] size_t computePackedIncompressibleSize() const noexcept; - - template - [[nodiscard]] dst_T* writeIncompressible(dst_T* dstBegin, dst_T* dstEnd) const; - - private: - const encoder_type* mEncoder{}; - std::vector mIncompressibleBuffer{}; - Packer mIncompressiblePacker{}; -}; - -template -ExternalEntropyCoder::ExternalEntropyCoder(const encoder_type& encoder) : mEncoder{&encoder} -{ - if (!getEncoder().getSymbolTable().hasEscapeSymbol()) { - throw std::runtime_error("External entropy encoder must be able to handle incompressible symbols."); - } -}; - -template -template -[[nodiscard]] inline size_t ExternalEntropyCoder::computePayloadSizeEstimate(size_t nElements, double_t safetyFactor) -{ - constexpr size_t Overhead = 10 * rans::utils::pow2(10); // 10KB overhead safety margin - const double_t RelativeSafetyFactor = 2.0 * safetyFactor; - const size_t messageSizeB = nElements * sizeof(source_type); - return rans::utils::nBytesTo(std::ceil(safetyFactor * messageSizeB) + Overhead); -} - -template -template -[[nodiscard]] dst_IT ExternalEntropyCoder::encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd) -{ - const size_t incompressibleSymbolFrequency = [&]() { - const auto& symbolTable = mEncoder->getSymbolTable(); - const double_t incompressibleSymbolProbability = static_cast(symbolTable.getEscapeSymbol().getFrequency()) / rans::utils::pow2(symbolTable.getPrecision()); - return std::ceil(std::distance(srcBegin, srcEnd) * incompressibleSymbolProbability); - }(); - - mIncompressibleBuffer.reserve(incompressibleSymbolFrequency); - auto [encodedMessageEnd, literalsEnd] = mEncoder->process(srcBegin, srcEnd, dstBegin, std::back_inserter(mIncompressibleBuffer)); - rans::utils::checkBounds(encodedMessageEnd, dstEnd); - mIncompressiblePacker = Packer{mIncompressibleBuffer.data(), mIncompressibleBuffer.data() + mIncompressibleBuffer.size()}; - - return encodedMessageEnd; -}; - -template -template -[[nodiscard]] inline size_t ExternalEntropyCoder::computePackedIncompressibleSize() const noexcept -{ - return mIncompressiblePacker.template getPackingBufferSize(mIncompressibleBuffer.size()); -}; - -template -template -[[nodiscard]] inline dst_T* ExternalEntropyCoder::writeIncompressible(dst_T* dstBegin, dst_T* dstEnd) const -{ - return mIncompressiblePacker.pack(mIncompressibleBuffer.data(), mIncompressibleBuffer.size(), dstBegin, dstEnd); -}; - -} // namespace o2::ctf::internal - -#endif /* ALICEO2_EXTERNALENTROPYCODER_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h deleted file mode 100644 index 54302d51d2666..0000000000000 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file CTFEntropyCoder.h -/// \author michael.lettrich@cern.ch -/// \brief ANS Entropy Coding and packing specialization for CTF Coders - -#ifndef ALICEO2_INPLACEENTROPYCODER_H_ -#define ALICEO2_INPLACEENTROPYCODER_H_ - -#include -#include -#include - -#include "DetectorsCommonDataFormats/internal/Packer.h" - -#include "rANS/encode.h" -#include "rANS/factory.h" -#include "rANS/histogram.h" -#include "rANS/metrics.h" -#include "rANS/serialize.h" - -namespace o2::ctf::internal -{ - -template -class InplaceEntropyCoder -{ - using dense_histogram_type = rans::DenseHistogram; - using adaptive_histogram_type = rans::AdaptiveHistogram; - using sparse_histogram_type = rans::SparseHistogram; - - using dense_encoder_type = rans::denseEncoder_type; - using adaptive_encoder_type = rans::adaptiveEncoder_type; - using sparse_encoder_type = rans::sparseEncoder_type; - - using dict_buffer_type = std::vector; - - public: - using source_type = source_T; - using metrics_type = rans::Metrics; - using packer_type = Packer; - using histogram_type = std::variant; - using encoder_type = std::variant; - using incompressible_buffer_type = std::vector; - - InplaceEntropyCoder() = default; - - template - InplaceEntropyCoder(source_IT srcBegin, source_IT srcEnd); - - template - InplaceEntropyCoder(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max); - - void makeEncoder(); - - // getters - - [[nodiscard]] inline const metrics_type& getMetrics() const noexcept { return mMetrics; }; - - [[nodiscard]] inline size_t getNIncompressibleSamples() const noexcept { return mIncompressibleBuffer.size(); }; - - [[nodiscard]] size_t getNStreams() const; - - [[nodiscard]] size_t getSymbolTablePrecision() const; - - template - [[nodiscard]] size_t getPackedIncompressibleSize() const noexcept; - - // operations - template - [[nodiscard]] dst_IT encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd); - - template - [[nodiscard]] dst_IT writeDictionary(dst_IT dstBegin, dst_IT dstEnd); - - template - [[nodiscard]] dst_T* writeIncompressible(dst_T* dstBegin, dst_T* dstEnd); - - private: - template ::value_type) < 4), bool> = true> - void init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max); - - template ::value_type) == 4), bool> = true> - void init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max); - - template ::value_type) < 4), bool> = true> - void init(source_IT srcBegin, source_IT srcEnd); - - template ::value_type) == 4), bool> = true> - void init(source_IT srcBegin, source_IT srcEnd); - - template - void serializeDictionary(const container_T&); - - std::optional mHistogram{}; - metrics_type mMetrics{}; - std::optional mEncoder{}; - incompressible_buffer_type mIncompressibleBuffer{}; - dict_buffer_type mDictBuffer{}; - packer_type mIncompressiblePacker{}; -}; - -template -template -InplaceEntropyCoder::InplaceEntropyCoder(src_IT srcBegin, src_IT srcEnd) -{ - static_assert(std::is_same_v::value_type>); - - const size_t nSamples = std::distance(srcBegin, srcEnd); - if constexpr (std::is_pointer_v) { - if (sizeof(source_type) > 2 && nSamples > 0) { - const auto [min, max] = rans::internal::minmax(gsl::span(srcBegin, srcEnd)); - init(srcBegin, srcEnd, min, max); - } else { - init(srcBegin, srcEnd); - } - } else { - init(srcBegin, srcEnd); - } - - mIncompressiblePacker = Packer(mMetrics); -}; - -template -template -InplaceEntropyCoder::InplaceEntropyCoder(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max) -{ - static_assert(std::is_same_v::value_type>); - init(srcBegin, srcEnd, min, max); - mIncompressiblePacker = Packer(mMetrics); -}; - -template -[[nodiscard]] inline size_t InplaceEntropyCoder::getNStreams() const -{ - size_t nStreams{}; - std::visit([&, this](auto&& encoder) { nStreams = encoder.getNStreams(); }, *mEncoder); - return nStreams; -} - -template -[[nodiscard]] inline size_t InplaceEntropyCoder::getSymbolTablePrecision() const -{ - size_t precision{}; - std::visit([&, this](auto&& encoder) { precision = encoder.getSymbolTable().getPrecision(); }, *mEncoder); - return precision; -} - -template -void InplaceEntropyCoder::makeEncoder() -{ - std::visit([this](auto&& histogram) { - auto renormed = rans::renorm(std::move(histogram), mMetrics); - - if (std::holds_alternative(*mHistogram)) { - serializeDictionary(renormed); - } - - const size_t rangeBits = rans::utils::getRangeBits(*mMetrics.getCoderProperties().min, *mMetrics.getCoderProperties().max); - const size_t nSamples = mMetrics.getDatasetProperties().numSamples; - const size_t nUsedAlphabetSymbols = mMetrics.getDatasetProperties().nUsedAlphabetSymbols; - - if (rangeBits <= 18) { - // dense symbol tables if they fit into cache, or source data covers the range of the alphabet well - mEncoder = encoder_type{std::in_place_type, renormed}; - } else if (nUsedAlphabetSymbols < rans::utils::pow2(14)) { - // sparse symbol table makes sense if it fits into L3 Cache - mEncoder = encoder_type{std::in_place_type, renormed}; - } else { - // adaptive symbol table otherwise - mEncoder = encoder_type{std::in_place_type, renormed}; - } - }, - *mHistogram); -}; - -template -template -[[nodiscard]] dst_IT InplaceEntropyCoder::encode(src_IT srcBegin, src_IT srcEnd, dst_IT dstBegin, dst_IT dstEnd) -{ - static_assert(std::is_same_v::value_type>); - - dst_IT messageEnd = dstBegin; - - std::visit([&, this](auto&& encoder) { - if (encoder.getSymbolTable().hasEscapeSymbol()) { - mIncompressibleBuffer.reserve(*mMetrics.getCoderProperties().nIncompressibleSamples); - auto [encodedMessageEnd, literalsEnd] = encoder.process(srcBegin, srcEnd, dstBegin, std::back_inserter(mIncompressibleBuffer)); - messageEnd = encodedMessageEnd; - } else { - messageEnd = encoder.process(srcBegin, srcEnd, dstBegin); - } - rans::utils::checkBounds(messageEnd, dstEnd); - }, - *mEncoder); - - return messageEnd; -}; - -template -template -[[nodiscard]] inline dst_IT InplaceEntropyCoder::writeDictionary(dst_IT dstBegin, dst_IT dstEnd) -{ - static_assert(std::is_pointer_v); - - using dst_type = std::remove_pointer_t; - - dst_IT ret{}; - if (mDictBuffer.empty()) { - std::visit([&, this](auto&& encoder) { ret = rans::compressRenormedDictionary(encoder.getSymbolTable(), dstBegin); }, *mEncoder); - } else { - // copy - std::memcpy(dstBegin, mDictBuffer.data(), mDictBuffer.size()); - - // determine location of end - auto end = reinterpret_cast(dstBegin) + mDictBuffer.size(); - // realign pointer - constexpr size_t alignment = std::alignment_of_v; - end += (alignment - reinterpret_cast(end) % alignment) % alignment; - // and convert it back to ret - ret = reinterpret_cast(end); - } - - rans::utils::checkBounds(ret, dstEnd); - return ret; -}; - -template -template -inline dst_T* InplaceEntropyCoder::writeIncompressible(dst_T* dstBegin, dst_T* dstEnd) -{ - return mIncompressiblePacker.pack(mIncompressibleBuffer.data(), mIncompressibleBuffer.size(), dstBegin, dstEnd); -}; - -template -template -[[nodiscard]] inline size_t InplaceEntropyCoder::getPackedIncompressibleSize() const noexcept -{ - return mIncompressiblePacker.template getPackingBufferSize(getNIncompressibleSamples()); -} - -template -template ::value_type) < 4), bool>> -void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max) -{ - mHistogram.emplace(histogram_type{rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd)}); - mMetrics = metrics_type{std::get(*mHistogram), min, max}; -}; - -template -template ::value_type) == 4), bool>> -void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd, source_type min, source_type max) -{ - const size_t nSamples = std::distance(srcBegin, srcEnd); - const size_t rangeBits = rans::utils::getRangeBits(min, max); - - if ((rangeBits <= 18) || ((nSamples / rans::utils::pow2(rangeBits)) >= 0.80)) { - // either the range of source symbols is distrubuted such that it fits into L3 Cache - // Or it is possible for the data to cover a very significant fraction of the total [min,max] range - mHistogram = histogram_type{std::in_place_type, rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd, min, max)}; - mMetrics = metrics_type{std::get(*mHistogram), min, max}; - } else if (nSamples / rans::utils::pow2(rangeBits) <= 0.3) { - // or the range of source symbols is spread very thinly accross a large range - mHistogram = histogram_type{std::in_place_type, rans::makeSparseHistogram::fromSamples(srcBegin, srcEnd)}; - mMetrics = metrics_type{std::get(*mHistogram), min, max}; - } else { - // no strong evidence of either extreme case - mHistogram = histogram_type{std::in_place_type, rans::makeAdaptiveHistogram::fromSamples(srcBegin, srcEnd)}; - mMetrics = metrics_type{std::get(*mHistogram), min, max}; - } -}; - -template -template ::value_type) < 4), bool>> -void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd) -{ - mHistogram = histogram_type{std::in_place_type, rans::makeDenseHistogram::fromSamples(srcBegin, srcEnd)}; - mMetrics = metrics_type{std::get(*mHistogram)}; -}; - -template -template ::value_type) == 4), bool>> -void InplaceEntropyCoder::init(source_IT srcBegin, source_IT srcEnd) -{ - mHistogram = histogram_type{std::in_place_type, rans::makeSparseHistogram::fromSamples(srcBegin, srcEnd)}; - mMetrics = metrics_type{std::get(*mHistogram)}; -}; - -template -template -void InplaceEntropyCoder::serializeDictionary(const container_T& renormedHistogram) -{ - - mDictBuffer.resize(mMetrics.getSizeEstimate().getCompressedDictionarySize(), 0); - auto end = rans::compressRenormedDictionary(renormedHistogram, mDictBuffer.data()); - rans::utils::checkBounds(end, mDictBuffer.data() + mDictBuffer.size()); - mDictBuffer.resize(std::distance(mDictBuffer.data(), end)); - - assert(mDictBuffer.size() > 0); -}; - -} // namespace o2::ctf::internal - -#endif /* ALICEO2_INPLACEENTROPYCODER_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h b/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h deleted file mode 100644 index 72f62e3245797..0000000000000 --- a/DataFormats/Detectors/Common/include/DetectorsCommonDataFormats/internal/Packer.h +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// \file Packer.h -/// \author michael.lettrich@cern.ch -/// \brief Interfaces for BitPacking using librans - -#ifndef ALICEO2_PACKER_H_ -#define ALICEO2_PACKER_H_ - -#include "rANS/pack.h" -#include "rANS/metrics.h" - -namespace o2::ctf::internal -{ - -template -class Packer -{ - public: - using source_type = source_T; - - Packer() = default; - - explicit Packer(rans::Metrics& metrics) : mOffset{metrics.getDatasetProperties().min}, - mPackingWidth{metrics.getDatasetProperties().alphabetRangeBits} {}; - - template - Packer(source_IT srcBegin, source_IT srcEnd); - - [[nodiscard]] inline source_type getOffset() const noexcept { return mOffset; }; - - [[nodiscard]] inline size_t getPackingWidth() const noexcept { return mPackingWidth; }; - - template - [[nodiscard]] size_t getPackingBufferSize(size_t messageLength) const noexcept; - - template - [[nodiscard]] dst_T* pack(source_IT srcBegin, source_IT srcEnd, dst_T* dstBegin, dst_T* dstEnd) const; - - template - [[nodiscard]] dst_T* pack(const source_T* __restrict srcBegin, size_t extent, dst_T* dstBegin, dst_T* dstEnd) const; - - private: - source_type mOffset{}; - size_t mPackingWidth{}; -}; - -template -template -Packer::Packer(source_IT srcBegin, source_IT srcEnd) -{ - static_assert(rans::utils::isCompatibleIter_v); - if (srcBegin != srcEnd) { - - const auto [min, max] = [&]() { - if constexpr (std::is_pointer_v) { - return rans::utils::minmax(gsl::span(srcBegin, srcEnd)); - } else { - const auto [minIter, maxIter] = std::minmax_element(srcBegin, srcEnd); - return std::make_pair(*minIter, *maxIter); - } - }(); - - mOffset = min; - mPackingWidth = rans::utils::getRangeBits(min, max); - } -}; - -template -template -[[nodiscard]] inline size_t Packer::getPackingBufferSize(size_t messageLength) const noexcept -{ - return rans::computePackingBufferSize(messageLength, mPackingWidth); -}; - -template -template -[[nodiscard]] inline dst_T* Packer::pack(const source_T* __restrict srcBegin, size_t extent, dst_T* dstBegin, dst_T* dstEnd) const -{ - return pack(srcBegin, srcBegin + extent, dstBegin, dstEnd); -} - -template -template -[[nodiscard]] dst_T* Packer::pack(source_IT srcBegin, source_IT srcEnd, dst_T* dstBegin, dst_T* dstEnd) const -{ - static_assert(std::is_same_v::value_type>); - size_t extent = std::distance(srcBegin, srcEnd); - - if (extent == 0) { - return dstBegin; - } - - rans::BitPtr packEnd = rans::pack(srcBegin, extent, dstBegin, mPackingWidth, mOffset); - auto* end = packEnd.toPtr(); - ++end; // one past end iterator; - rans::utils::checkBounds(end, dstEnd); - return end; -}; - -} // namespace o2::ctf::internal - -#endif /* ALICEO2_PACKER_H_ */ \ No newline at end of file diff --git a/DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx b/DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx deleted file mode 100644 index 3471a6c9e4dff..0000000000000 --- a/DataFormats/Detectors/Common/test/testCTFEntropyCoder.cxx +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright 2019-2020 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file testCTFEntropyCoder -/// @author Michael Lettrich -/// @brief Test entropy coding using rANS algorithm - -#define BOOST_TEST_MODULE Test CTFEntropyCoder class -#define BOOST_TEST_MAIN -#define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "DetectorsCommonDataFormats/internal/Packer.h" -#include "DetectorsCommonDataFormats/internal/ExternalEntropyCoder.h" -#include "DetectorsCommonDataFormats/internal/InplaceEntropyCoder.h" -#include "rANS/histogram.h" -#include "rANS/metrics.h" -#include "rANS/factory.h" -#include "rANS/iterator.h" - -using namespace o2; - -using buffer_type = uint32_t; -using source_types = boost::mp11::mp_list; - -template -class SourceMessage -{ - public: - SourceMessage(size_t messageSize, source_T max = std::numeric_limits::max(), source_T min = std::numeric_limits::min()) : mMin{min}, mMax{max} - { - if (mSourceMessage.empty()) { - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - assert(max >= min); - const size_t draws = (max - min) + 1; - const double probability = 0.5; - std::binomial_distribution dist(draws, probability); - mSourceMessage.resize(messageSize); - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt, min]() -> source_T { return static_cast(dist(mt)) + min; }); - } - } - - inline constexpr source_T getMin() const noexcept { return mMin; }; - inline constexpr source_T getMax() const noexcept { return mMax; }; - inline constexpr auto& get() const noexcept { return mSourceMessage; }; - - private: - source_T mMin{}; - source_T mMax{}; - std::vector mSourceMessage{}; -}; - -class SourceMessageProxy -{ - public: - SourceMessageProxy() = default; - - template - const auto& getMessage() const noexcept - { - if constexpr (std::is_same_v) { - return sourceMessage8u.get(); - } else if constexpr (std::is_same_v) { - return sourceMessage8.get(); - } else if constexpr (std::is_same_v) { - return sourceMessage16u.get(); - } else if constexpr (std::is_same_v) { - return sourceMessage16.get(); - } else if constexpr (std::is_same_v) { - return sourceMessage32u.get(); - } else if constexpr (std::is_same_v) { - return sourceMessage32.get(); - } else { - throw std::runtime_error{"unsupported source type"}; - } - }; - - private: - inline static constexpr size_t MessageSize = rans::utils::pow2(10); - SourceMessage sourceMessage8u{MessageSize}; - SourceMessage sourceMessage8{MessageSize}; - SourceMessage sourceMessage16u{MessageSize}; - SourceMessage sourceMessage16{MessageSize}; - SourceMessage sourceMessage32u{MessageSize, rans::utils::pow2(27)}; - SourceMessage sourceMessage32{MessageSize, rans::utils::pow2(26), -static_cast(rans::utils::pow2(26))}; -}; - -inline const SourceMessageProxy MessageProxy{}; - -template -void encodeInplace(source_IT begin, source_IT end) -{ - using source_type = typename std::iterator_traits::value_type; - - ctf::internal::InplaceEntropyCoder entropyCoder{begin, end}; - // BOOST_CHECK_THROW(entropyCoder.getEncoder(), std::runtime_error); - entropyCoder.makeEncoder(); - - const rans::Metrics& metrics = entropyCoder.getMetrics(); - const rans::SizeEstimate sizeEstimate = metrics.getSizeEstimate(); - - LOGP(info, "dataset[{},{}], coder[{},{}]", metrics.getDatasetProperties().min, metrics.getDatasetProperties().max, *metrics.getCoderProperties().min, *metrics.getCoderProperties().max); - - std::vector encodeBuffer(sizeEstimate.getCompressedDatasetSize(), 0); - std::vector literalSymbolsBuffer(sizeEstimate.getIncompressibleSize(), 0); - std::vector dictBuffer(sizeEstimate.getCompressedDictionarySize(), 0); - - auto encoderEnd = entropyCoder.encode(begin, end, encodeBuffer.data(), encodeBuffer.data() + encodeBuffer.size()); - auto literalsEnd = entropyCoder.writeIncompressible(literalSymbolsBuffer.data(), literalSymbolsBuffer.data() + literalSymbolsBuffer.size()); - auto dictEnd = entropyCoder.writeDictionary(dictBuffer.data(), dictBuffer.data() + dictBuffer.size()); - // decode - const auto& coderProperties = metrics.getCoderProperties(); - auto decoder = rans::makeDecoder<>::fromRenormed(rans::readRenormedDictionary(dictBuffer.data(), dictEnd, - *coderProperties.min, *coderProperties.max, - *coderProperties.renormingPrecisionBits)); - std::vector literals(entropyCoder.getNIncompressibleSamples()); - - const auto& datasetPropterties = metrics.getDatasetProperties(); - rans::unpack(literalSymbolsBuffer.data(), literals.size(), literals.data(), - datasetPropterties.alphabetRangeBits, datasetPropterties.min); - - size_t messageLength = std::distance(begin, end); - std::vector sourceBuffer(messageLength, 0); - - decoder.process(encoderEnd, sourceBuffer.data(), messageLength, entropyCoder.getNStreams(), literals.end()); - - BOOST_CHECK_EQUAL_COLLECTIONS(sourceBuffer.begin(), sourceBuffer.end(), begin, end); -}; - -BOOST_AUTO_TEST_CASE_TEMPLATE(testInplaceEncoderEmpty, source_T, source_types) -{ - std::vector testMessage{}; - encodeInplace(testMessage.data(), testMessage.data() + testMessage.size()); -}; - -BOOST_AUTO_TEST_CASE_TEMPLATE(testInplaceEncoderPTR, source_T, source_types) -{ - const auto& testMessage = MessageProxy.getMessage(); - encodeInplace(testMessage.data(), testMessage.data() + testMessage.size()); -}; - -BOOST_AUTO_TEST_CASE_TEMPLATE(testInplaceEncoderIter, source_T, source_types) -{ - const auto& testMessage = MessageProxy.getMessage(); - encodeInplace(testMessage.begin(), testMessage.end()); -}; - -template -class ShiftFunctor - -{ - public: - template - inline value_T operator()(iterA_T iterA, iterB_T iterB) const - { - return *iterB + (static_cast(*iterA) << shift); - }; - - template - inline void operator()(iterA_T iterA, iterB_T iterB, value_T value) const - { - *iterA = value >> shift; - *iterB = value & ((0x1 << shift) - 0x1); - }; -}; - -template -auto makeInputIterators(iterA_T iterA, iterB_T iterB, size_t nElements, F functor) -{ - using namespace o2::rans::utils; - - return std::make_tuple(rans::CombinedInputIterator{iterA, iterB, functor}, - rans::CombinedInputIterator{advanceIter(iterA, nElements), advanceIter(iterB, nElements), functor}); -}; - -BOOST_AUTO_TEST_CASE(testInplaceEncoderCombinedIterator) -{ - - const auto& testMessage1 = MessageProxy.getMessage(); - const auto& testMessage2 = MessageProxy.getMessage(); - - auto [begin, end] = makeInputIterators(testMessage1.data(), testMessage2.data(), testMessage1.size(), ShiftFunctor()>{}); - - encodeInplace(begin, end); -}; - -class ExternalEncoderDecoderProxy -{ - public: - ExternalEncoderDecoderProxy() - { - SourceMessageProxy proxy{}; - - auto renormed8u = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); - auto renormed8 = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); - auto renormed16u = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); - auto renormed16 = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); - auto renormed32u = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); - auto renormed32 = rans::renorm(rans::makeDenseHistogram::fromSamples(proxy.getMessage().begin(), proxy.getMessage().end()), rans::RenormingPolicy::ForceIncompressible); - - encoder8u = rans::makeDenseEncoder<>::fromRenormed(renormed8u); - encoder8 = rans::makeDenseEncoder<>::fromRenormed(renormed8); - encoder16u = rans::makeDenseEncoder<>::fromRenormed(renormed16u); - encoder16 = rans::makeDenseEncoder<>::fromRenormed(renormed16); - encoder32u = rans::makeDenseEncoder<>::fromRenormed(renormed32u); - encoder32 = rans::makeDenseEncoder<>::fromRenormed(renormed32); - - decoder8u = rans::makeDecoder<>::fromRenormed(renormed8u); - decoder8 = rans::makeDecoder<>::fromRenormed(renormed8); - decoder16u = rans::makeDecoder<>::fromRenormed(renormed16u); - decoder16 = rans::makeDecoder<>::fromRenormed(renormed16); - decoder32u = rans::makeDecoder<>::fromRenormed(renormed32u); - decoder32 = rans::makeDecoder<>::fromRenormed(renormed32); - } - - template - const auto& getEncoder() const noexcept - { - if constexpr (std::is_same_v) { - return encoder8u; - } else if constexpr (std::is_same_v) { - return encoder8; - } else if constexpr (std::is_same_v) { - return encoder16u; - } else if constexpr (std::is_same_v) { - return encoder16; - } else if constexpr (std::is_same_v) { - return encoder32u; - } else if constexpr (std::is_same_v) { - return encoder32; - } else { - throw std::runtime_error{"unsupported encoder type"}; - } - }; - - template - const auto& getDecoder() const noexcept - { - if constexpr (std::is_same_v) { - return decoder8u; - } else if constexpr (std::is_same_v) { - return decoder8; - } else if constexpr (std::is_same_v) { - return decoder16u; - } else if constexpr (std::is_same_v) { - return decoder16; - } else if constexpr (std::is_same_v) { - return decoder32u; - } else if constexpr (std::is_same_v) { - return decoder32; - } else { - throw std::runtime_error{"unsupported encoder type"}; - } - }; - - private: - rans::denseEncoder_type encoder8u{}; - rans::denseEncoder_type encoder8{}; - rans::denseEncoder_type encoder16u{}; - rans::denseEncoder_type encoder16{}; - rans::denseEncoder_type encoder32u{}; - rans::denseEncoder_type encoder32{}; - - rans::defaultDecoder_type decoder8u{}; - rans::defaultDecoder_type decoder8{}; - rans::defaultDecoder_type decoder16u{}; - rans::defaultDecoder_type decoder16{}; - rans::defaultDecoder_type decoder32u{}; - rans::defaultDecoder_type decoder32{}; -}; - -ExternalEncoderDecoderProxy ExternalEncoders{}; - -template -void encodeExternal(source_IT begin, source_IT end) -{ - using source_type = typename std::iterator_traits::value_type; - - ctf::internal::ExternalEntropyCoder entropyCoder{ExternalEncoders.getEncoder()}; - - const size_t sourceExtent = std::distance(begin, end); - std::vector encodeBuffer(entropyCoder.template computePayloadSizeEstimate(sourceExtent), 0); - auto encoderEnd = entropyCoder.encode(begin, end, encodeBuffer.data(), encodeBuffer.data() + encodeBuffer.size()); - - std::vector literalSymbolsBuffer(entropyCoder.template computePackedIncompressibleSize(), 0); - auto literalsEnd = entropyCoder.writeIncompressible(literalSymbolsBuffer.data(), literalSymbolsBuffer.data() + literalSymbolsBuffer.size()); - - // decode - auto decoder = ExternalEncoders.getDecoder(); - std::vector literals((entropyCoder.getNIncompressibleSamples())); - - rans::unpack(literalSymbolsBuffer.data(), literals.size(), literals.data(), - entropyCoder.getIncompressibleSymbolPackingBits(), entropyCoder.getIncompressibleSymbolOffset()); - - size_t messageLength = std::distance(begin, end); - std::vector sourceBuffer(messageLength, 0); - - decoder.process(encoderEnd, sourceBuffer.data(), messageLength, entropyCoder.getEncoder().getNStreams(), literals.end()); - - BOOST_CHECK_EQUAL_COLLECTIONS(sourceBuffer.begin(), sourceBuffer.end(), begin, end); -}; - -BOOST_AUTO_TEST_CASE_TEMPLATE(testExternalEncoderEmpty, source_T, source_types) -{ - std::vector testMessage{}; - encodeExternal(testMessage.data(), testMessage.data() + testMessage.size()); -}; - -BOOST_AUTO_TEST_CASE_TEMPLATE(testExternalEncoderPTR, source_T, source_types) -{ - const auto& testMessage = MessageProxy.getMessage(); - encodeExternal(testMessage.data(), testMessage.data() + testMessage.size()); -}; - -BOOST_AUTO_TEST_CASE_TEMPLATE(testExternalEncoderIter, source_T, source_types) -{ - const auto& testMessage = MessageProxy.getMessage(); - encodeExternal(testMessage.begin(), testMessage.end()); -}; - -BOOST_AUTO_TEST_CASE(testExternalEncoderCombinedIterator) -{ - - const auto& testMessage1 = MessageProxy.getMessage(); - const auto& testMessage2 = MessageProxy.getMessage(); - - auto [begin, end] = makeInputIterators(testMessage1.data(), testMessage2.data(), testMessage1.size(), ShiftFunctor()>{}); - - encodeExternal(begin, end); -}; \ No newline at end of file diff --git a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h index 594f924e16598..3c1df8d89ee4d 100644 --- a/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h +++ b/DataFormats/Detectors/FIT/FDD/include/DataFormatsFDD/CTF.h @@ -41,8 +41,8 @@ struct CompressedDigits { // BC data std::vector trigger; // trigger bits - std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc - std::vector orbitInc; // increment in orbit + std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc + std::vector orbitInc; // increment in orbit std::vector nChan; // number of fired channels // channel data diff --git a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h index 3ddea05158577..fdc073edfd5c1 100644 --- a/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h +++ b/DataFormats/Detectors/FIT/FT0/include/DataFormatsFT0/CTF.h @@ -41,8 +41,8 @@ struct CompressedDigits { // trigger data std::vector trigger; // trigger bits - std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc - std::vector orbitInc; // increment in orbit + std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc + std::vector orbitInc; // increment in orbit std::vector nChan; // number of fired channels std::vector eventStatus; // special flags about event conditions: pile-up, not use for collision time, not use for event plane, etc. diff --git a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h index 275b682a15276..e71e42ca2f4ad 100644 --- a/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h +++ b/DataFormats/Detectors/FIT/FV0/include/DataFormatsFV0/CTF.h @@ -41,8 +41,8 @@ struct CompressedDigits { // trigger data std::vector trigger; // trigger bits - std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc - std::vector orbitInc; // increment in orbit + std::vector bcInc; // increment in BC if the same orbit, otherwise abs bc + std::vector orbitInc; // increment in orbit std::vector nChan; // number of fired channels // channel data diff --git a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h index 314523aa878ba..b465aa5b3d66b 100644 --- a/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h +++ b/DataFormats/Detectors/ITSMFT/common/include/DataFormatsITSMFT/CTF.h @@ -46,12 +46,12 @@ struct CompressedClusters { // ROF header data std::vector firstChipROF; /// 1st chip ID in the ROF - std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF - std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF + std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF + std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF std::vector nclusROF; /// number of clusters in ROF // Chip data - std::vector chipInc; /// increment of chipID wrt that of prev. chip + std::vector chipInc; /// increment of chipID wrt that of prev. chip std::vector chipMul; /// clusters in chip std::vector row; /// row of fired pixel std::vector colInc; /// increment of pixel column wrt that of prev. pixel (sometimes can be slightly negative) diff --git a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h index a31ece315d408..57d5784dc3b66 100644 --- a/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h +++ b/DataFormats/Detectors/TOF/include/DataFormatsTOF/CTF.h @@ -56,15 +56,15 @@ struct CompressedInfos { */ // ROF header data - std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF - std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF + std::vector bcIncROF; /// increment of ROF BC wrt BC of previous ROF + std::vector orbitIncROF; /// increment of ROF orbit wrt orbit of previous ROF std::vector ndigROF; /// number of digits in ROF std::vector ndiaROF; /// number of diagnostic/pattern words in ROF std::vector ndiaCrate; /// number of diagnostic/pattern words per crate in ROF // Hit data - std::vector timeFrameInc; /// time increment with respect of previous digit in TimeFrame units - std::vector timeTDCInc; /// time increment with respect of previous digit in TDC channel (about 24.4 ps) within timeframe + std::vector timeFrameInc; /// time increment with respect of previous digit in TimeFrame units + std::vector timeTDCInc; /// time increment with respect of previous digit in TDC channel (about 24.4 ps) within timeframe std::vector stripID; /// increment of stripID wrt that of prev. strip std::vector chanInStrip; /// channel in strip 0-95 (ordered in time) std::vector tot; /// Time-Over-Threshold in TOF channel (about 48.8 ps) diff --git a/Detectors/Base/include/DetectorsBase/CTFCoderBase.h b/Detectors/Base/include/DetectorsBase/CTFCoderBase.h index bf4f37ecbeff5..823d03279245f 100644 --- a/Detectors/Base/include/DetectorsBase/CTFCoderBase.h +++ b/Detectors/Base/include/DetectorsBase/CTFCoderBase.h @@ -26,15 +26,11 @@ #include "DetectorsCommonDataFormats/CTFHeader.h" #include "DetectorsCommonDataFormats/CTFIOSize.h" #include "DataFormatsCTP/TriggerOffsetsParam.h" -#include "DetectorsCommonDataFormats/ANSHeader.h" -#include "rANS/factory.h" -#include "rANS/compat.h" -#include "rANS/histogram.h" +#include "rANS/rans.h" #include #include "Framework/InitContext.h" #include "Framework/ConcreteDataMatcher.h" #include "Framework/ConfigParamRegistry.h" -#include namespace o2 { @@ -81,30 +77,19 @@ class CTFCoderBase void createCodersFromFile(const std::string& dictPath, o2::ctf::CTFCoderBase::OpType op, bool mayFail = false); template - void createCoder(OpType op, const o2::rans::RenormedDenseHistogram& renormedHistogram, int slot) + void createCoder(OpType op, const o2::rans::RenormedFrequencyTable& renormedFrequencyTable, int slot) { - LOG_IF(warning, renormedHistogram.empty()) << fmt::format("Empty dictionary provided for slot {}, {} will assume literal symbols only", slot, (op == OpType::Encoder ? "encoding" : "decoding")); - - if (mANSVersion == ANSVersionCompat) { - switch (op) { - case OpType::Encoder: - mCoders[slot] = std::make_any>(rans::compat::makeEncoder::fromRenormed(renormedHistogram)); - break; - case OpType::Decoder: - mCoders[slot] = std::make_any>(rans::compat::makeDecoder::fromRenormed(renormedHistogram)); - break; - } - } else if (mANSVersion == ANSVersion1) { - switch (op) { - case OpType::Encoder: - mCoders[slot] = std::make_any>(rans::makeDenseEncoder<>::fromRenormed(renormedHistogram)); - break; - case OpType::Decoder: - mCoders[slot] = std::make_any>(rans::makeDecoder<>::fromRenormed(renormedHistogram)); - break; - } - } else { - throw std::runtime_error("unsupported ANS version"); + if (renormedFrequencyTable.empty()) { + LOG(warning) << "Empty dictionary provided for slot " << slot << ", " << (op == OpType::Encoder ? "encoding" : "decoding") << " will assume literal symbols only"; + } + + switch (op) { + case OpType::Encoder: + mCoders[slot].reset(new o2::rans::LiteralEncoder64(renormedFrequencyTable)); + break; + case OpType::Decoder: + mCoders[slot].reset(new o2::rans::LiteralDecoder64(renormedFrequencyTable)); + break; } } @@ -143,9 +128,6 @@ class CTFCoderBase size_t getIRFrameSelMarginFwd() const { return mIRFrameSelMarginFwd; } long getIRFrameSelShift() const { return mIRFrameSelShift; } - inline const ctf::ANSHeader& getANSVersion() const noexcept { return mANSVersion; }; - inline ctf::ANSHeader& getANSVersion() { return const_cast(const_cast(*this).getANSVersion()); }; - inline void setANSVersion(const ctf::ANSHeader& ansVersion) noexcept { mANSVersion = ansVersion; }; void setBCShift(int64_t n) { mBCShift = n; } void setFirstTFOrbit(uint32_t n) { mFirstTFOrbit = n; } auto getBCShift() const { return mBCShift; } @@ -173,32 +155,19 @@ class CTFCoderBase } bool canApplyBCShift(const o2::InteractionRecord& ir) const { return canApplyBCShift(ir, mBCShift); } - template - [[nodiscard]] size_t estimateBufferSize(size_t slot, source_IT samplesBegin, source_IT samplesEnd); - - template - size_t estimateBufferSize(size_t slot, size_t nSamples); - - template - [[nodiscard]] size_t estimateBufferSize(size_t slot, const std::vector& samples) - { - return estimateBufferSize(slot, samples.begin(), samples.end()); - } - template std::vector loadDictionaryFromTree(TTree* tree); - std::vector mCoders; // encoders/decoders + std::vector> mCoders; // encoders/decoders DetID mDet; std::string mDictBinding{"ctfdict"}; std::string mTrigOffsBinding{"trigoffset"}; - CTFDictHeader mExtHeader; // external dictionary header + CTFDictHeader mExtHeader; // external dictionary header o2::utils::IRFrameSelector mIRFrameSelector; // optional IR frames selector - float mMemMarginFactor = 1.0f; // factor for memory allocation in EncodedBlocks + float mMemMarginFactor = 1.0f; // factor for memory allocation in EncodedBlocks bool mLoadDictFromCCDB{true}; bool mSupportBCShifts{false}; - OpType mOpType; // Encoder or Decoder - ctf::ANSHeader mANSVersion{ctf::ANSVersionCompat}; // Version of the ANSEncoder/Decoder - int64_t mBCShift = 0; // shift to apply to decoded IR (i.e. CTP offset if was not corrected on raw data decoding level) + OpType mOpType; // Encoder or Decoder + int64_t mBCShift = 0; // shift to apply to decoded IR (i.e. CTP offset if was not corrected on raw data decoding level) uint32_t mFirstTFOrbit = 0; size_t mIRFrameSelMarginBwd = 0; // margin in BC to add to the IRFrame lower boundary when selection is requested size_t mIRFrameSelMarginFwd = 0; // margin in BC to add to the IRFrame upper boundary when selection is requested @@ -313,18 +282,6 @@ void CTFCoderBase::init(o2::framework::InitContext& ic) if (ic.options().hasOption("irframe-shift")) { mIRFrameSelShift = (long)ic.options().get("irframe-shift"); } - if (ic.options().hasOption("ans-version")) { - if (ic.options().isSet("ans-version")) { - const std::string ansVersionString = ic.options().get("ans-version"); - if (!ansVersionString.empty()) { - mANSVersion = ansVersionFromString(ansVersionString); - LOGP(info, "parsing ansVersionString {} into {}", ansVersionString, static_cast(mANSVersion)); - if (mANSVersion == ANSVersionUnspecified) { - throw std::invalid_argument(fmt::format("Invalid ANS Version {}", ansVersionString)); - } - } - } - } auto dict = ic.options().get("ctf-dict"); if (dict.empty() || dict == "ccdb") { // load from CCDB mLoadDictFromCCDB = true; @@ -386,39 +343,6 @@ bool CTFCoderBase::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, voi return match; } -template -[[nodiscard]] inline size_t CTFCoderBase::estimateBufferSize(size_t slot, IT samplesBegin, IT samplesEnd) -{ - using source_type = typename std::iterator_traits::value_type; - const size_t nSamples = std::distance(samplesBegin, samplesEnd); - return estimateBufferSize(slot, nSamples); -}; - -template -[[nodiscard]] inline size_t CTFCoderBase::estimateBufferSize(size_t slot, size_t nSamples) -{ - - std::any& coder = mCoders[slot]; - if (coder.has_value()) { - const size_t alphabetRangeBits = [this, &coder]() { - if (mANSVersion == ANSVersionCompat) { - const auto& encoder = std::any_cast&>(coder); - auto view = rans::trim(rans::makeHistogramView(encoder.getSymbolTable())); - return rans::utils::getRangeBits(view.getMin(), view.getMax()); - } else if (mANSVersion == ANSVersion1) { - const auto& encoder = std::any_cast&>(coder); - auto view = rans::trim(rans::makeHistogramView(encoder.getSymbolTable())); - return rans::utils::getRangeBits(view.getMin(), view.getMax()); - } else { - throw std::runtime_error("unsupported ANS version"); - } - }(); - return rans::compat::calculateMaxBufferSizeB(nSamples, alphabetRangeBits); - } else { - return nSamples * sizeof(source_T); - } -}; - } // namespace ctf } // namespace o2 diff --git a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h index e9bd0f7249ef1..31f4227742385 100644 --- a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h +++ b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsCPV/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "CPVReconstruction/CTFHelper.h" class TTree; @@ -46,7 +47,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase template o2::ctf::CTFIOSize decode(const CTF::base& ec, VTRG& trigVec, VCLUSTER& cluVec); - void createCoders(const std::vector& bufVec, ctf::CTFCoderBase::OpType op) final; + void createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBase::OpType op) final; private: template @@ -85,13 +86,13 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODECPV(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODECPV(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODECPV(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODECPV(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -132,13 +134,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCLUSTER auto header = ec.getHeader(); checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector bcInc; - std::vector orbitInc; - std::vector entries, posX, posZ; + std::vector bcInc, entries, posX, posZ; + std::vector orbitInc; std::vector energy, status; o2::ctf::CTFIOSize iosize; -#define DECODECPV(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODECPV(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODECPV(bcInc, CTF::BLC_bcIncTrig); iosize += DECODECPV(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h index 0c86c7ccc7f17..607d426d11c89 100644 --- a/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h +++ b/Detectors/CPV/reconstruction/include/CPVReconstruction/CTFHelper.h @@ -50,7 +50,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -59,86 +59,92 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) { - return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc); + return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc; } else { - return value_type(mData[mIndex].getBCData().bc); + return mData[mIndex].getBCData().bc; } } return 0; @@ -148,9 +154,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) { - return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc); + return mData[id].getBCData().bc - mData[id - 1].getBCData().bc; } else { - return value_type(mData[id].getBCData().bc); + return mData[id].getBCData().bc; } } return 0; @@ -159,15 +165,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncTrig : public _Iter + class Iter_orbitIncTrig : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0); + return id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0; } }; @@ -200,10 +206,10 @@ class CTFHelper }; //_______________________________________________ - class Iter_energy : public _Iter + class Iter_energy : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { return mData[mIndex].getPackedEnergy(); } value_type operator[](difference_type i) const { return mData[mIndex + i].getPackedEnergy(); } }; diff --git a/Detectors/CPV/reconstruction/src/CTFCoder.cxx b/Detectors/CPV/reconstruction/src/CTFCoder.cxx index b86706dee6729..7c94e125dc449 100644 --- a/Detectors/CPV/reconstruction/src/CTFCoder.cxx +++ b/Detectors/CPV/reconstruction/src/CTFCoder.cxx @@ -41,11 +41,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc = 0; - int32_t orbitInc = 0; - uint16_t entries = 0, cluPosX = 0, cluPosZ = 0; + uint16_t bcInc = 0, entries = 0, cluPosX = 0, cluPosZ = 0; + uint32_t orbitInc = 0; uint8_t energy = 0, status = 0; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncTrig); MAKECODER(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx b/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx index 7c14dc70dd430..c05bfc833559e 100644 --- a/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/CPV/workflow/src/EntropyDecoderSpec.cxx @@ -52,7 +52,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_CPV"); auto& triggers = pc.outputs().make>(OutputRef{"triggers"}); @@ -83,7 +83,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_CPV", "CPV", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_CPV", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_CPV", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -91,8 +91,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace cpv diff --git a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx index 97f69fe928342..8901028e7ac66 100644 --- a/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/CPV/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto triggers = pc.inputs().get>("triggers"); auto clusters = pc.inputs().get>("clusters"); if (mSelIR) { @@ -75,7 +75,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("triggers", "CPV", "CLUSTERTRIGRECS", 0, Lifetime::Timeframe); inputs.emplace_back("clusters", "CPV", "CLUSTERS", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "CPV", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("CPV/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -88,8 +88,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace cpv diff --git a/Detectors/CTF/test/test_ctf_io_cpv.cxx b/Detectors/CTF/test/test_ctf_io_cpv.cxx index e4b91569d1df3..2648cd40374cb 100644 --- a/Detectors/CTF/test/test_ctf_io_cpv.cxx +++ b/Detectors/CTF/test/test_ctf_io_cpv.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test CPVCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "CPVReconstruction/CTFCoder.h" #include "DataFormatsCPV/CTF.h" @@ -31,11 +25,8 @@ #include using namespace o2::cpv; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector triggers; std::vector clusters; @@ -64,7 +55,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, triggers, clusters); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_ctp.cxx b/Detectors/CTF/test/test_ctf_io_ctp.cxx index be9cdd1667ba8..25c93f360baf1 100644 --- a/Detectors/CTF/test/test_ctf_io_ctp.cxx +++ b/Detectors/CTF/test/test_ctf_io_ctp.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test CTPCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "CTPReconstruction/CTFCoder.h" #include "DataFormatsCTP/CTF.h" @@ -31,11 +25,8 @@ #include using namespace o2::ctp; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled()) { std::vector digits; TStopwatch sw; @@ -58,7 +49,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, digits, lumi); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_emcal.cxx b/Detectors/CTF/test/test_ctf_io_emcal.cxx index 4c6f6f8193672..95ecf49ea6f05 100644 --- a/Detectors/CTF/test/test_ctf_io_emcal.cxx +++ b/Detectors/CTF/test/test_ctf_io_emcal.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test EMCCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "DataFormatsEMCAL/Constants.h" #include "EMCALReconstruction/CTFCoder.h" @@ -31,11 +25,8 @@ #include using namespace o2::emcal; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector triggers; std::vector cells; @@ -88,7 +79,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, triggers, cells); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_fdd.cxx b/Detectors/CTF/test/test_ctf_io_fdd.cxx index 3dd87c1618598..a1da64af04968 100644 --- a/Detectors/CTF/test/test_ctf_io_fdd.cxx +++ b/Detectors/CTF/test/test_ctf_io_fdd.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test FDDCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "FDDReconstruction/CTFCoder.h" #include "FDDBase/Constants.h" @@ -29,11 +23,8 @@ #include using namespace o2::fdd; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector digits; std::vector channels; @@ -90,7 +81,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, digits, channels); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_ft0.cxx b/Detectors/CTF/test/test_ctf_io_ft0.cxx index 6e2746444e662..4b7bee2460909 100644 --- a/Detectors/CTF/test/test_ctf_io_ft0.cxx +++ b/Detectors/CTF/test/test_ctf_io_ft0.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test FT0CTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "FT0Reconstruction/CTFCoder.h" #include "FT0Base/FT0DigParam.h" @@ -29,11 +23,8 @@ #include using namespace o2::ft0; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector digits; std::vector channels; @@ -96,7 +87,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, digits, channels); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_fv0.cxx b/Detectors/CTF/test/test_ctf_io_fv0.cxx index 2e558b91a8e1c..f9bf4a8a878e5 100644 --- a/Detectors/CTF/test/test_ctf_io_fv0.cxx +++ b/Detectors/CTF/test/test_ctf_io_fv0.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test FV0CTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "FV0Reconstruction/CTFCoder.h" #include "FV0Base/Constants.h" @@ -29,11 +23,8 @@ #include using namespace o2::fv0; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector digits; std::vector channels; @@ -68,7 +59,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, digits, channels); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_hmpid.cxx b/Detectors/CTF/test/test_ctf_io_hmpid.cxx index fd12ee7eb1e36..88df30c8419c0 100644 --- a/Detectors/CTF/test/test_ctf_io_hmpid.cxx +++ b/Detectors/CTF/test/test_ctf_io_hmpid.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test HMPIDCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "HMPIDReconstruction/CTFCoder.h" #include "DataFormatsHMP/CTF.h" @@ -30,11 +24,8 @@ #include using namespace o2::hmpid; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector triggers; std::vector digits; @@ -62,7 +53,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, triggers, digits); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_itsmft.cxx b/Detectors/CTF/test/test_ctf_io_itsmft.cxx index 13cbdf7745961..806ad8536d69d 100644 --- a/Detectors/CTF/test/test_ctf_io_itsmft.cxx +++ b/Detectors/CTF/test/test_ctf_io_itsmft.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test ITSMFTCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "DataFormatsITSMFT/CompCluster.h" #include "DataFormatsITSMFT/CTF.h" #include "DataFormatsITSMFT/ROFRecord.h" @@ -32,11 +26,8 @@ #include using namespace o2::itsmft; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CompressedClustersTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CompressedClustersTest) { std::vector rofRecVec; @@ -82,7 +73,6 @@ BOOST_DATA_TEST_CASE(CompressedClustersTest, boost_data::make(ANSVersions), ansV std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder, o2::detectors::DetID::ITS); - coder.setANSVersion(ansVersion); coder.encode(vec, rofRecVec, cclusVec, pattVec, pattIdConverter, 0); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_mch.cxx b/Detectors/CTF/test/test_ctf_io_mch.cxx index a12ca58a574da..9a6553ba70d81 100644 --- a/Detectors/CTF/test/test_ctf_io_mch.cxx +++ b/Detectors/CTF/test/test_ctf_io_mch.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test MCHCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "MCHCTF/CTFCoder.h" #include "DataFormatsMCH/CTF.h" @@ -32,11 +26,8 @@ #include using namespace o2::mch; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest, *boost::unit_test::enabled()) { std::vector rofs; std::vector digs; @@ -68,7 +59,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, rofs, digs); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_mid.cxx b/Detectors/CTF/test/test_ctf_io_mid.cxx index 2a7122f96e1da..2e8ee98b87e7a 100644 --- a/Detectors/CTF/test/test_ctf_io_mid.cxx +++ b/Detectors/CTF/test/test_ctf_io_mid.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test MIDCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "CommonUtils/IRFrameSelector.h" #include "MIDCTF/CTFCoder.h" @@ -31,11 +25,8 @@ #include using namespace o2::mid; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::array, NEvTypes> colData{}; std::array, NEvTypes> rofData{}; @@ -82,7 +73,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, tfData); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_phos.cxx b/Detectors/CTF/test/test_ctf_io_phos.cxx index 6850642ba129e..f564dfc2b8fc8 100644 --- a/Detectors/CTF/test/test_ctf_io_phos.cxx +++ b/Detectors/CTF/test/test_ctf_io_phos.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test PHSCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "PHOSReconstruction/CTFCoder.h" #include "DataFormatsPHOS/CTF.h" @@ -30,11 +24,8 @@ #include using namespace o2::phos; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector triggers; std::vector cells; @@ -60,7 +51,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, triggers, cells); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_tof.cxx b/Detectors/CTF/test/test_ctf_io_tof.cxx index 276e773f77c69..b770dbb2f78d5 100644 --- a/Detectors/CTF/test/test_ctf_io_tof.cxx +++ b/Detectors/CTF/test/test_ctf_io_tof.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test TOFCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "DataFormatsTOF/CTF.h" #include "TOFBase/Geo.h" #include "TOFBase/Digit.h" @@ -31,11 +25,8 @@ #include using namespace o2::tof; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CompressedClustersTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CompressedClustersTest) { std::vector digits; @@ -95,7 +86,6 @@ BOOST_DATA_TEST_CASE(CompressedClustersTest, boost_data::make(ANSVersions), ansV std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, rows, digits, pattVec); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_tpc.cxx b/Detectors/CTF/test/test_ctf_io_tpc.cxx index 210ab0af105bd..10b5316524baa 100644 --- a/Detectors/CTF/test/test_ctf_io_tpc.cxx +++ b/Detectors/CTF/test/test_ctf_io_tpc.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test TPCCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "DataFormatsTPC/CompressedClusters.h" #include "DataFormatsTPC/CTF.h" #include "CommonUtils/NameConf.h" @@ -30,12 +24,8 @@ #include using namespace o2::tpc; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -inline std::vector CombineColumns(true, false); -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions) ^ boost_data::make(CombineColumns), ansVersion, combineColumns) +BOOST_AUTO_TEST_CASE(CTFTest) { CompressedClusters c; c.nAttachedClusters = 99; @@ -53,7 +43,7 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions) ^ boost_data::make(C { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); coder.setCompClusAddresses(c, buff); - coder.setCombineColumns(combineColumns); + coder.setCombineColumns(true); } ccFlat->set(sz, c); @@ -97,8 +87,7 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions) ^ boost_data::make(C std::vector vecIO; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Decoder); - coder.setCombineColumns(combineColumns); - coder.setANSVersion(ansVersion); + coder.setCombineColumns(true); coder.encode(vecIO, c, c); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_trd.cxx b/Detectors/CTF/test/test_ctf_io_trd.cxx index 66203074ce817..158ccd187e6b8 100644 --- a/Detectors/CTF/test/test_ctf_io_trd.cxx +++ b/Detectors/CTF/test/test_ctf_io_trd.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test TRDCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "TRDReconstruction/CTFCoder.h" #include "DataFormatsTRD/CTF.h" @@ -30,11 +24,8 @@ #include using namespace o2::trd; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector triggers; std::vector tracklets; @@ -79,7 +70,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, triggers, tracklets, digits); // compress } sw.Stop(); diff --git a/Detectors/CTF/test/test_ctf_io_zdc.cxx b/Detectors/CTF/test/test_ctf_io_zdc.cxx index 87563f128a737..6222951c080e6 100644 --- a/Detectors/CTF/test/test_ctf_io_zdc.cxx +++ b/Detectors/CTF/test/test_ctf_io_zdc.cxx @@ -12,13 +12,7 @@ #define BOOST_TEST_MODULE Test ZDCCTFIO #define BOOST_TEST_MAIN #define BOOST_TEST_DYN_LINK - -#undef NDEBUG -#include - #include -#include -#include #include "CommonUtils/NameConf.h" #include "ZDCReconstruction/CTFCoder.h" #include "DataFormatsZDC/CTF.h" @@ -30,11 +24,8 @@ #include using namespace o2::zdc; -namespace boost_data = boost::unit_test::data; - -inline std::vector ANSVersions{o2::ctf::ANSVersionCompat, o2::ctf::ANSVersion1}; -BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) +BOOST_AUTO_TEST_CASE(CTFTest) { std::vector bcdata; std::vector chandata; @@ -90,7 +81,6 @@ BOOST_DATA_TEST_CASE(CTFTest, boost_data::make(ANSVersions), ansVersion) std::vector vec; { CTFCoder coder(o2::ctf::CTFCoderBase::OpType::Encoder); - coder.setANSVersion(ansVersion); coder.encode(vec, bcdata, chandata, pedsdata); // compress } sw.Stop(); diff --git a/Detectors/CTF/utils/CMakeLists.txt b/Detectors/CTF/utils/CMakeLists.txt index f1b55e027f64a..025f7586a0e45 100644 --- a/Detectors/CTF/utils/CMakeLists.txt +++ b/Detectors/CTF/utils/CMakeLists.txt @@ -29,7 +29,3 @@ o2_add_test_root_macro(dumpCTF.C o2_add_test_root_macro(CTFdict2CCDBfiles.C PUBLIC_LINK_LIBRARIES O2::CTFWorkflow fmt::fmt LABELS ctf COMPILE_ONLY) - -o2_add_test_root_macro(convCTFDict.C - PUBLIC_LINK_LIBRARIES O2::CTFWorkflow fmt::fmt - LABELS ctf COMPILE_ONLY) diff --git a/Detectors/CTF/utils/convCTFDict.C b/Detectors/CTF/utils/convCTFDict.C deleted file mode 100644 index 1b42cda3e2a1b..0000000000000 --- a/Detectors/CTF/utils/convCTFDict.C +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -#if !defined(__CLING__) || defined(__ROOTCLING__) - -#include -#include "DetectorsCommonDataFormats/CTFHeader.h" -#include "DetectorsCommonDataFormats/CTFDictHeader.h" -#include "DetectorsCommonDataFormats/EncodedBlocks.h" -#include "DataFormatsCTP/CTF.h" -#include "DataFormatsTRD/CTF.h" -#include "DataFormatsTOF/CTF.h" -#include "DataFormatsTPC/CTF.h" -#include "DataFormatsFT0/CTF.h" -#include "DataFormatsFV0/CTF.h" -#include "DataFormatsFDD/CTF.h" -#include "DataFormatsEMCAL/CTF.h" -#include "DataFormatsITSMFT/CTF.h" -#include "DataFormatsPHOS/CTF.h" -#include "DataFormatsZDC/CTF.h" -#include "DataFormatsMID/CTF.h" -#include "DataFormatsMCH/CTF.h" -#include "DataFormatsHMP/CTF.h" -#include "DataFormatsCPV/CTF.h" - -#include -#include - -#endif - -template -size_t appendToTree(TTree& tree, const std::string brname, T& ptr) -{ - size_t s = 0; - auto* br = tree.GetBranch(brname.c_str()); - auto* pptr = &ptr; - if (br) { - br->SetAddress(&pptr); - } else { - br = tree.Branch(brname.c_str(), &pptr); - } - int res = br->Fill(); - if (res < 0) { - throw std::runtime_error(fmt::format("Failed to fill CTF branch {}", brname)); - } - s += res; - br->ResetAddress(); - return s; -} - -template -void conv(const std::vector* buff) -{ - const auto& ctf = C::getImage(buff->data()); - const auto& ctfhead = ctf.getHeader(); - const auto& dictHead = (const o2::ctf::CTFDictHeader&)ctfhead; - o2::ctf::CTFHeader header{}; - TFile fl(Form("ctf_dictionaryTree_%s_%u_0.root", dictHead.det.getName(), dictHead.dictTimeStamp), "recreate"); - TTree* tree = new TTree("ccdb_object", "O2 CTF dictionary"); - header.detectors.set(dictHead.det); - ctf.appendToTree(*tree, dictHead.det.getName()); - appendToTree(*tree, "CTFHeader", header); - tree->SetEntries(1); - tree->Write(tree->GetName(), TObject::kSingleKey); - delete tree; - fl.Close(); - printf("Stored dictionary to %s\n", fl.GetName()); -} - -void convCTFDict(const char* det, long ts = 0, const char* ccdburl = "http://alice-ccdb.cern.ch") -{ - auto& cm = o2::ccdb::BasicCCDBManager::instance(); - if (ts > 0) { - cm.setTimestamp(ts); - } - std::string pth = std::string(Form("%s/Calib/CTFDictionary", det)); - const auto* buff = cm.get>(pth); - if (!buff) { - printf("Failed to fetch from %s for %ld\n", pth.c_str(), cm.getTimestamp()); - } - std::string dets{det}; - if (dets == "ITS" || dets == "MFT") { - conv(buff); - } else if (dets == "TPC") { - conv(buff); - } else if (dets == "TRD") { - conv(buff); - } else if (dets == "TOF") { - conv(buff); - } else if (dets == "FT0") { - conv(buff); - } else if (dets == "FV0") { - conv(buff); - } else if (dets == "FDD") { - conv(buff); - } else if (dets == "EMC") { - conv(buff); - } else if (dets == "PHS") { - conv(buff); - } else if (dets == "ZDC") { - conv(buff); - } else if (dets == "MID") { - conv(buff); - } else if (dets == "MCH") { - conv(buff); - } else if (dets == "HMP") { - conv(buff); - } else if (dets == "CTP") { - conv(buff); - } else if (dets == "CPV") { - conv(buff); - } -} diff --git a/Detectors/CTF/workflow/src/CTFWriterSpec.cxx b/Detectors/CTF/workflow/src/CTFWriterSpec.cxx index aecb34abb08d9..9556d024f3abc 100644 --- a/Detectors/CTF/workflow/src/CTFWriterSpec.cxx +++ b/Detectors/CTF/workflow/src/CTFWriterSpec.cxx @@ -44,10 +44,7 @@ #include "DataFormatsCPV/CTF.h" #include "DataFormatsZDC/CTF.h" #include "DataFormatsCTP/CTF.h" - -#include "rANS/histogram.h" -#include "rANS/compat.h" - +#include "rANS/rans.h" #include #include #include @@ -91,7 +88,7 @@ size_t appendToTree(TTree& tree, const std::string brname, T& ptr) } using DetID = o2::detectors::DetID; -using FTrans = o2::rans::DenseHistogram; +using FTrans = o2::rans::FrequencyTable; class CTFWriterSpec : public o2::framework::Task { @@ -134,22 +131,22 @@ class CTFWriterSpec : public o2::framework::Task int mSaveDictAfter = 0; // if positive and mWriteCTF==true, save dictionary after each mSaveDictAfter TFs processed uint32_t mPrevDictTimeStamp = 0; // timestamp of the previously stored dictionary uint32_t mDictTimeStamp = 0; // timestamp of the currently stored dictionary - size_t mMinSize = 0; // if > 0, accumulate CTFs in the same tree until the total size exceeds this minimum - size_t mMaxSize = 0; // if > MinSize, and accumulated size will exceed this value, stop accumulation (even if mMinSize is not reached) - size_t mChkSize = 0; // if > 0 and fallback storage provided, reserve this size per CTF file in production on primary storage - size_t mAccCTFSize = 0; // so far accumulated size (if any) - size_t mCurrCTFSize = 0; // size of currently processed CTF - size_t mNCTF = 0; // total number of CTFs written - size_t mNCTFPrevDict = 0; // total number of CTFs used for previous dictionary version - size_t mNAccCTF = 0; // total number of CTFs accumulated in the current file - int mWaitDiskFull = 0; // if mCheckDiskFull triggers, pause for this amount of ms before new attempt - int mWaitDiskFullMax = -1; // produce fatal mCheckDiskFull block the workflow for more than this time (in ms) - float mCheckDiskFull = 0.; // wait for if available abs. disk space is < mCheckDiskFull (if >0) or if its fraction is < -mCheckDiskFull (if <0) - long mCTFAutoSave = 0; // if > 0, autosave after so many TFs - size_t mNCTFFiles = 0; // total number of CTF files written - int mMaxCTFPerFile = 0; // max CTFs per files to store - int mRejRate = 0; // CTF rejection rule (>0: percentage to reject randomly, <0: reject if timeslice%|value|!=0) - int mCTFFileCompression = 0; // CTF file compression level (if >= 0) + size_t mMinSize = 0; // if > 0, accumulate CTFs in the same tree until the total size exceeds this minimum + size_t mMaxSize = 0; // if > MinSize, and accumulated size will exceed this value, stop accumulation (even if mMinSize is not reached) + size_t mChkSize = 0; // if > 0 and fallback storage provided, reserve this size per CTF file in production on primary storage + size_t mAccCTFSize = 0; // so far accumulated size (if any) + size_t mCurrCTFSize = 0; // size of currently processed CTF + size_t mNCTF = 0; // total number of CTFs written + size_t mNCTFPrevDict = 0; // total number of CTFs used for previous dictionary version + size_t mNAccCTF = 0; // total number of CTFs accumulated in the current file + int mWaitDiskFull = 0; // if mCheckDiskFull triggers, pause for this amount of ms before new attempt + int mWaitDiskFullMax = -1; // produce fatal mCheckDiskFull block the workflow for more than this time (in ms) + float mCheckDiskFull = 0.; // wait for if available abs. disk space is < mCheckDiskFull (if >0) or if its fraction is < -mCheckDiskFull (if <0) + long mCTFAutoSave = 0; // if > 0, autosave after so many TFs + size_t mNCTFFiles = 0; // total number of CTF files written + int mMaxCTFPerFile = 0; // max CTFs per files to store + int mRejRate = 0; // CTF rejection rule (>0: percentage to reject randomly, <0: reject if timeslice%|value|!=0) + int mCTFFileCompression = 0; // CTF file compression level (if >= 0) bool mFillMD5 = false; std::vector mTFOrbits{}; // 1st orbits of TF accumulated in current file o2::framework::DataTakingContext mDataTakingContext{}; @@ -319,7 +316,7 @@ size_t CTFWriterSpec::processDet(o2::framework::ProcessingContext& pc, DetID det sz = ctfBuffer.size(); } if (mCreateDict) { - if (mFreqsAccumulation[det].empty()) { + if (!mFreqsAccumulation[det].size()) { mFreqsAccumulation[det].resize(C::getNBlocks()); mFreqsMetaData[det].resize(C::getNBlocks()); } @@ -345,13 +342,8 @@ size_t CTFWriterSpec::processDet(o2::framework::ProcessingContext& pc, DetID det } return true; }()) { - auto newProbBits = static_cast(o2::rans::compat::computeRenormingPrecision(countNUsedAlphabetSymbols(freq))); - auto histogramView = o2::rans::trim(o2::rans::makeHistogramView(freq)); - mdSave = ctf::detail::makeMetadataRansDict(newProbBits, - static_cast(histogramView.getMin()), - static_cast(histogramView.getMax()), - static_cast(histogramView.size()), - md.opt); + auto newProbBits = static_cast(o2::rans::computeRenormingPrecision(freq)); + mdSave = o2::ctf::Metadata{0, 0, md.messageWordSize, md.coderType, md.streamSize, newProbBits, md.opt, freq.getMinSymbol(), freq.getMaxSymbol(), static_cast(freq.size()), 0, 0}; mFreqsAccumulation[det][ib] = std::move(freq); } } diff --git a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h index deef6149e0d0e..534bc3e546cc4 100644 --- a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h +++ b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsCTP/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "CTPReconstruction/CTFHelper.h" #include "CTPReconstruction/RawDataDecoder.h" @@ -88,10 +89,10 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader(lumi)); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODECTP(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODECTP(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODECTP(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODECTP(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -128,12 +130,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& data, LumiInfo& l auto header = ec.getHeader(); checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector bcInc; - std::vector orbitInc; + std::vector bcInc; + std::vector orbitInc; std::vector bytesInput, bytesClass; o2::ctf::CTFIOSize iosize; -#define DECODECTP(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODECTP(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODECTP(bcInc, CTF::BLC_bcIncTrig); iosize += DECODECTP(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h index 89859e8cdbb3a..ceb99c36a8f26 100644 --- a/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h +++ b/Detectors/CTP/reconstruction/include/CTPReconstruction/CTFHelper.h @@ -55,7 +55,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -64,86 +64,91 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? M * data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - inline I& operator--() noexcept + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } + + const I& operator-(size_t idx) { - mIndex--; - return static_cast(*this); + mIndex -= idx; + return (I&)(*this); + } + const I operator++(int) + { + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } - - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; - - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].intRecord.orbit == mData[mIndex - 1].intRecord.orbit) { - return value_type(mData[mIndex].intRecord.bc - mData[mIndex - 1].intRecord.bc); + return mData[mIndex].intRecord.bc - mData[mIndex - 1].intRecord.bc; } else { - return value_type(mData[mIndex].intRecord.bc); + return mData[mIndex].intRecord.bc; } } return 0; @@ -153,9 +158,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].intRecord.orbit == mData[id - 1].intRecord.orbit) { - return value_type(mData[id].intRecord.bc - mData[id - 1].intRecord.bc); + return mData[id].intRecord.bc - mData[id - 1].intRecord.bc; } else { - return value_type(mData[id].intRecord.bc); + return mData[id].intRecord.bc; } } return 0; @@ -164,15 +169,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncTrig : public _Iter + class Iter_orbitIncTrig : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].intRecord.orbit - mData[mIndex - 1].intRecord.orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].intRecord.orbit - mData[mIndex - 1].intRecord.orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].intRecord.orbit - mData[id - 1].intRecord.orbit : 0); + return id ? mData[id].intRecord.orbit - mData[id - 1].intRecord.orbit : 0; } }; diff --git a/Detectors/CTP/reconstruction/src/CTFCoder.cxx b/Detectors/CTP/reconstruction/src/CTFCoder.cxx index e182032e12605..ea68485cc6879 100644 --- a/Detectors/CTP/reconstruction/src/CTFCoder.cxx +++ b/Detectors/CTP/reconstruction/src/CTFCoder.cxx @@ -40,10 +40,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc = 0; - int32_t orbitInc = 0; + uint16_t bcInc = 0; + uint32_t orbitInc = 0; uint8_t bytesInput = 0, bytesClass = 0; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncTrig); MAKECODER(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx index 0e71d05af5815..323a119429952 100644 --- a/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/CTP/workflow/src/EntropyDecoderSpec.cxx @@ -95,8 +95,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) outputs, AlgorithmSpec{adaptFromTask(verbosity)}, Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ctpinputs-decoding-ctf", VariantType::Bool, false, {"Inputs alignment: true - CTF decoder - has to be compatible with reco: allowed options: 10,01,00"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"ctpinputs-decoding-ctf", VariantType::Bool, false, {"Inputs alignment: true - CTF decoder - has to be compatible with reco: allowed options: 10,01,00"}}}}; } } // namespace ctp diff --git a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx index 3e51226b687c4..0f6236b0675cc 100644 --- a/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/CTP/workflow/src/EntropyEncoderSpec.cxx @@ -86,8 +86,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR, bool nolumi) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace ctp diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h index 9bef0b9184117..9a4011a183f32 100644 --- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h +++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsEMCAL/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "EMCALReconstruction/CTFHelper.h" class TTree; @@ -85,15 +86,15 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEEMC(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEEMC(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEEMC(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODEEMC(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -136,13 +138,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCELL& c const auto& header = ec.getHeader(); checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector orbitInc; - std::vector bcInc; - std::vector entries, energy, cellTime, tower, trigger; + std::vector bcInc, entries, energy, cellTime, tower, trigger; + std::vector orbitInc; std::vector status; o2::ctf::CTFIOSize iosize; -#define DECODEEMCAL(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEEMCAL(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEEMCAL(bcInc, CTF::BLC_bcIncTrig); iosize += DECODEEMCAL(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h index b75dac976b47d..ff2d59f879306 100644 --- a/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h +++ b/Detectors/EMCAL/reconstruction/include/EMCALReconstruction/CTFHelper.h @@ -50,7 +50,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -59,86 +59,91 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) { - return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc); + return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc; } else { - return value_type(mData[mIndex].getBCData().bc); + return mData[mIndex].getBCData().bc; } } return 0; @@ -148,9 +153,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) { - return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc); + return mData[id].getBCData().bc - mData[id - 1].getBCData().bc; } else { - return value_type(mData[id].getBCData().bc); + return mData[id].getBCData().bc; } } return 0; @@ -159,15 +164,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncTrig : public _Iter + class Iter_orbitIncTrig : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0); + return id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0; } }; diff --git a/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx b/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx index 22cad97dd7e76..db20df8618155 100644 --- a/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx +++ b/Detectors/EMCAL/reconstruction/src/CTFCoder.cxx @@ -41,11 +41,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc = 0; - int32_t orbitInc = 0; - uint16_t entries = 0, cellTime = 0, energy = 0, tower = 0, trigger = 0; + uint16_t bcInc = 0, entries = 0, cellTime = 0, energy = 0, tower = 0, trigger = 0; + uint32_t orbitInc = 0; uint8_t status = 0; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncTrig); MAKECODER(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx b/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx index 700f468e9e73d..09ef12a5a626d 100644 --- a/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/EMCAL/workflow/src/EntropyDecoderSpec.cxx @@ -52,7 +52,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_EMC"); auto& triggers = pc.outputs().make>(OutputRef{"triggers", mSSpecOut}); @@ -83,7 +83,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspecInp, un std::vector inputs; inputs.emplace_back("ctf_EMC", "EMC", "CTFDATA", sspecInp, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_EMC", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_EMC", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -91,8 +91,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspecInp, un inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity, sspecOut)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace emcal diff --git a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx index da34fcfd8c534..b7e40f71603cb 100644 --- a/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/EMCAL/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto triggers = pc.inputs().get>("triggers"); auto cells = pc.inputs().get>("cells"); @@ -76,7 +76,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("triggers", "EMC", "CELLSTRGR", 0, Lifetime::Timeframe); inputs.emplace_back("cells", "EMC", "CELLS", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "EMC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("EMC/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -90,8 +90,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) {"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace emcal diff --git a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h index dc11174908c75..d5811514dd057 100644 --- a/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h +++ b/Detectors/FIT/FDD/reconstruction/include/FDDReconstruction/CTFCoder.h @@ -25,6 +25,7 @@ #include "DataFormatsFDD/ChannelData.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" class TTree; @@ -71,15 +72,15 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig using MD = o2::ctf::Metadata::OptStore; // what to do which each field: see o2::ctd::Metadata explanation constexpr MD optField[CTF::getNBlocks()] = { - MD::EENCODE_OR_PACK, // BLC_trigger - MD::EENCODE_OR_PACK, // BLC_bcInc - MD::EENCODE_OR_PACK, // BLC_orbitInc - MD::EENCODE_OR_PACK, // BLC_nChan + MD::EENCODE, // BLC_trigger + MD::EENCODE, // BLC_bcInc + MD::EENCODE, // BLC_orbitInc + MD::EENCODE, // BLC_nChan - MD::EENCODE_OR_PACK, // BLC_idChan - MD::EENCODE_OR_PACK, // BLC_time - MD::EENCODE_OR_PACK, // BLC_charge - MD::EENCODE_OR_PACK // BLC_feeBits + MD::EENCODE, // BLC_idChan + MD::EENCODE, // BLC_time + MD::EENCODE, // BLC_charge + MD::EENCODE // BLC_feeBits }; CompressedDigits cd; if (mExtHeader.isValidDictTimeStamp()) { @@ -100,11 +101,11 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig ec->setHeader(cd.header); assignDictVersion(static_cast(ec->getHeader())); - - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEFDD(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEFDD(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEFDD(cd.trigger, CTF::BLC_trigger, 0); iosize += ENCODEFDD(cd.bcInc, CTF::BLC_bcInc, 0); @@ -132,7 +133,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VDIG& digitVec, VCHAN& checkDictVersion(hd); ec.print(getPrefix(), mVerbosity); o2::ctf::CTFIOSize iosize; -#define DECODEFDD(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEFDD(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEFDD(cd.trigger, CTF::BLC_trigger); iosize += DECODEFDD(cd.bcInc, CTF::BLC_bcInc); diff --git a/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx b/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx index 0122b95927f45..315f9ac346d89 100644 --- a/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx +++ b/Detectors/FIT/FDD/reconstruction/src/CTFCoder.cxx @@ -51,7 +51,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); CompressedDigits cd; // just to get member types -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(cd.trigger, CTF::BLC_trigger); MAKECODER(cd.bcInc, CTF::BLC_bcInc); @@ -69,16 +69,21 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd) { size_t sz = 0; + // clang-format off // RS FIXME this is very crude estimate, instead, an empirical values should be used - sz += estimateBufferSize(static_cast(CTF::BLC_trigger), cd.trigger); - sz += estimateBufferSize(static_cast(CTF::BLC_bcInc), cd.bcInc); - sz += estimateBufferSize(static_cast(CTF::BLC_orbitInc), cd.orbitInc); - sz += estimateBufferSize(static_cast(CTF::BLC_nChan), cd.nChan); +#define VTP(vec) typename std::remove_reference::type::value_type +#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \ + rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec)) + sz += ESTSIZE(cd.trigger, CTF::BLC_trigger); + sz += ESTSIZE(cd.bcInc, CTF::BLC_bcInc); + sz += ESTSIZE(cd.orbitInc, CTF::BLC_orbitInc); + sz += ESTSIZE(cd.nChan, CTF::BLC_nChan); - sz += estimateBufferSize(static_cast(CTF::BLC_idChan), cd.idChan); - sz += estimateBufferSize(static_cast(CTF::BLC_time), cd.time); - sz += estimateBufferSize(static_cast(CTF::BLC_charge), cd.charge); - sz += estimateBufferSize(static_cast(CTF::BLC_feeBits), cd.feeBits); + sz += ESTSIZE(cd.idChan, CTF::BLC_idChan); + sz += ESTSIZE(cd.time, CTF::BLC_time); + sz += ESTSIZE(cd.charge, CTF::BLC_charge); + sz += ESTSIZE(cd.feeBits, CTF::BLC_feeBits); + // clang-format on LOG(info) << "Estimated output size is " << sz << " bytes"; return sz; diff --git a/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx b/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx index fb5b173fb7a94..8a606f92c220a 100644 --- a/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/FIT/FDD/workflow/src/EntropyDecoderSpec.cxx @@ -51,7 +51,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_FDD"); auto& digits = pc.outputs().make>(OutputRef{"digits"}); @@ -82,7 +82,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_FDD", "FDD", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_FDD", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_FDD", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -90,8 +90,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace fdd diff --git a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx index b79c033771743..7f448c1d62d7b 100644 --- a/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/FIT/FDD/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto digits = pc.inputs().get>("digits"); auto channels = pc.inputs().get>("channels"); if (mSelIR) { @@ -74,7 +74,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("digits", "FDD", "DIGITSBC", 0, Lifetime::Timeframe); inputs.emplace_back("channels", "FDD", "DIGITSCH", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "FDD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FDD/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -86,8 +86,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace fdd diff --git a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h index 4d749dbc90b42..0566bf9883baa 100644 --- a/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h +++ b/Detectors/FIT/FT0/reconstruction/include/FT0Reconstruction/CTFCoder.h @@ -26,6 +26,7 @@ #include "DetectorsCommonDataFormats/DetID.h" #include "FT0Base/FT0DigParam.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" class TTree; @@ -72,15 +73,15 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig using MD = o2::ctf::Metadata::OptStore; // what to do which each field: see o2::ctd::Metadata explanation constexpr MD optField[CTF::getNBlocks()] = { - MD::EENCODE_OR_PACK, // BLC_trigger - MD::EENCODE_OR_PACK, // BLC_bcInc - MD::EENCODE_OR_PACK, // BLC_orbitInc - MD::EENCODE_OR_PACK, // BLC_nChan - MD::EENCODE_OR_PACK, // BLC_status - MD::EENCODE_OR_PACK, // BLC_idChan - MD::EENCODE_OR_PACK, // BLC_qtcChain - MD::EENCODE_OR_PACK, // BLC_cfdTime - MD::EENCODE_OR_PACK // BLC_qtcAmpl + MD::EENCODE, // BLC_trigger + MD::EENCODE, // BLC_bcInc + MD::EENCODE, // BLC_orbitInc + MD::EENCODE, // BLC_nChan + MD::EENCODE, // BLC_status + MD::EENCODE, // BLC_idChan + MD::EENCODE, // BLC_qtcChain + MD::EENCODE, // BLC_cfdTime + MD::EENCODE // BLC_qtcAmpl }; CompressedDigits cd; if (mExtHeader.isValidDictTimeStamp()) { @@ -101,10 +102,11 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig ec->setHeader(cd.header); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEFT0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEFT0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEFT0(cd.trigger, CTF::BLC_trigger, 0); iosize += ENCODEFT0(cd.bcInc, CTF::BLC_bcInc, 0); @@ -132,7 +134,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VDIG& digitVec, VCHAN& checkDictVersion(hd); ec.print(getPrefix(), mVerbosity); o2::ctf::CTFIOSize iosize; -#define DECODEFT0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEFT0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEFT0(cd.trigger, CTF::BLC_trigger); iosize += DECODEFT0(cd.bcInc, CTF::BLC_bcInc); diff --git a/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx b/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx index be50a8b06180f..6e840c52772cb 100644 --- a/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx +++ b/Detectors/FIT/FT0/reconstruction/src/CTFCoder.cxx @@ -53,7 +53,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); CompressedDigits cd; // just to get member types -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(cd.trigger, CTF::BLC_trigger); MAKECODER(cd.bcInc, CTF::BLC_bcInc); @@ -71,18 +71,22 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd) { size_t sz = 0; + // clang-format off // RS FIXME this is very crude estimate, instead, an empirical values should be used - sz += estimateBufferSize(static_cast(CTF::BLC_trigger), cd.trigger); - sz += estimateBufferSize(static_cast(CTF::BLC_bcInc), cd.bcInc); - sz += estimateBufferSize(static_cast(CTF::BLC_orbitInc), cd.orbitInc); - sz += estimateBufferSize(static_cast(CTF::BLC_nChan), cd.nChan); - sz += estimateBufferSize(static_cast(CTF::BLC_status), cd.eventStatus); - sz += estimateBufferSize(static_cast(CTF::BLC_idChan), cd.idChan); - sz += estimateBufferSize(static_cast(CTF::BLC_qtcChain), cd.qtcChain); - sz += estimateBufferSize(static_cast(CTF::BLC_cfdTime), cd.cfdTime); - sz += estimateBufferSize(static_cast(CTF::BLC_qtcAmpl), cd.qtcAmpl); +#define VTP(vec) typename std::remove_reference::type::value_type +#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \ + rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec)) + sz += ESTSIZE(cd.trigger, CTF::BLC_trigger); + sz += ESTSIZE(cd.bcInc, CTF::BLC_bcInc); + sz += ESTSIZE(cd.orbitInc, CTF::BLC_orbitInc); + sz += ESTSIZE(cd.nChan, CTF::BLC_nChan); + sz += ESTSIZE(cd.eventStatus, CTF::BLC_status); + sz += ESTSIZE(cd.idChan, CTF::BLC_idChan); + sz += ESTSIZE(cd.qtcChain, CTF::BLC_qtcChain); + sz += ESTSIZE(cd.cfdTime, CTF::BLC_cfdTime); + sz += ESTSIZE(cd.qtcAmpl, CTF::BLC_qtcAmpl); // clang-format on LOG(info) << "Estimated output size is " << sz << " bytes"; return sz; -} \ No newline at end of file +} diff --git a/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx b/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx index 65d3585350888..4e161fc11c6c7 100644 --- a/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/FIT/FT0/workflow/src/EntropyDecoderSpec.cxx @@ -51,7 +51,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_FT0"); auto& digits = pc.outputs().make>(OutputRef{"digits"}); @@ -82,7 +82,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_FT0", "FT0", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_FT0", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_FT0", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -90,7 +90,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace ft0 diff --git a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx index edf86b3d7da0d..8f9e865ae8f99 100644 --- a/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/FIT/FT0/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto digits = pc.inputs().get>("digits"); auto channels = pc.inputs().get>("channels"); if (mSelIR) { @@ -75,7 +75,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("digits", "FT0", "DIGITSBC", 0, Lifetime::Timeframe); inputs.emplace_back("channels", "FT0", "DIGITSCH", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "FT0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FT0/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -87,8 +87,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace ft0 diff --git a/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h b/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h index cbec444ef11be..f2b9c8e02c7f1 100644 --- a/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h +++ b/Detectors/FIT/FV0/reconstruction/include/FV0Reconstruction/CTFCoder.h @@ -68,15 +68,15 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig using MD = o2::ctf::Metadata::OptStore; // what to do which each field: see o2::ctd::Metadata explanation constexpr MD optField[CTF::getNBlocks()] = { - MD::EENCODE_OR_PACK, // BLC_bcInc - MD::EENCODE_OR_PACK, // BLC_orbitInc - MD::EENCODE_OR_PACK, // BLC_nChan - MD::EENCODE_OR_PACK, // BLC_idChan - MD::EENCODE_OR_PACK, // BLC_cfdTime - MD::EENCODE_OR_PACK, // BLC_qtcAmpl + MD::EENCODE, // BLC_bcInc + MD::EENCODE, // BLC_orbitInc + MD::EENCODE, // BLC_nChan + MD::EENCODE, // BLC_idChan + MD::EENCODE, // BLC_cfdTime + MD::EENCODE, // BLC_qtcAmpl // extra slot was added in the end - MD::EENCODE_OR_PACK, // BLC_trigger - MD::EENCODE_OR_PACK // BLC_qtcChain + MD::EENCODE, // BLC_trigger + MD::EENCODE // BLC_qtcChain }; CompressedDigits cd; if (mExtHeader.isValidDictTimeStamp()) { @@ -97,10 +97,11 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& dig ec->setHeader(cd.header); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEFV0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEFV0(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEFV0(cd.bcInc, CTF::BLC_bcInc, 0); iosize += ENCODEFV0(cd.orbitInc, CTF::BLC_orbitInc, 0); @@ -128,7 +129,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VDIG& digitVec, VCHAN& checkDictVersion(hd); ec.print(getPrefix(), mVerbosity); o2::ctf::CTFIOSize iosize; -#define DECODEFV0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEFV0(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEFV0(cd.bcInc, CTF::BLC_bcInc); iosize += DECODEFV0(cd.orbitInc, CTF::BLC_orbitInc); diff --git a/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx b/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx index ab3fde0f32510..e718dbd3c07f0 100644 --- a/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx +++ b/Detectors/FIT/FV0/reconstruction/src/CTFCoder.cxx @@ -53,7 +53,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); CompressedDigits cd; // just to get member types -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(cd.bcInc, CTF::BLC_bcInc); MAKECODER(cd.orbitInc, CTF::BLC_orbitInc); @@ -73,17 +73,22 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa size_t CTFCoder::estimateCompressedSize(const CompressedDigits& cd) { size_t sz = 0; + // clang-format off // RS FIXME this is very crude estimate, instead, an empirical values should be used - sz += estimateBufferSize(static_cast(CTF::BLC_trigger), cd.trigger); - sz += estimateBufferSize(static_cast(CTF::BLC_bcInc), cd.bcInc); - sz += estimateBufferSize(static_cast(CTF::BLC_orbitInc), cd.orbitInc); - sz += estimateBufferSize(static_cast(CTF::BLC_nChan), cd.nChan); +#define VTP(vec) typename std::remove_reference::type::value_type +#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \ + rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec)) + sz += ESTSIZE(cd.trigger, CTF::BLC_trigger); + sz += ESTSIZE(cd.bcInc, CTF::BLC_bcInc); + sz += ESTSIZE(cd.orbitInc, CTF::BLC_orbitInc); + sz += ESTSIZE(cd.nChan, CTF::BLC_nChan); - sz += estimateBufferSize(static_cast(CTF::BLC_idChan), cd.idChan); - sz += estimateBufferSize(static_cast(CTF::BLC_qtcChain), cd.qtcChain); - sz += estimateBufferSize(static_cast(CTF::BLC_cfdTime), cd.cfdTime); - sz += estimateBufferSize(static_cast(CTF::BLC_qtcAmpl), cd.qtcAmpl); + sz += ESTSIZE(cd.idChan, CTF::BLC_idChan); + sz += ESTSIZE(cd.qtcChain, CTF::BLC_qtcChain); + sz += ESTSIZE(cd.cfdTime, CTF::BLC_cfdTime); + sz += ESTSIZE(cd.qtcAmpl, CTF::BLC_qtcAmpl); + // clang-format on LOG(info) << "Estimated output size is " << sz << " bytes"; return sz; -}; \ No newline at end of file +} diff --git a/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx b/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx index 9310905ad41b9..dc684f6cb6538 100644 --- a/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/FIT/FV0/workflow/src/EntropyDecoderSpec.cxx @@ -51,7 +51,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_FV0"); auto& digits = pc.outputs().make>(OutputRef{"digits"}); @@ -82,7 +82,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_FV0", "FV0", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_FV0", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_FV0", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -90,8 +90,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace fv0 diff --git a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx index a115712a0c980..a4a34313107d1 100644 --- a/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/FIT/FV0/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto digits = pc.inputs().get>("digits"); auto channels = pc.inputs().get>("channels"); if (mSelIR) { @@ -76,7 +76,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("digits", "FV0", "DIGITSBC", 0, Lifetime::Timeframe); inputs.emplace_back("channels", "FV0", "DIGITSCH", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "FV0", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("FV0/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -89,8 +89,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace fv0 diff --git a/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFCoder.h b/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFCoder.h index da2461c2759ba..d1c825f603e83 100644 --- a/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFCoder.h +++ b/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsHMP/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "HMPIDReconstruction/CTFHelper.h" class TTree; @@ -84,14 +85,14 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEHMP(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEHMP(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEHMP(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODEHMP(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -134,14 +136,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VDIG& di auto header = ec.getHeader(); checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector bcInc; - std::vector orbitInc; - std::vector q; - std::vector entriesDig; + std::vector bcInc, q; + std::vector orbitInc, entriesDig; std::vector chID, ph, x, y; o2::ctf::CTFIOSize iosize; -#define DECODEHMP(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEHMP(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEHMP(bcInc, CTF::BLC_bcIncTrig); iosize += DECODEHMP(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFHelper.h b/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFHelper.h index 8e7c1823a0340..094aa7f367b3f 100644 --- a/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFHelper.h +++ b/Detectors/HMPID/reconstruction/include/HMPIDReconstruction/CTFHelper.h @@ -60,7 +60,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -69,86 +69,92 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? M * data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I operator++(int) { - I res = *(static_cast(this)); + auto res = *this; ++mIndex; return res; } - inline I& operator--() noexcept + const I& operator--() { mIndex--; - return static_cast(*this); + return (I&)(*this); } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + difference_type operator-(size_t idx) const { return mIndex - idx; } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } + + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].getOrbit() == mData[mIndex - 1].getOrbit()) { - return value_type(mData[mIndex].getBc() - mData[mIndex - 1].getBc()); + return mData[mIndex].getBc() - mData[mIndex - 1].getBc(); } else { - return value_type(mData[mIndex].getBc()); + return mData[mIndex].getBc(); } } return 0; @@ -158,9 +164,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].getOrbit() == mData[id - 1].getOrbit()) { - return value_type(mData[id].getBc() - mData[id - 1].getBc()); + return mData[id].getBc() - mData[id - 1].getBc(); } else { - return value_type(mData[id].getBc()); + return mData[id].getBc(); } } return 0; @@ -169,15 +175,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncTrig : public _Iter + class Iter_orbitIncTrig : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].getOrbit() - mData[mIndex - 1].getOrbit() : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].getOrbit() - mData[mIndex - 1].getOrbit() : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].getOrbit() - mData[id - 1].getOrbit() : 0); + return id ? mData[id].getOrbit() - mData[id - 1].getOrbit() : 0; } }; diff --git a/Detectors/HMPID/reconstruction/src/CTFCoder.cxx b/Detectors/HMPID/reconstruction/src/CTFCoder.cxx index c09a4c924cfe9..72fbfd66180ee 100644 --- a/Detectors/HMPID/reconstruction/src/CTFCoder.cxx +++ b/Detectors/HMPID/reconstruction/src/CTFCoder.cxx @@ -41,13 +41,11 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc; - int32_t orbitInc; - uint16_t HCIDTrk, q; - uint32_t entriesDig; + uint16_t bcInc, HCIDTrk, q; + uint32_t orbitInc, entriesDig; uint8_t chID, ph, x, y; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncTrig); MAKECODER(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/HMPID/workflow/src/EntropyDecoderSpec.cxx b/Detectors/HMPID/workflow/src/EntropyDecoderSpec.cxx index aa22979bc305f..102c7f3e13fdf 100644 --- a/Detectors/HMPID/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/HMPID/workflow/src/EntropyDecoderSpec.cxx @@ -69,7 +69,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_HMP"); auto& triggers = pc.outputs().make>(OutputRef{"triggers"}); @@ -100,7 +100,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_HMP", "HMP", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_HMP", "HMP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("HMP/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_HMP", "HMP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("HMP/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -108,8 +108,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace hmpid diff --git a/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx b/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx index 15bd0987a0ccc..d9f032ec5ef7f 100644 --- a/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/HMPID/workflow/src/EntropyEncoderSpec.cxx @@ -66,14 +66,13 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto triggers = pc.inputs().get>("triggers"); auto digits = pc.inputs().get>("digits"); if (mSelIR) { mCTFCoder.setSelectedIRFrames(pc.inputs().get>("selIRFrames")); } auto& buffer = pc.outputs().make>(Output{"HMP", "CTFDATA", 0, Lifetime::Timeframe}); - auto iosize = mCTFCoder.encode(buffer, triggers, digits); pc.outputs().snapshot({"ctfrep", 0}, iosize); if (mSelIR) { @@ -94,7 +93,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("triggers", "HMP", "INTRECORDS", 0, Lifetime::Timeframe); inputs.emplace_back("digits", "HMP", "DIGITS", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "HMP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("HMP/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "HMP", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("HMP/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -107,8 +106,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace hmpid diff --git a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/CTFCoder.h b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/CTFCoder.h index 2d4aabc94fc82..a5af3f7e8b4ae 100644 --- a/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/CTFCoder.h +++ b/Detectors/ITSMFT/common/reconstruction/include/ITSMFTReconstruction/CTFCoder.h @@ -29,6 +29,7 @@ #include "ITSMFTReconstruction/Clusterer.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" //#define _CHECK_INCREMENTES_ // Uncoment this the check the incremements being non-negative @@ -91,16 +92,16 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& using MD = o2::ctf::Metadata::OptStore; // what to do which each field: see o2::ctd::Metadata explanation constexpr MD optField[CTF::getNBlocks()] = { - MD::EENCODE_OR_PACK, // BLCfirstChipROF - MD::EENCODE_OR_PACK, // BLCbcIncROF - MD::EENCODE_OR_PACK, // BLCorbitIncROF - MD::EENCODE_OR_PACK, // BLCnclusROF - MD::EENCODE_OR_PACK, // BLCchipInc - MD::EENCODE_OR_PACK, // BLCchipMul - MD::EENCODE_OR_PACK, // BLCrow - MD::EENCODE_OR_PACK, // BLCcolInc - MD::EENCODE_OR_PACK, // BLCpattID - MD::EENCODE_OR_PACK // BLCpattMap + MD::EENCODE, //BLCfirstChipROF + MD::EENCODE, //BLCbcIncROF + MD::EENCODE, //BLCorbitIncROF + MD::EENCODE, //BLCnclusROF + MD::EENCODE, //BLCchipInc + MD::EENCODE, //BLCchipMul + MD::EENCODE, //BLCrow + MD::EENCODE, //BLCcolInc + MD::EENCODE, //BLCpattID + MD::EENCODE //BLCpattMap }; CompressedClusters compCl; compress(compCl, rofRecVec, cclusVec, pattVec, clPattLookup, strobeLength); @@ -113,10 +114,11 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::span& ec->setHeader(compCl.header); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEITSMFT(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEITSMFT(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEITSMFT(compCl.firstChipROF, CTF::BLCfirstChipROF, 0); iosize += ENCODEITSMFT(compCl.bcIncROF, CTF::BLCbcIncROF, 0); @@ -249,10 +251,10 @@ void CTFCoder::decompress(const CompressedClusters& compCl, VROF& rofRecVec, VCL patt = clPattLookup.getPattern(clus.getPatternID()); } int rowSpan = patt.getRowSpan(), colSpan = patt.getColumnSpan(), nMasked = 0; - if (rowSpan == 1 && colSpan == 1) { // easy case: 1 pixel cluster - if (noiseMap->isNoisy(clus.getChipID(), rowRef, colRef)) { // just kill the cluster - std::copy(pattItStored, pattItPrev, back_inserter(pattVec)); // save patterns from after last saved to the one before killing this - pattItStored = pattIt; // advance to the head of the pattern iterator + if (rowSpan == 1 && colSpan == 1) { // easy case: 1 pixel cluster + if (noiseMap->isNoisy(clus.getChipID(), rowRef, colRef)) { // just kill the cluster + std::copy(pattItStored, pattItPrev, back_inserter(pattVec)); // save patterns from after last saved to the one before killing this + pattItStored = pattIt; // advance to the head of the pattern iterator cclusVec.pop_back(); } // otherwise do nothing: cluster was already added, eventual patterns will be copied in large block at next modified cluster writing diff --git a/Detectors/ITSMFT/common/reconstruction/src/CTFCoder.cxx b/Detectors/ITSMFT/common/reconstruction/src/CTFCoder.cxx index 022efaf6d75d6..74f52db8055a2 100644 --- a/Detectors/ITSMFT/common/reconstruction/src/CTFCoder.cxx +++ b/Detectors/ITSMFT/common/reconstruction/src/CTFCoder.cxx @@ -127,9 +127,9 @@ void CTFCoder::compress(CompressedClusters& cc, LOG(warning) << "Negative BC increment " << intRec.bc << " -> " << prevBC; } #endif - cc.bcIncROF[irofOut] = int16_t(intRec.bc - prevBC); // store increment of BC if in the same orbit + cc.bcIncROF[irofOut] = intRec.bc - prevBC; // store increment of BC if in the same orbit } else { - cc.orbitIncROF[irofOut] = int32_t(intRec.orbit - prevOrbit); + cc.orbitIncROF[irofOut] = intRec.orbit - prevOrbit; #ifdef _CHECK_INCREMENTES_ if (intRec.orbit < prevOrbit) { LOG(warning) << "Negative Orbit increment " << intRec.orbit << " -> " << prevOrbit; @@ -168,13 +168,13 @@ void CTFCoder::compress(CompressedClusters& cc, prevCol = cl.getCol(); } else { // for new chips store chipID increment and abs. column // cc.chipInc[iclOut] = cl.getChipID() - prevChip; // this is the version with chipInc stored for every pixel - cc.chipInc.push_back(int16_t(cl.getChipID() - prevChip)); // this is the version with chipInc stored once per new chip + cc.chipInc.push_back(cl.getChipID() - prevChip); // this is the version with chipInc stored once per new chip #ifdef _CHECK_INCREMENTES_ if (cl.getChipID() < prevChip) { LOG(warning) << "Negative Chip increment " << cl.getChipID() << " -> " << prevChip; } #endif - cc.chipMul.push_back(1); // this is the version with chipInc stored once per new chip + cc.chipMul.push_back(1); // this is the version with chipInc stored once per new chip prevCol = cc.colInc[iclOut] = cl.getCol(); prevChip = cl.getChipID(); } @@ -195,7 +195,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); CompressedClusters cc; // just to get member types -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(cc.firstChipROF, CTF::BLCfirstChipROF); MAKECODER(cc.bcIncROF, CTF::BLCbcIncROF ); @@ -215,19 +215,24 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa size_t CTFCoder::estimateCompressedSize(const CompressedClusters& cc) { size_t sz = 0; + // clang-format off // RS FIXME this is very crude estimate, instead, an empirical values should be used - - sz += estimateBufferSize(static_cast(CTF::BLCfirstChipROF), cc.firstChipROF); - sz += estimateBufferSize(static_cast(CTF::BLCbcIncROF), cc.bcIncROF); - sz += estimateBufferSize(static_cast(CTF::BLCorbitIncROF), cc.orbitIncROF); - sz += estimateBufferSize(static_cast(CTF::BLCnclusROF), cc.nclusROF); +#define VTP(vec) typename std::remove_reference::type::value_type +#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \ + rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec)) + sz += ESTSIZE(cc.firstChipROF, CTF::BLCfirstChipROF); + sz += ESTSIZE(cc.bcIncROF, CTF::BLCbcIncROF ); + sz += ESTSIZE(cc.orbitIncROF, CTF::BLCorbitIncROF ); + sz += ESTSIZE(cc.nclusROF, CTF::BLCnclusROF ); // - sz += estimateBufferSize(static_cast(CTF::BLCchipInc), cc.chipInc); - sz += estimateBufferSize(static_cast(CTF::BLCchipMul), cc.chipMul); - sz += estimateBufferSize(static_cast(CTF::BLCrow), cc.row); - sz += estimateBufferSize(static_cast(CTF::BLCcolInc), cc.colInc); - sz += estimateBufferSize(static_cast(CTF::BLCpattID), cc.pattID); - sz += estimateBufferSize(static_cast(CTF::BLCpattMap), cc.pattMap); + sz += ESTSIZE(cc.chipInc, CTF::BLCchipInc ); + sz += ESTSIZE(cc.chipMul, CTF::BLCchipMul ); + sz += ESTSIZE(cc.row, CTF::BLCrow ); + sz += ESTSIZE(cc.colInc, CTF::BLCcolInc ); + sz += ESTSIZE(cc.pattID, CTF::BLCpattID ); + sz += ESTSIZE(cc.pattMap, CTF::BLCpattMap ); + + // clang-format on sz *= 2. / 3; // if needed, will be autoexpanded LOG(info) << "Estimated output size is " << sz << " bytes"; return sz; @@ -240,7 +245,7 @@ CompressedClusters CTFCoder::decodeCompressedClusters(const CTF::base& ec, o2::c cc.header = ec.getHeader(); checkDictVersion(static_cast(cc.header)); ec.print(getPrefix(), mVerbosity); -#define DECODEITSMFT(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEITSMFT(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEITSMFT(cc.firstChipROF, CTF::BLCfirstChipROF); iosize += DECODEITSMFT(cc.bcIncROF, CTF::BLCbcIncROF); diff --git a/Detectors/ITSMFT/common/workflow/src/EntropyDecoderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/EntropyDecoderSpec.cxx index 4edbc10d5bfbd..d6c7dadf00862 100644 --- a/Detectors/ITSMFT/common/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/ITSMFT/common/workflow/src/EntropyDecoderSpec.cxx @@ -99,7 +99,7 @@ void EntropyDecoderSpec::updateTimeDependentParams(ProcessingContext& pc) pc.inputs().get(std::string("cldict") + mDetPrefix); } } - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); } void EntropyDecoderSpec::finaliseCCDB(o2::framework::ConcreteDataMatcher& matcher, void* obj) @@ -141,7 +141,7 @@ DataProcessorSpec getEntropyDecoderSpec(o2::header::DataOrigin orig, int verbosi inputs.emplace_back(std::string("ctf") + nm, orig, "CTFDATA", sspec, Lifetime::Timeframe); inputs.emplace_back(std::string("noise") + nm, orig, "NOISEMAP", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/NoiseMap", orig.as()))); inputs.emplace_back(std::string("cldict") + nm, orig, "CLUSDICT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/ClusterDictionary", orig.as()))); - inputs.emplace_back(std::string("ctfdict") + nm, orig, "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/CTFDictionaryTree", orig.as()))); + inputs.emplace_back(std::string("ctfdict") + nm, orig, "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/CTFDictionary", orig.as()))); inputs.emplace_back(std::string("trigoffset"), "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -152,8 +152,7 @@ DataProcessorSpec getEntropyDecoderSpec(o2::header::DataOrigin orig, int verbosi Options{ {"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"mask-noise", VariantType::Bool, false, {"apply noise mask to digits or clusters (involves reclusterization)"}}, - {"ignore-cluster-dictionary", VariantType::Bool, false, {"do not use cluster dictionary, always store explicit patterns"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"ignore-cluster-dictionary", VariantType::Bool, false, {"do not use cluster dictionary, always store explicit patterns"}}}}; } } // namespace itsmft diff --git a/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx b/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx index ccb6f0539909e..5bba8d4747d42 100644 --- a/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/ITSMFT/common/workflow/src/EntropyEncoderSpec.cxx @@ -74,7 +74,7 @@ void EntropyEncoderSpec::endOfStream(EndOfStreamContext& ec) void EntropyEncoderSpec::updateTimeDependentParams(ProcessingContext& pc) { - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); if (pc.services().get().globalRunNumberChanged) { // this params need to be queried only once if (mSelIR) { pc.inputs().get("cldict"); @@ -123,7 +123,7 @@ DataProcessorSpec getEntropyEncoderSpec(o2::header::DataOrigin orig, bool selIR) inputs.emplace_back("cldict", orig, "CLUSDICT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/ClusterDictionary", orig.as()))); inputs.emplace_back("alppar", orig, "ALPIDEPARAM", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Config/AlpideParam", orig.as()))); } - inputs.emplace_back("ctfdict", orig, "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/CTFDictionaryTree", orig.as()))); + inputs.emplace_back("ctfdict", orig, "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec(fmt::format("{}/Calib/CTFDictionary", orig.as()))); return DataProcessorSpec{ orig == o2::header::gDataOriginITS ? "its-entropy-encoder" : "mft-entropy-encoder", inputs, @@ -133,8 +133,7 @@ DataProcessorSpec getEntropyEncoderSpec(o2::header::DataOrigin orig, bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace itsmft diff --git a/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFCoder.h b/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFCoder.h index fc090c5c7e16d..cdf1d0d64baa8 100644 --- a/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFCoder.h +++ b/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFCoder.h @@ -26,6 +26,7 @@ #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" #include "MCHCTF/CTFHelper.h" +#include "rANS/rans.h" class TTree; @@ -87,15 +88,15 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEMCH(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEMCH(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEMCH(helper.begin_bcIncROF(), helper.end_bcIncROF(), CTF::BLC_bcIncROF, 0); iosize += ENCODEMCH(helper.begin_orbitIncROF(), helper.end_orbitIncROF(), CTF::BLC_orbitIncROF, 0); @@ -137,15 +139,14 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VROF& rofVec, VCOL& dig checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector nSamples; - std::vector ADC, nDigits; - std::vector orbitInc; + std::vector bcInc, nSamples; + std::vector orbitInc, ADC, nDigits; std::vector tfTime; - std::vector bcInc, detID, padID; + std::vector detID, padID; std::vector isSaturated; o2::ctf::CTFIOSize iosize; -#define DECODEMCH(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEMCH(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEMCH(bcInc, CTF::BLC_bcIncROF); iosize += DECODEMCH(orbitInc, CTF::BLC_orbitIncROF); diff --git a/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFHelper.h b/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFHelper.h index d28941bca8001..0ff1aab046552 100644 --- a/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFHelper.h +++ b/Detectors/MUON/MCH/CTF/include/MCHCTF/CTFHelper.h @@ -52,7 +52,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -61,86 +61,92 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? M * data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncROF : public _Iter + class Iter_bcIncROF : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) { - return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc); + return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc; } else { - return value_type(mData[mIndex].getBCData().bc); + return mData[mIndex].getBCData().bc; } } return 0; @@ -150,9 +156,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) { - return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc); + return mData[id].getBCData().bc - mData[id - 1].getBCData().bc; } else { - return value_type(mData[id].getBCData().bc); + return mData[id].getBCData().bc; } } return 0; @@ -161,15 +167,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncROF : public _Iter + class Iter_orbitIncROF : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0); + return id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0; } }; diff --git a/Detectors/MUON/MCH/CTF/src/CTFCoder.cxx b/Detectors/MUON/MCH/CTF/src/CTFCoder.cxx index 58c30762c56c2..48064eee29861 100644 --- a/Detectors/MUON/MCH/CTF/src/CTFCoder.cxx +++ b/Detectors/MUON/MCH/CTF/src/CTFCoder.cxx @@ -41,13 +41,12 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - uint16_t nSamples; - int32_t orbitInc; - uint32_t ADC, nDigits; + uint16_t bcInc, nDigits, nSamples; + uint32_t orbitInc, ADC; int32_t tfTime; - int16_t bcInc, detID, padID; + int16_t detID, padID; uint8_t isSaturated; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncROF); diff --git a/Detectors/MUON/MCH/CTF/src/EntropyDecoderSpec.cxx b/Detectors/MUON/MCH/CTF/src/EntropyDecoderSpec.cxx index 9ec13fed85690..543f39037a21c 100644 --- a/Detectors/MUON/MCH/CTF/src/EntropyDecoderSpec.cxx +++ b/Detectors/MUON/MCH/CTF/src/EntropyDecoderSpec.cxx @@ -69,7 +69,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_MCH"); auto& rofs = pc.outputs().make>(OutputRef{"rofs", 0}); @@ -101,7 +101,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, const char* specName, uns std::vector inputs; inputs.emplace_back("ctf_MCH", "MCH", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_MCH", "MCH", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MCH/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_MCH", "MCH", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MCH/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -109,8 +109,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, const char* specName, uns inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace mch diff --git a/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx b/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx index 50e085e238d55..5047be8950ca5 100644 --- a/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx +++ b/Detectors/MUON/MCH/Workflow/src/entropy-encoder-workflow.cxx @@ -66,7 +66,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto rofs = pc.inputs().get>("rofs", 0); auto digits = pc.inputs().get>("digits", 0); if (mSelIR) { @@ -90,7 +90,7 @@ DataProcessorSpec getEntropyEncoderSpec(const char* specName, bool selIR) std::vector inputs; inputs.emplace_back("rofs", "MCH", "DIGITROFS", 0, Lifetime::Timeframe); inputs.emplace_back("digits", "MCH", "DIGITS", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "MCH", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MCH/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "MCH", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MCH/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -103,8 +103,7 @@ DataProcessorSpec getEntropyEncoderSpec(const char* specName, bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace mch diff --git a/Detectors/MUON/MID/CTF/include/MIDCTF/CTFCoder.h b/Detectors/MUON/MID/CTF/include/MIDCTF/CTFCoder.h index 3071b65db47b1..25a11525b949e 100644 --- a/Detectors/MUON/MID/CTF/include/MIDCTF/CTFCoder.h +++ b/Detectors/MUON/MID/CTF/include/MIDCTF/CTFCoder.h @@ -25,6 +25,7 @@ #include "DataFormatsMID/ColumnData.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "MIDCTF/CTFHelper.h" class TTree; @@ -62,13 +63,13 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const CTFHelper::TFData& tfData) using MD = o2::ctf::Metadata::OptStore; // what to do which each field: see o2::ctd::Metadata explanation constexpr MD optField[CTF::getNBlocks()] = { - MD::EENCODE_OR_PACK, // BLC_bcIncROF - MD::EENCODE_OR_PACK, // BLC_orbitIncROF - MD::EENCODE_OR_PACK, // BLC_entriesROF - MD::EENCODE_OR_PACK, // BLC_evtypeROF - MD::EENCODE_OR_PACK, // BLC_pattern - MD::EENCODE_OR_PACK, // BLC_deId - MD::EENCODE_OR_PACK // BLC_colId + MD::EENCODE, // BLC_bcIncROF + MD::EENCODE, // BLC_orbitIncROF + MD::EENCODE, // BLC_entriesROF + MD::EENCODE, // BLC_evtypeROF + MD::EENCODE, // BLC_pattern + MD::EENCODE, // BLC_deId + MD::EENCODE // BLC_colId }; CTFHelper helper(tfData); @@ -81,10 +82,11 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const CTFHelper::TFData& tfData) ec->setHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEMID(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEMID(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEMID(helper.begin_bcIncROF(), helper.end_bcIncROF(), CTF::BLC_bcIncROF, 0); iosize += ENCODEMID(helper.begin_orbitIncROF(), helper.end_orbitIncROF(), CTF::BLC_orbitIncROF, 0); @@ -108,13 +110,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, std::array(header)); ec.print(getPrefix(), mVerbosity); - std::vector bcInc; - std::vector orbitInc; - std::vector entries, pattern; + std::vector bcInc, entries, pattern; + std::vector orbitInc; std::vector evType, deId, colId; o2::ctf::CTFIOSize iosize; -#define DECODEMID(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEMID(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEMID(bcInc, CTF::BLC_bcIncROF); iosize += DECODEMID(orbitInc, CTF::BLC_orbitIncROF); diff --git a/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h b/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h index 01e519701b501..ec6a22a76c9ea 100644 --- a/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h +++ b/Detectors/MUON/MID/CTF/include/MIDCTF/CTFHelper.h @@ -68,7 +68,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -77,89 +77,94 @@ class CTFHelper _Iter(const std::vector& ord, const std::array, NEvTypes>& data, bool end = false) : mOrder(ord), mData(&data), mIndex(end ? M * ord.size() : 0) {} _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mOrder{}; const std::array, NEvTypes>* mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncROF : public _Iter + class Iter_bcIncROF : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { const auto ir = (*mData)[mOrder[mIndex].getSource()][mOrder[mIndex].getIndex()].interactionRecord; if (mIndex) { const auto irP = (*mData)[mOrder[mIndex - 1].getSource()][mOrder[mIndex - 1].getIndex()].interactionRecord; if (ir.orbit == irP.orbit) { - return value_type(ir.bc - irP.bc); + return ir.bc - irP.bc; } else { - return value_type(ir.bc); + return ir.bc; } } return 0; @@ -171,9 +176,9 @@ class CTFHelper if (id) { const auto irP = (*mData)[mOrder[id - 1].getSource()][mOrder[id - 1].getIndex()].interactionRecord; if (ir.orbit == irP.orbit) { - return value_type(ir.bc - irP.bc); + return ir.bc - irP.bc; } else { - return value_type(ir.bc); + return ir.bc; } } return 0; @@ -182,16 +187,16 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncROF : public _Iter + class Iter_orbitIncROF : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { const auto ir = (*mData)[mOrder[mIndex].getSource()][mOrder[mIndex].getIndex()].interactionRecord; const auto irP = (*mData)[mOrder[mIndex - 1].getSource()][mOrder[mIndex - 1].getIndex()].interactionRecord; - return value_type(ir.orbit - irP.orbit); + return ir.orbit - irP.orbit; } return 0; } @@ -201,7 +206,7 @@ class CTFHelper if (id) { const auto ir = (*mData)[mOrder[id].getSource()][mOrder[id].getIndex()].interactionRecord; const auto irP = (*mData)[mOrder[id - 1].getSource()][mOrder[id - 1].getIndex()].interactionRecord; - return value_type(ir.orbit - irP.orbit); + return ir.orbit - irP.orbit; } return 0; } diff --git a/Detectors/MUON/MID/CTF/src/CTFCoder.cxx b/Detectors/MUON/MID/CTF/src/CTFCoder.cxx index a49b3b2faaeaa..e44a608acf987 100644 --- a/Detectors/MUON/MID/CTF/src/CTFCoder.cxx +++ b/Detectors/MUON/MID/CTF/src/CTFCoder.cxx @@ -42,11 +42,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc = 0; - int32_t orbitInc = 0; - uint16_t entries = 0, pattern = 0; + uint16_t bcInc = 0, entries = 0, pattern = 0; + uint32_t orbitInc = 0; uint8_t evType = 0, deId = 0, colId = 0; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncROF); MAKECODER(orbitInc, CTF::BLC_orbitIncROF); diff --git a/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx index e0030eb102c8d..14016bc53e6b7 100644 --- a/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/MUON/MID/Workflow/src/EntropyDecoderSpec.cxx @@ -53,9 +53,8 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_MID"); - std::array, NEvTypes> rofs{}; std::array, NEvTypes> cols{}; // since the buff is const, we cannot use EncodedBlocks::relocate directly, instead we wrap its data to another flat object @@ -94,7 +93,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) outputs.emplace_back(OutputSpec{{"ctfrep"}, "MID", "CTFDECREP", 0, Lifetime::Timeframe}); std::vector inputs; inputs.emplace_back("ctf_MID", "MID", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_MID", "MID", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MID/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_MID", "MID", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MID/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -102,8 +101,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace mid diff --git a/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx b/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx index 44bf3dcc0ae6f..501aae10d2a30 100644 --- a/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/MUON/MID/Workflow/src/EntropyEncoderSpec.cxx @@ -55,7 +55,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); CTFHelper::TFData tfData; std::vector filter = { @@ -105,7 +105,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("rofs", ConcreteDataTypeMatcher(header::gDataOriginMID, "DATAROF"), Lifetime::Timeframe); inputs.emplace_back("cols", ConcreteDataTypeMatcher(header::gDataOriginMID, "DATA"), Lifetime::Timeframe); - inputs.emplace_back("ctfdict", header::gDataOriginMID, "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MID/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", header::gDataOriginMID, "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("MID/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -118,8 +118,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace mid diff --git a/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFCoder.h b/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFCoder.h index 96ee5093bacca..07e6083a0cfca 100644 --- a/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFCoder.h +++ b/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsPHOS/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "PHOSReconstruction/CTFHelper.h" class TTree; @@ -84,13 +85,13 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEPHS(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEPHS(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEPHS(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODEPHS(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -131,13 +133,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCELL& c const auto& header = ec.getHeader(); checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector bcInc; - std::vector orbitInc; - std::vector entries, energy, cellTime, packedID; + std::vector bcInc, entries, energy, cellTime, packedID; + std::vector orbitInc; std::vector status; o2::ctf::CTFIOSize iosize; -#define DECODEPHOS(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEPHOS(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEPHOS(bcInc, CTF::BLC_bcIncTrig); iosize += DECODEPHOS(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFHelper.h b/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFHelper.h index f8bfbbec1bec9..c1b5553b5c098 100644 --- a/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFHelper.h +++ b/Detectors/PHOS/reconstruction/include/PHOSReconstruction/CTFHelper.h @@ -50,7 +50,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -59,86 +59,92 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) { - return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc); + return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc; } else { - return value_type(mData[mIndex].getBCData().bc); + return mData[mIndex].getBCData().bc; } } return 0; @@ -148,9 +154,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) { - return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc); + return mData[id].getBCData().bc - mData[id - 1].getBCData().bc; } else { - return value_type(mData[id].getBCData().bc); + return mData[id].getBCData().bc; } } return 0; @@ -159,15 +165,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncTrig : public _Iter + class Iter_orbitIncTrig : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0); + return id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0; } }; diff --git a/Detectors/PHOS/reconstruction/src/CTFCoder.cxx b/Detectors/PHOS/reconstruction/src/CTFCoder.cxx index 424b6851aacae..03fd8e8f40e1b 100644 --- a/Detectors/PHOS/reconstruction/src/CTFCoder.cxx +++ b/Detectors/PHOS/reconstruction/src/CTFCoder.cxx @@ -41,11 +41,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc = 0; - int32_t orbitInc = 0; - uint16_t entries = 0, cellTime = 0, energy = 0, packedid = 0; + uint16_t bcInc = 0, entries = 0, cellTime = 0, energy = 0, packedid = 0; + uint32_t orbitInc = 0; uint8_t status = 0; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncTrig); MAKECODER(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/PHOS/workflow/src/EntropyDecoderSpec.cxx b/Detectors/PHOS/workflow/src/EntropyDecoderSpec.cxx index a3d15862a2057..572f8671c1ad8 100644 --- a/Detectors/PHOS/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/PHOS/workflow/src/EntropyDecoderSpec.cxx @@ -52,7 +52,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_PHS"); auto& triggers = pc.outputs().make>(OutputRef{"triggers"}); @@ -83,7 +83,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_PHS", "PHS", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_PHS", "PHS", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("PHS/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_PHS", "PHS", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("PHS/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -91,8 +91,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace phos diff --git a/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx b/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx index 226184c1179c6..2fd6263378ed8 100644 --- a/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/PHOS/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto triggers = pc.inputs().get>("triggers"); auto cells = pc.inputs().get>("cells"); if (mSelIR) { @@ -75,7 +75,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) std::vector inputs; inputs.emplace_back("triggers", "PHS", "CELLTRIGREC", 0, Lifetime::Timeframe); inputs.emplace_back("cells", "PHS", "CELLS", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "PHS", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("PHS/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "PHS", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("PHS/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -88,8 +88,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace phos diff --git a/Detectors/TOF/reconstruction/include/TOFReconstruction/CTFCoder.h b/Detectors/TOF/reconstruction/include/TOFReconstruction/CTFCoder.h index e559dcce7a1da..dedf82e204dfc 100644 --- a/Detectors/TOF/reconstruction/include/TOFReconstruction/CTFCoder.h +++ b/Detectors/TOF/reconstruction/include/TOFReconstruction/CTFCoder.h @@ -21,6 +21,7 @@ #include #include "DataFormatsTOF/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" +#include "rANS/rans.h" #include "DetectorsBase/CTFCoderBase.h" #include "TOFBase/Digit.h" @@ -67,17 +68,17 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const gsl::spansetHeader(cc.header); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODETOF(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODETOF(part, slot, bits) CTF::get(buff.data())->encode(part, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODETOF(cc.bcIncROF, CTF::BLCbcIncROF, 0); iosize += ENCODETOF(cc.orbitIncROF, CTF::BLCorbitIncROF, 0); @@ -123,7 +125,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VROF& rofRecVec, VDIG& cc.header = ec.getHeader(); checkDictVersion(static_cast(cc.header)); o2::ctf::CTFIOSize iosize; -#define DECODETOF(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODETOF(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODETOF(cc.bcIncROF, CTF::BLCbcIncROF); iosize += DECODETOF(cc.orbitIncROF, CTF::BLCorbitIncROF); diff --git a/Detectors/TOF/reconstruction/src/CTFCoder.cxx b/Detectors/TOF/reconstruction/src/CTFCoder.cxx index 753433fda3e00..630039eae215b 100644 --- a/Detectors/TOF/reconstruction/src/CTFCoder.cxx +++ b/Detectors/TOF/reconstruction/src/CTFCoder.cxx @@ -151,7 +151,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); CompressedInfos cc; // just to get member types -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(cc.bcIncROF, CTF::BLCbcIncROF); MAKECODER(cc.orbitIncROF, CTF::BLCorbitIncROF); @@ -172,20 +172,25 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa size_t CTFCoder::estimateCompressedSize(const CompressedInfos& cc) { size_t sz = 0; + // clang-format off // RS FIXME this is very crude estimate, instead, an empirical values should be used +#define VTP(vec) typename std::remove_reference::type::value_type +#define ESTSIZE(vec, slot) mCoders[int(slot)] ? \ + rans::calculateMaxBufferSize(vec.size(), reinterpret_cast*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), sizeof(VTP(vec)) ) : vec.size()*sizeof(VTP(vec)) - sz += estimateBufferSize(static_cast(CTF::BLCbcIncROF), cc.bcIncROF); - sz += estimateBufferSize(static_cast(CTF::BLCorbitIncROF), cc.orbitIncROF); - sz += estimateBufferSize(static_cast(CTF::BLCndigROF), cc.ndigROF); - sz += estimateBufferSize(static_cast(CTF::BLCndiaROF), cc.ndiaROF); - sz += estimateBufferSize(static_cast(CTF::BLCndiaCrate), cc.ndiaCrate); - sz += estimateBufferSize(static_cast(CTF::BLCtimeFrameInc), cc.timeFrameInc); - sz += estimateBufferSize(static_cast(CTF::BLCtimeTDCInc), cc.timeTDCInc); - sz += estimateBufferSize(static_cast(CTF::BLCstripID), cc.stripID); - sz += estimateBufferSize(static_cast(CTF::BLCchanInStrip), cc.chanInStrip); - sz += estimateBufferSize(static_cast(CTF::BLCtot), cc.tot); - sz += estimateBufferSize(static_cast(CTF::BLCpattMap), cc.pattMap); + sz += ESTSIZE(cc.bcIncROF, CTF::BLCbcIncROF); + sz += ESTSIZE(cc.orbitIncROF, CTF::BLCorbitIncROF); + sz += ESTSIZE(cc.ndigROF, CTF::BLCndigROF); + sz += ESTSIZE(cc.ndiaROF, CTF::BLCndiaROF); + sz += ESTSIZE(cc.ndiaCrate, CTF::BLCndiaCrate); + sz += ESTSIZE(cc.timeFrameInc, CTF::BLCtimeFrameInc); + sz += ESTSIZE(cc.timeTDCInc, CTF::BLCtimeTDCInc); + sz += ESTSIZE(cc.stripID, CTF::BLCstripID); + sz += ESTSIZE(cc.chanInStrip, CTF::BLCchanInStrip); + sz += ESTSIZE(cc.tot, CTF::BLCtot); + sz += ESTSIZE(cc.pattMap, CTF::BLCpattMap); + // clang-format on sz *= 2. / 3; // if needed, will be autoexpanded LOG(debug) << "Estimated output size is " << sz << " bytes"; return sz; -} \ No newline at end of file +} diff --git a/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx b/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx index b2c735af87241..f68f5a485b5a9 100644 --- a/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/TOF/workflow/src/EntropyDecoderSpec.cxx @@ -52,7 +52,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_TOF"); auto& digitheader = pc.outputs().make(OutputRef{"digitheader"}); @@ -105,7 +105,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_TOF", "TOF", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_TOF", "TOF", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TOF/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_TOF", "TOF", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TOF/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -113,8 +113,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace tof diff --git a/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx index 8f215a0572961..83e3f53edfb45 100644 --- a/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/TOF/workflow/src/EntropyEncoderSpec.cxx @@ -48,7 +48,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto compDigits = pc.inputs().get>("compDigits"); auto pspan = pc.inputs().get>("patterns"); auto rofs = pc.inputs().get>("ROframes"); @@ -77,7 +77,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) inputs.emplace_back("compDigits", o2::header::gDataOriginTOF, "DIGITS", 0, Lifetime::Timeframe); inputs.emplace_back("patterns", o2::header::gDataOriginTOF, "PATTERNS", 0, Lifetime::Timeframe); inputs.emplace_back("ROframes", o2::header::gDataOriginTOF, "READOUTWINDOW", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "TOF", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TOF/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "TOF", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TOF/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -91,8 +91,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, {"irframe-shift", VariantType::Int, o2::tof::Geo::LATENCYWINDOW_IN_BC, {"IRFrame shift to account for latency"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace tof diff --git a/Detectors/TPC/reconstruction/include/TPCReconstruction/CTFCoder.h b/Detectors/TPC/reconstruction/include/TPCReconstruction/CTFCoder.h index 6242b50eab449..f029ad5571db1 100644 --- a/Detectors/TPC/reconstruction/include/TPCReconstruction/CTFCoder.h +++ b/Detectors/TPC/reconstruction/include/TPCReconstruction/CTFCoder.h @@ -28,7 +28,8 @@ #include "DataFormatsTPC/CompressedClusters.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" -#include "rANS/iterator.h" +#include "rANS/rans.h" +#include "rANS/utils.h" class TTree; @@ -69,7 +70,7 @@ class ShiftFunctor template auto makeInputIterators(iterA_T iterA, iterB_T iterB, size_t nElements, F functor) { - using namespace o2::rans; + using namespace o2::rans::utils; auto advanceIter = [](auto iter, size_t nElements) { auto tmp = iter; @@ -89,9 +90,8 @@ struct MergedColumnsDecoder { template static void decode(iterA_T iterA, iterB_T iterB, CTF::Slots slot, F decodingFunctor) { - using namespace o2::rans; ShiftFunctor f{}; - auto iter = CombinedOutputIteratorFactory::makeIter(iterA, iterB, f); + auto iter = rans::utils::CombinedOutputIteratorFactory::makeIter(iterA, iterB, f); decodingFunctor(iter, slot); } @@ -156,7 +156,7 @@ class CTFCoder : public o2::ctf::CTFCoderBase template void CTFCoder::buildCoder(ctf::CTFCoderBase::OpType coderType, const CTF::container_t& ctf, CTF::Slots slot) { - this->createCoder(coderType, std::get>(ctf.getDictionary(slot, mANSVersion)), static_cast(slot)); + this->createCoder(coderType, ctf.getFrequencyTable(slot), static_cast(slot)); } /// entropy-encode clusters to buffer with CTF @@ -167,29 +167,29 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const CompressedClusters& ccl, co using namespace detail; // what to do which each field: see o2::ctf::Metadata explanation constexpr MD optField[CTF::getNBlocks()] = { - MD::EENCODE_OR_PACK, // qTotA - MD::EENCODE_OR_PACK, // qMaxA - MD::EENCODE_OR_PACK, // flagsA - MD::EENCODE_OR_PACK, // rowDiffA - MD::EENCODE_OR_PACK, // sliceLegDiffA - MD::EENCODE_OR_PACK, // padResA - MD::EENCODE_OR_PACK, // timeResA - MD::EENCODE_OR_PACK, // sigmaPadA - MD::EENCODE_OR_PACK, // sigmaTimeA - MD::EENCODE_OR_PACK, // qPtA - MD::EENCODE_OR_PACK, // rowA - MD::EENCODE_OR_PACK, // sliceA - MD::EENCODE_OR_PACK, // timeA - MD::EENCODE_OR_PACK, // padA - MD::EENCODE_OR_PACK, // qTotU - MD::EENCODE_OR_PACK, // qMaxU - MD::EENCODE_OR_PACK, // flagsU - MD::EENCODE_OR_PACK, // padDiffU - MD::EENCODE_OR_PACK, // timeDiffU - MD::EENCODE_OR_PACK, // sigmaPadU - MD::EENCODE_OR_PACK, // sigmaTimeU - MD::EENCODE_OR_PACK, // nTrackClusters - MD::EENCODE_OR_PACK // nSliceRowClusters + MD::EENCODE, // qTotA + MD::EENCODE, // qMaxA + MD::EENCODE, // flagsA + MD::EENCODE, // rowDiffA + MD::EENCODE, // sliceLegDiffA + MD::EENCODE, // padResA + MD::EENCODE, // timeResA + MD::EENCODE, // sigmaPadA + MD::EENCODE, // sigmaTimeA + MD::EENCODE, // qPtA + MD::EENCODE, // rowA + MD::EENCODE, // sliceA + MD::EENCODE, // timeA + MD::EENCODE, // padA + MD::EENCODE, // qTotU + MD::EENCODE, // qMaxU + MD::EENCODE, // flagsU + MD::EENCODE, // padDiffU + MD::EENCODE, // timeDiffU + MD::EENCODE, // sigmaPadU + MD::EENCODE, // sigmaTimeU + MD::EENCODE, // nTrackClusters + MD::EENCODE // nSliceRowClusters }; // book output size with some margin @@ -204,7 +204,8 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const CompressedClusters& ccl, co ec->setHeader(CTFHeader{o2::detectors::DetID::TPC, 0, 1, 0, // dummy timestamp, version 1.0 cclFiltered, flags}); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; o2::ctf::CTFIOSize iosize; auto encodeTPC = [&buff, &optField, &coders = mCoders, mfc = this->getMemMarginFactor(), &iosize](auto begin, auto end, CTF::Slots slot, size_t probabilityBits, std::vector* reject = nullptr) { @@ -218,9 +219,9 @@ o2::ctf::CTFIOSize CTFCoder::encode(VEC& buff, const CompressedClusters& ccl, co tmp.emplace_back(*i); } } - iosize += CTF::get(buff.data())->encode(tmp.begin(), tmp.end(), slotVal, probabilityBits, optField[slotVal], &buff, coders[slotVal], mfc); + iosize += CTF::get(buff.data())->encode(tmp.begin(), tmp.end(), slotVal, probabilityBits, optField[slotVal], &buff, coders[slotVal].get(), mfc); } else { - iosize += CTF::get(buff.data())->encode(begin, end, slotVal, probabilityBits, optField[slotVal], &buff, coders[slotVal], mfc); + iosize += CTF::get(buff.data())->encode(begin, end, slotVal, probabilityBits, optField[slotVal], &buff, coders[slotVal].get(), mfc); } }; @@ -319,7 +320,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VEC& buffVec) o2::ctf::CTFIOSize iosize; auto decodeTPC = [&ec, &coders = mCoders, &iosize](auto begin, CTF::Slots slot) { const auto slotVal = static_cast(slot); - iosize += ec.decode(begin, slotVal, coders[slotVal]); + iosize += ec.decode(begin, slotVal, coders[slotVal].get()); }; if (mCombineColumns) { diff --git a/Detectors/TPC/reconstruction/src/CTFCoder.cxx b/Detectors/TPC/reconstruction/src/CTFCoder.cxx index 5b621a707caaf..aee39caa54c78 100644 --- a/Detectors/TPC/reconstruction/src/CTFCoder.cxx +++ b/Detectors/TPC/reconstruction/src/CTFCoder.cxx @@ -147,7 +147,7 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa /// make sure loaded dictionaries (if any) are consistent with data void CTFCoder::checkDataDictionaryConsistency(const CTFHeader& h) { - if (mCoders[0].has_value()) { // if external dictionary is provided (it will set , make sure its columns combining option is the same as + if (mCoders[0]) { // if external dictionary is provided (it will set , make sure its columns combining option is the same as if (mCombineColumns != (h.flags & CTFHeader::CombinedColumns)) { throw std::runtime_error(fmt::format("Mismatch in columns combining mode, Dictionary:{:s} CTFHeader:{:s}", mCombineColumns ? "ON" : "OFF", (h.flags & CTFHeader::CombinedColumns) ? "ON" : "OFF")); @@ -161,55 +161,36 @@ void CTFCoder::checkDataDictionaryConsistency(const CTFHeader& h) ///________________________________ size_t CTFCoder::estimateCompressedSize(const CompressedClusters& ccl) { - using namespace detail; size_t sz = 0; - + // clang-format off // RS FIXME this is very crude estimate, instead, an empirical values should be used - if (mCombineColumns) { - sz += estimateBufferSize>(CTF::BLCqTotA, ccl.nAttachedClusters); - } else { - sz += estimateBufferSize(CTF::BLCqTotA, ccl.qTotA, ccl.qTotA + ccl.nAttachedClusters); - } - sz += estimateBufferSize(CTF::BLCqMaxA, ccl.qMaxA, ccl.qMaxA + (mCombineColumns ? 0 : ccl.nAttachedClusters)); - sz += estimateBufferSize(CTF::BLCflagsA, ccl.flagsA, ccl.flagsA + ccl.nAttachedClusters); - - if (mCombineColumns) { - sz += estimateBufferSize>(CTF::BLCrowDiffA, ccl.nAttachedClustersReduced); - } else { - sz += estimateBufferSize(CTF::BLCrowDiffA, ccl.rowDiffA, ccl.rowDiffA + ccl.nAttachedClustersReduced); - } - sz += estimateBufferSize(CTF::BLCsliceLegDiffA, ccl.sliceLegDiffA, ccl.sliceLegDiffA + (mCombineColumns ? 0 : ccl.nAttachedClustersReduced)); - sz += estimateBufferSize(CTF::BLCpadResA, ccl.padResA, ccl.padResA + ccl.nAttachedClustersReduced); - sz += estimateBufferSize(CTF::BLCtimeResA, ccl.timeResA, ccl.timeResA + ccl.nAttachedClustersReduced); - if (mCombineColumns) { - sz += estimateBufferSize>(CTF::BLCsigmaPadA, ccl.nAttachedClusters); - } else { - sz += estimateBufferSize(CTF::BLCsigmaPadA, ccl.sigmaPadA, ccl.sigmaPadA + ccl.nAttachedClusters); - } - sz += estimateBufferSize(CTF::BLCsigmaTimeA, ccl.sigmaTimeA, ccl.sigmaTimeA + (mCombineColumns ? 0 : ccl.nAttachedClusters)); - sz += estimateBufferSize(CTF::BLCqPtA, ccl.qPtA, ccl.qPtA + ccl.nTracks); - sz += estimateBufferSize(CTF::BLCrowA, ccl.rowA, ccl.rowA + ccl.nTracks); - sz += estimateBufferSize(CTF::BLCsliceA, ccl.sliceA, ccl.sliceA + ccl.nTracks); - sz += estimateBufferSize(CTF::BLCtimeA, ccl.timeA, ccl.timeA + ccl.nTracks); - sz += estimateBufferSize(CTF::BLCpadA, ccl.padA, ccl.padA + ccl.nTracks); - if (mCombineColumns) { - sz += estimateBufferSize>(CTF::BLCqTotU, ccl.nUnattachedClusters); - } else { - sz += estimateBufferSize(CTF::BLCqTotU, ccl.qTotU, ccl.qTotU + ccl.nUnattachedClusters); - } - sz += estimateBufferSize(CTF::BLCqMaxU, ccl.qMaxU, ccl.qMaxU + (mCombineColumns ? 0 : ccl.nUnattachedClusters)); - sz += estimateBufferSize(CTF::BLCflagsU, ccl.flagsU, ccl.flagsU + ccl.nUnattachedClusters); - sz += estimateBufferSize(CTF::BLCpadDiffU, ccl.padDiffU, ccl.padDiffU + ccl.nUnattachedClusters); - sz += estimateBufferSize(CTF::BLCtimeDiffU, ccl.timeDiffU, ccl.timeDiffU + ccl.nUnattachedClusters); - if (mCombineColumns) { - sz += estimateBufferSize>(CTF::BLCsigmaPadU, ccl.nUnattachedClusters); - } else { - sz += estimateBufferSize(CTF::BLCsigmaPadU, ccl.sigmaPadU, ccl.sigmaPadU + ccl.nUnattachedClusters); - } - sz += estimateBufferSize(CTF::BLCsigmaTimeU, ccl.sigmaTimeU, ccl.sigmaTimeU + (mCombineColumns ? 0 : ccl.nUnattachedClusters)); - sz += estimateBufferSize(CTF::BLCnTrackClusters, ccl.nTrackClusters, ccl.nTrackClusters + ccl.nTracks); - sz += estimateBufferSize(CTF::BLCnSliceRowClusters, ccl.nSliceRowClusters, ccl.nSliceRowClusters + ccl.nSliceRows); - +#define ESTSIZE(slot, ptr, n) mCoders[int(slot)] ? \ + rans::calculateMaxBufferSize(n, reinterpret_cast::type>*>(mCoders[int(slot)].get())->getAlphabetRangeBits(), \ + sizeof(std::remove_pointer::type)) : n*sizeof(std::remove_pointer) + sz += ESTSIZE(CTF::BLCqTotA, ccl.qTotA, ccl.nAttachedClusters); + sz += ESTSIZE(CTF::BLCqMaxA, ccl.qMaxA, ccl.nAttachedClusters); + sz += ESTSIZE(CTF::BLCflagsA, ccl.flagsA, ccl.nAttachedClusters); + sz += ESTSIZE(CTF::BLCrowDiffA, ccl.rowDiffA, ccl.nAttachedClustersReduced); + sz += ESTSIZE(CTF::BLCsliceLegDiffA, ccl.sliceLegDiffA, ccl.nAttachedClustersReduced); + sz += ESTSIZE(CTF::BLCpadResA, ccl.padResA, ccl.nAttachedClustersReduced); + sz += ESTSIZE(CTF::BLCtimeResA, ccl.timeResA, ccl.nAttachedClustersReduced); + sz += ESTSIZE(CTF::BLCsigmaPadA, ccl.sigmaPadA, ccl.nAttachedClusters); + sz += ESTSIZE(CTF::BLCsigmaTimeA, ccl.sigmaTimeA, ccl.nAttachedClusters); + sz += ESTSIZE(CTF::BLCqPtA, ccl.qPtA, ccl.nTracks); + sz += ESTSIZE(CTF::BLCrowA, ccl.rowA, ccl.nTracks); + sz += ESTSIZE(CTF::BLCsliceA, ccl.sliceA, ccl.nTracks); + sz += ESTSIZE(CTF::BLCtimeA, ccl.timeA, ccl.nTracks); + sz += ESTSIZE(CTF::BLCpadA, ccl.padA, ccl.nTracks); + sz += ESTSIZE(CTF::BLCqTotU, ccl.qTotU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCqMaxU, ccl.qMaxU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCflagsU, ccl.flagsU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCpadDiffU, ccl.padDiffU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCtimeDiffU, ccl.timeDiffU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCsigmaPadU, ccl.sigmaPadU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCsigmaTimeU, ccl.sigmaTimeU, ccl.nUnattachedClusters); + sz += ESTSIZE(CTF::BLCnTrackClusters, ccl.nTrackClusters, ccl.nTracks); + sz += ESTSIZE(CTF::BLCnSliceRowClusters, ccl.nSliceRowClusters, ccl.nSliceRows); + // clang-format on sz *= 2. / 3; // if needed, will be autoexpanded LOG(info) << "Estimated output size is " << sz << " bytes"; return sz; diff --git a/Detectors/TPC/workflow/src/EntropyDecoderSpec.cxx b/Detectors/TPC/workflow/src/EntropyDecoderSpec.cxx index e0f27913c9d2b..772e4c373813e 100644 --- a/Detectors/TPC/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/TPC/workflow/src/EntropyDecoderSpec.cxx @@ -77,8 +77,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) Outputs{OutputSpec{{"output"}, "TPC", "COMPCLUSTERSFLAT", 0, Lifetime::Timeframe}, OutputSpec{{"ctfrep"}, "TPC", "CTFDECREP", 0, Lifetime::Timeframe}}, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace tpc diff --git a/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx index 28226752528bf..231c2a61942d4 100644 --- a/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/TPC/workflow/src/EntropyEncoderSpec.cxx @@ -299,8 +299,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool inputFromFile, bool selIR) {"irframe-clusters-maxeta", VariantType::Float, 1.5f, {"Max eta for non-assigned clusters"}}, {"irframe-clusters-maxz", VariantType::Float, 25.f, {"Max z for non assigned clusters (combined with maxeta)"}}, {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"nThreads-tpc-encoder", VariantType::UInt32, 1u, {"number of threads to use for decoding"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"nThreads-tpc-encoder", VariantType::UInt32, 1u, {"number of threads to use for decoding"}}}}; } } // namespace tpc diff --git a/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFCoder.h b/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFCoder.h index 107a05a397c00..33ae563441db2 100644 --- a/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFCoder.h +++ b/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsTRD/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "TRDReconstruction/CTFHelper.h" #include "CommonConstants/LHCConstants.h" @@ -117,21 +118,21 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODETRD(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODETRD(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODETRD(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODETRD(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -188,7 +190,7 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VTRK& tr std::vector padrowTrk, colTrk, slopeTrk, ROBDig, MCMDig, chanDig; o2::ctf::CTFIOSize iosize; -#define DECODETRD(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODETRD(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODETRD(bcInc, CTF::BLC_bcIncTrig); iosize += DECODETRD(orbitInc, CTF::BLC_orbitIncTrig); @@ -223,8 +225,8 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VTRK& tr for (uint32_t itrig = 0; itrig < header.nTriggers; itrig++) { // restore TrigRecord - if (orbitInc[itrig]) { // non-0 increment => new orbit - bc = bcInc[itrig]; // bcInc has absolute meaning + if (orbitInc[itrig]) { // non-0 increment => new orbit + bc = bcInc[itrig]; // bcInc has absolute meaning orbit += orbitInc[itrig]; } else { bc += bcInc[itrig]; @@ -246,8 +248,8 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VTRK& tr } orbitPrev = orbit; o2::InteractionRecord ir{bc, orbit}; - if (triggerOK && (checkIROK || canApplyBCShift(ir))) { // correction will be ok - checkIROK = true; // don't check anymore since the following checks will yield same + if (triggerOK && (checkIROK || canApplyBCShift(ir))) { // correction will be ok + checkIROK = true; // don't check anymore since the following checks will yield same orbitPrevGood = orbit; uint32_t firstEntryTrk = trkVec.size(); uint16_t hcid = 0; diff --git a/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFHelper.h b/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFHelper.h index 316f2c8a5c7f0..e426992da17a9 100644 --- a/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFHelper.h +++ b/Detectors/TRD/reconstruction/include/TRDReconstruction/CTFHelper.h @@ -68,7 +68,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -77,86 +77,92 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? M * data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].getBCData().orbit == mData[mIndex - 1].getBCData().orbit) { - return value_type(mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc); + return mData[mIndex].getBCData().bc - mData[mIndex - 1].getBCData().bc; } else { - return value_type(mData[mIndex].getBCData().bc); + return mData[mIndex].getBCData().bc; } } return 0; @@ -166,9 +172,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].getBCData().orbit == mData[id - 1].getBCData().orbit) { - return value_type(mData[id].getBCData().bc - mData[id - 1].getBCData().bc); + return mData[id].getBCData().bc - mData[id - 1].getBCData().bc; } else { - return value_type(mData[id].getBCData().bc); + return mData[id].getBCData().bc; } } return 0; @@ -181,11 +187,11 @@ class CTFHelper { public: using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0); } + value_type operator*() const { return mIndex ? mData[mIndex].getBCData().orbit - mData[mIndex - 1].getBCData().orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0); + return id ? mData[id].getBCData().orbit - mData[id - 1].getBCData().orbit : 0; } }; diff --git a/Detectors/TRD/reconstruction/src/CTFCoder.cxx b/Detectors/TRD/reconstruction/src/CTFCoder.cxx index a69f2cb96f8b3..61ac13a12d001 100644 --- a/Detectors/TRD/reconstruction/src/CTFCoder.cxx +++ b/Detectors/TRD/reconstruction/src/CTFCoder.cxx @@ -41,13 +41,11 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcInc; - int32_t orbitInc; - uint16_t HCIDTrk, posTrk, CIDDig, ADCDig; - uint32_t entriesTrk, entriesDig, pidTrk; + uint16_t bcInc, HCIDTrk, posTrk, CIDDig, ADCDig; + uint32_t orbitInc, entriesTrk, entriesDig, pidTrk; uint8_t padrowTrk, colTrk, slopeTrk, ROBDig, MCMDig, chanDig; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcInc, CTF::BLC_bcIncTrig); MAKECODER(orbitInc, CTF::BLC_orbitIncTrig); diff --git a/Detectors/TRD/workflow/src/EntropyDecoderSpec.cxx b/Detectors/TRD/workflow/src/EntropyDecoderSpec.cxx index b30732927c182..c3d47c0c69fd9 100644 --- a/Detectors/TRD/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/TRD/workflow/src/EntropyDecoderSpec.cxx @@ -86,7 +86,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_TRD"); auto& triggers = pc.outputs().make>(OutputRef{"triggers"}); @@ -119,7 +119,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_TRD", "TRD", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_TRD", "TRD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TRD/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_TRD", "TRD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TRD/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -129,8 +129,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) AlgorithmSpec{adaptFromTask(verbosity)}, Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"correct-trd-trigger-offset", VariantType::Bool, false, {"Correct decoded IR by TriggerOffsetsParam::LM_L0"}}, - {"bogus-trigger-rejection", VariantType::Int, 10, {">0 : discard, warn N times, <0 : warn only, =0: no check for triggers with no tracklets or bogus IR"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"bogus-trigger-rejection", VariantType::Int, 10, {">0 : discard, warn N times, <0 : warn only, =0: no check for triggers with no tracklets or bogus IR"}}}}; } } // namespace trd diff --git a/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx b/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx index 07243d2b316cb..0a352fbc7d90f 100644 --- a/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/TRD/workflow/src/EntropyEncoderSpec.cxx @@ -68,7 +68,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto triggers = pc.inputs().get>("triggers"); auto tracklets = pc.inputs().get>("tracklets"); auto digits = pc.inputs().get>("digits"); @@ -98,7 +98,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) inputs.emplace_back("triggers", "TRD", "TRKTRGRD", 0, Lifetime::Timeframe); inputs.emplace_back("tracklets", "TRD", "TRACKLETS", 0, Lifetime::Timeframe); inputs.emplace_back("digits", "TRD", "DIGITS", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "TRD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TRD/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "TRD", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("TRD/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -112,8 +112,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"bogus-trigger-check", VariantType::Int, 10, {"max bogus triggers to report, all if < 0"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"bogus-trigger-check", VariantType::Int, 10, {"max bogus triggers to report, all if < 0"}}}}; } } // namespace trd diff --git a/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFCoder.h b/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFCoder.h index c2f2163600f29..db7f785385ade 100644 --- a/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFCoder.h +++ b/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFCoder.h @@ -23,6 +23,7 @@ #include "DataFormatsZDC/CTF.h" #include "DetectorsCommonDataFormats/DetID.h" #include "DetectorsBase/CTFCoderBase.h" +#include "rANS/rans.h" #include "ZDCReconstruction/CTFHelper.h" class TTree; @@ -98,20 +99,20 @@ o2::ctf::CTFIOSize CTFCoder::encode_impl(VEC& buff, const gsl::spansetHeader(helper.createHeader()); assignDictVersion(static_cast(ec->getHeader())); - ec->setANSHeader(mANSVersion); + ec->getANSHeader().majorVersion = 0; + ec->getANSHeader().minorVersion = 1; // at every encoding the buffer might be autoexpanded, so we don't work with fixed pointer ec o2::ctf::CTFIOSize iosize; -#define ENCODEZDC(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)], getMemMarginFactor()); +#define ENCODEZDC(beg, end, slot, bits) CTF::get(buff.data())->encode(beg, end, int(slot), bits, optField[int(slot)], &buff, mCoders[int(slot)].get(), getMemMarginFactor()); // clang-format off iosize += ENCODEZDC(helper.begin_bcIncTrig(), helper.end_bcIncTrig(), CTF::BLC_bcIncTrig, 0); iosize += ENCODEZDC(helper.begin_orbitIncTrig(), helper.end_orbitIncTrig(), CTF::BLC_orbitIncTrig, 0); @@ -159,13 +161,12 @@ o2::ctf::CTFIOSize CTFCoder::decode(const CTF::base& ec, VTRG& trigVec, VCHAN& c auto header = ec.getHeader(); checkDictVersion(static_cast(header)); ec.print(getPrefix(), mVerbosity); - std::vector bcIncTrig, scalerInc; - std::vector orbitIncTrig, orbitIncEOD; - std::vector moduleTrig, nchanTrig, chanData, pedData, triggersHL, channelsHL; + std::vector bcIncTrig, moduleTrig, nchanTrig, chanData, pedData, scalerInc, triggersHL, channelsHL; + std::vector orbitIncTrig, orbitIncEOD; std::vector extTriggers, chanID; o2::ctf::CTFIOSize iosize; -#define DECODEZDC(part, slot) ec.decode(part, int(slot), mCoders[int(slot)]) +#define DECODEZDC(part, slot) ec.decode(part, int(slot), mCoders[int(slot)].get()) // clang-format off iosize += DECODEZDC(bcIncTrig, CTF::BLC_bcIncTrig); iosize += DECODEZDC(orbitIncTrig, CTF::BLC_orbitIncTrig); diff --git a/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFHelper.h b/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFHelper.h index 17e93a6cee11a..45d008d531fdc 100644 --- a/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFHelper.h +++ b/Detectors/ZDC/reconstruction/include/ZDCReconstruction/CTFHelper.h @@ -60,7 +60,7 @@ class CTFHelper class _Iter { public: - using difference_type = std::ptrdiff_t; + using difference_type = int64_t; using value_type = T; using pointer = const T*; using reference = const T&; @@ -69,86 +69,92 @@ class CTFHelper _Iter(const gsl::span& data, bool end = false) : mData(data), mIndex(end ? M * data.size() : 0){}; _Iter() = default; - inline I& operator++() noexcept + const I& operator++() { ++mIndex; - return static_cast(*this); + return (I&)(*this); } - inline I operator++(int) + const I& operator--() { - I res = *(static_cast(this)); - ++mIndex; - return res; + mIndex--; + return (I&)(*this); } - inline I& operator--() noexcept + const I operator++(int) { - mIndex--; - return static_cast(*this); + auto res = *this; + ++mIndex; + return res; } - inline I operator--(int) + const I operator--(int) { - I res = *(static_cast(this)); + auto res = *this; --mIndex; return res; } - I& operator+=(difference_type i) noexcept + const I& operator+=(difference_type i) { mIndex += i; - return static_cast(*this); + return (I&)(*this); } - I operator+(difference_type i) const + const I operator+=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res += i; + auto tmp = *const_cast(this); + return tmp += i; } - I& operator-=(difference_type i) noexcept + const I& operator-=(difference_type i) { mIndex -= i; - return static_cast(*this); + return (I&)(*this); } - I operator-(difference_type i) const + const I operator-=(difference_type i) const { - I res = *(const_cast(static_cast(this))); - return res -= i; + auto tmp = *const_cast(this); + return tmp -= i; } - difference_type operator-(const I& other) const noexcept { return mIndex - other.mIndex; } + difference_type operator-(const I& other) const { return mIndex - other.mIndex; } + + difference_type operator-(size_t idx) const { return mIndex - idx; } - inline friend I operator+(difference_type i, const I& iter) { return iter + i; }; + const I& operator-(size_t idx) + { + mIndex -= idx; + return (I&)(*this); + } - bool operator!=(const I& other) const noexcept { return mIndex != other.mIndex; } - bool operator==(const I& other) const noexcept { return mIndex == other.mIndex; } - bool operator>(const I& other) const noexcept { return mIndex > other.mIndex; } - bool operator<(const I& other) const noexcept { return mIndex < other.mIndex; } - bool operator>=(const I& other) const noexcept { return mIndex >= other.mIndex; } - bool operator<=(const I& other) const noexcept { return mIndex <= other.mIndex; } + bool operator!=(const I& other) const { return mIndex != other.mIndex; } + bool operator==(const I& other) const { return mIndex == other.mIndex; } + bool operator>(const I& other) const { return mIndex > other.mIndex; } + bool operator<(const I& other) const { return mIndex < other.mIndex; } + bool operator>=(const I& other) const { return mIndex >= other.mIndex; } + bool operator<=(const I& other) const { return mIndex <= other.mIndex; } protected: gsl::span mData{}; - difference_type mIndex = 0; + size_t mIndex = 0; }; //_______________________________________________ // BC difference wrt previous if in the same orbit, otherwise the abs.value. // For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_bcIncTrig : public _Iter + class Iter_bcIncTrig : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { if (mIndex) { if (mData[mIndex].ir.orbit == mData[mIndex - 1].ir.orbit) { - return value_type(mData[mIndex].ir.bc - mData[mIndex - 1].ir.bc); + return mData[mIndex].ir.bc - mData[mIndex - 1].ir.bc; } else { - return value_type(mData[mIndex].ir.bc); + return mData[mIndex].ir.bc; } } return 0; @@ -158,9 +164,9 @@ class CTFHelper size_t id = mIndex + i; if (id) { if (mData[id].ir.orbit == mData[id - 1].ir.orbit) { - return value_type(mData[id].ir.bc - mData[id - 1].ir.bc); + return mData[id].ir.bc - mData[id - 1].ir.bc; } else { - return value_type(mData[id].ir.bc); + return mData[id].ir.bc; } } return 0; @@ -170,15 +176,15 @@ class CTFHelper /////////////////////////////////// BCData iterators //////////////////////////////////////// //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncTrig : public _Iter + class Iter_orbitIncTrig : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].ir.orbit - mData[mIndex - 1].ir.orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].ir.orbit - mData[mIndex - 1].ir.orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].ir.orbit - mData[id - 1].ir.orbit : 0); + return id ? mData[id].ir.orbit - mData[id - 1].ir.orbit : 0; } }; @@ -272,15 +278,15 @@ class CTFHelper //_______________________________________________ // Orbit difference wrt previous. For the very 1st entry return 0 (diff wrt 1st BC in the CTF header) - class Iter_orbitIncEOD : public _Iter + class Iter_orbitIncEOD : public _Iter { public: - using _Iter::_Iter; - value_type operator*() const { return value_type(mIndex ? mData[mIndex].ir.orbit - mData[mIndex - 1].ir.orbit : 0); } + using _Iter::_Iter; + value_type operator*() const { return mIndex ? mData[mIndex].ir.orbit - mData[mIndex - 1].ir.orbit : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; - return value_type(id ? mData[id].ir.orbit - mData[id - 1].ir.orbit : 0); + return id ? mData[id].ir.orbit - mData[id - 1].ir.orbit : 0; } }; @@ -298,21 +304,21 @@ class CTFHelper }; //_______________________________________________ - class Iter_sclInc : public _Iter + class Iter_sclInc : public _Iter { public: - using _Iter::_Iter; + using _Iter::_Iter; value_type operator*() const { // define with respect to previous orbit int slot = mIndex / NChannels, chan = mIndex % NChannels; - return value_type(slot ? mData[slot].scaler[chan] - mData[slot - 1].scaler[chan] : 0); + return slot ? mData[slot].scaler[chan] - mData[slot - 1].scaler[chan] : 0; } value_type operator[](difference_type i) const { size_t id = mIndex + i; int slot = id / NChannels, chan = id % NChannels; - return value_type(slot ? mData[slot].scaler[chan] - mData[slot - 1].scaler[chan] : 0); + return slot ? mData[slot].scaler[chan] - mData[slot - 1].scaler[chan] : 0; } }; diff --git a/Detectors/ZDC/reconstruction/src/CTFCoder.cxx b/Detectors/ZDC/reconstruction/src/CTFCoder.cxx index 060a4a5f2c6b2..ffc5f7ab92e00 100644 --- a/Detectors/ZDC/reconstruction/src/CTFCoder.cxx +++ b/Detectors/ZDC/reconstruction/src/CTFCoder.cxx @@ -41,11 +41,10 @@ void CTFCoder::createCoders(const std::vector& bufVec, o2::ctf::CTFCoderBa { const auto ctf = CTF::getImage(bufVec.data()); // just to get types - int16_t bcIncTrig, sclInc; - int32_t orbitIncTrig, orbitIncEOD; - uint16_t moduleTrig, nchanTrig, chanData, pedData, triggersHL, channelsHL; + uint16_t bcIncTrig, moduleTrig, nchanTrig, chanData, pedData, sclInc, triggersHL, channelsHL; + uint32_t orbitIncTrig, orbitIncEOD; uint8_t extTriggers, chanID; -#define MAKECODER(part, slot) createCoder(op, std::get>(ctf.getDictionary(slot, mANSVersion)), int(slot)) +#define MAKECODER(part, slot) createCoder(op, ctf.getFrequencyTable(slot), int(slot)) // clang-format off MAKECODER(bcIncTrig, CTF::BLC_bcIncTrig); MAKECODER(orbitIncTrig, CTF::BLC_orbitIncTrig); diff --git a/Detectors/ZDC/workflow/src/EntropyDecoderSpec.cxx b/Detectors/ZDC/workflow/src/EntropyDecoderSpec.cxx index bf870324ce442..15b297a511ca9 100644 --- a/Detectors/ZDC/workflow/src/EntropyDecoderSpec.cxx +++ b/Detectors/ZDC/workflow/src/EntropyDecoderSpec.cxx @@ -58,7 +58,7 @@ void EntropyDecoderSpec::run(ProcessingContext& pc) mTimer.Start(false); o2::ctf::CTFIOSize iosize; - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto buff = pc.inputs().get>("ctf_ZDC"); auto& bcdata = pc.outputs().make>(OutputRef{"trig"}); @@ -91,7 +91,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) std::vector inputs; inputs.emplace_back("ctf_ZDC", "ZDC", "CTFDATA", sspec, Lifetime::Timeframe); - inputs.emplace_back("ctfdict_ZDC", "ZDC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("ZDC/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict_ZDC", "ZDC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("ZDC/Calib/CTFDictionary")); inputs.emplace_back("trigoffset", "CTP", "Trig_Offset", 0, Lifetime::Condition, ccdbParamSpec("CTP/Config/TriggerOffsets")); return DataProcessorSpec{ @@ -99,8 +99,7 @@ DataProcessorSpec getEntropyDecoderSpec(int verbosity, unsigned int sspec) inputs, outputs, AlgorithmSpec{adaptFromTask(verbosity)}, - Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}}}; } } // namespace zdc diff --git a/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx b/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx index 7c30a0f34d151..aebf6b3271bf6 100644 --- a/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx +++ b/Detectors/ZDC/workflow/src/EntropyEncoderSpec.cxx @@ -50,7 +50,7 @@ void EntropyEncoderSpec::run(ProcessingContext& pc) { auto cput = mTimer.CpuTime(); mTimer.Start(false); - mCTFCoder.updateTimeDependentParams(pc, true); + mCTFCoder.updateTimeDependentParams(pc); auto bcdata = pc.inputs().get>("trig"); auto chans = pc.inputs().get>("chan"); auto peds = pc.inputs().get>("peds"); @@ -80,7 +80,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) inputs.emplace_back("trig", "ZDC", "DIGITSBC", 0, Lifetime::Timeframe); inputs.emplace_back("chan", "ZDC", "DIGITSCH", 0, Lifetime::Timeframe); inputs.emplace_back("peds", "ZDC", "DIGITSPD", 0, Lifetime::Timeframe); - inputs.emplace_back("ctfdict", "ZDC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("ZDC/Calib/CTFDictionaryTree")); + inputs.emplace_back("ctfdict", "ZDC", "CTFDICT", 0, Lifetime::Condition, ccdbParamSpec("ZDC/Calib/CTFDictionary")); if (selIR) { inputs.emplace_back("selIRFrames", "CTF", "SELIRFRAMES", 0, Lifetime::Timeframe); } @@ -93,8 +93,7 @@ DataProcessorSpec getEntropyEncoderSpec(bool selIR) Options{{"ctf-dict", VariantType::String, "ccdb", {"CTF dictionary: empty or ccdb=CCDB, none=no external dictionary otherwise: local filename"}}, {"irframe-margin-bwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame lower boundary when selection is requested"}}, {"irframe-margin-fwd", VariantType::UInt32, 0u, {"margin in BC to add to the IRFrame upper boundary when selection is requested"}}, - {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}, - {"ans-version", VariantType::String, {"version of ans entropy coder implementation to use"}}}}; + {"mem-factor", VariantType::Float, 1.f, {"Memory allocation margin factor"}}}}; } } // namespace zdc diff --git a/Utilities/rANS/CMakeLists.txt b/Utilities/rANS/CMakeLists.txt index 1e929d204c3a7..840e1a51fc4b1 100644 --- a/Utilities/rANS/CMakeLists.txt +++ b/Utilities/rANS/CMakeLists.txt @@ -9,270 +9,77 @@ # granted to it by virtue of its status as an Intergovernmental Organization # or submit itself to any jurisdiction. -option(RANS_ENABLE_JSON "compile librans with rapidjson" ON) -set(RANS_ARCH "" CACHE STRING "Compile librans for a given CPU type,\ - see https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html.") - set(RANS_COMPILE_OPTIONS "" CACHE STRING "Compile librans with these compiler option") - -if(NOT $ENV{RANS_ARCH} STREQUAL "") -set(RANS_ARCH $ENV{RANS_ARCH}) -endif() - -if(NOT RANS_ARCH STREQUAL "") -set(RANS_TARGET_ARCH -march=${RANS_ARCH}) -message(STATUS "building librans for arch ${RANS_ARCH}" ) -endif() -set(RANS_OPTIONS ${RANS_TARGET_ARCH} ${RANS_COMPILE_OPTIONS}) - -# compile options for tests -if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64") - set(RANS_TEST_ARCH -march=native) -endif() - -o2_add_header_only_library(rANS - TARGETVARNAME LIBRANS - INTERFACE_LINK_LIBRARIES FairLogger::FairLogger Microsoft.GSL::GSL absl::flat_hash_map) - target_compile_options(${LIBRANS} INTERFACE ${RANS_OPTIONS}) -if(${RANS_ENABLE_JSON}) - target_compile_definitions(${LIBRANS} INTERFACE RANS_ENABLE_JSON) - target_link_libraries(${LIBRANS} INTERFACE RapidJSON::RapidJSON) -endif() -if (OpenMP_CXX_FOUND AND ( NOT APPLE ) ) - target_compile_definitions(${LIBRANS} INTERFACE RANS_OPENMP) - target_link_libraries(${LIBRANS} INTERFACE OpenMP::OpenMP_CXX) -endif() -if(${ENABLE_VTUNE_PROFILER}) - target_compile_definitions(${LIBRANS} INTERFACE ENABLE_VTUNE_PROFILER) - target_link_libraries(${LIBRANS} INTERFACE PkgConfig::Vtune) -endif() - -o2_add_test(Utils - NAME ransUtils - SOURCES test/test_ransUtils.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_UTILS - LABELS utils) - target_compile_options(${TEST_UTILS} PRIVATE ${RANS_TEST_ARCH}) +o2_add_library(rANS + TARGETVARNAME targetName + SOURCES src/FrequencyTable.cxx + src/RenormedFrequencyTable.cxx + PUBLIC_LINK_LIBRARIES FairLogger::FairLogger) +#target_compile_definitions(${targetName} PUBLIC O2_RANS_PRINT_PROCESSED_DATA) o2_add_test(Iterators - NAME ransIterators + NAME Iterators SOURCES test/test_ransIterators.cxx PUBLIC_LINK_LIBRARIES O2::rANS COMPONENT_NAME rANS - TARGETVARNAME TEST_ITERATORS - LABELS utils) - target_compile_options(${TEST_ITERATORS} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(BitPtr - NAME ransBitPtr - SOURCES test/test_ransBitPtr.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_BITPTR - LABELS utils) - target_compile_options(${TEST_BITPTR} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(SparseVector - NAME ransSparseVector - SOURCES test/test_ransSparseVector.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_SPARSEVECTOR - LABELS utils) - target_compile_options(${TEST_SPARSEVECTOR} PRIVATE ${RANS_TEST_ARCH}) - - -o2_add_test(Pack - NAME ransPack - SOURCES test/test_ransPack.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_PACK - LABELS utils) - target_compile_options(${TEST_PACK} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(SIMDEncoderKernels - NAME ransSIMDEncoderKernels - SOURCES test/test_ransSIMDEncoderKernels.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_SIMD_ENCODER_KERNELS - LABELS utils) - target_compile_options(${TEST_SIMD_ENCODER_KERNELS} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(SIMD - NAME ransSIMD - SOURCES test/test_ransSIMD.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_SIMD LABELS utils) - target_compile_options(${TEST_SIMD} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(AlignedArray - NAME ransAlignedArray - SOURCES test/test_ransAlignedArray.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_ALIGNED_ARRAY - LABELS utils) - target_compile_options(${TEST_ALIGNED_ARRAY} PRIVATE ${RANS_TEST_ARCH}) o2_add_test(HistogramView - NAME ransHistogramView + NAME HistogramView SOURCES test/test_ransHistogramView.cxx PUBLIC_LINK_LIBRARIES O2::rANS COMPONENT_NAME rANS - TARGETVARNAME TEST_HISTOGRAM_VIEW LABELS utils) - target_compile_options(${TEST_HISTOGRAM_VIEW} PRIVATE ${RANS_TEST_ARCH}) -o2_add_test(Histograms - NAME ransHistograms - SOURCES test/test_ransHistograms.cxx +o2_add_test(FrequencyTable + NAME FrequencyTable + SOURCES test/test_ransFrequencyTable.cxx PUBLIC_LINK_LIBRARIES O2::rANS COMPONENT_NAME rANS - TARGETVARNAME TEST_HISTOGRAMS LABELS utils) - target_compile_options(${TEST_HISTOGRAMS} PRIVATE ${RANS_TEST_ARCH}) -o2_add_test(Metrics - NAME ransMetrics - SOURCES test/test_ransMetrics.cxx +o2_add_test(SymbolTable + NAME SymbolTable + SOURCES test/test_ransSymbolTable.cxx PUBLIC_LINK_LIBRARIES O2::rANS COMPONENT_NAME rANS - TARGETVARNAME TEST_METRICS LABELS utils) - target_compile_options(${TEST_METRICS} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(SymbolTables - NAME ransSymbolTables - SOURCES test/test_ransSymbolTables.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_SYMBOL_TABLES - LABELS utils) - target_compile_options(${TEST_SYMBOL_TABLES} PRIVATE ${RANS_TEST_ARCH}) o2_add_test(ReverseSymbolLookupTable - NAME ransReverseSymbolLookupTable + NAME ReverseSymbolLookupTable SOURCES test/test_ransReverseSymbolLookupTable.cxx PUBLIC_LINK_LIBRARIES O2::rANS COMPONENT_NAME rANS - TARGETVARNAME TEST_REVERSE_SYMBOL_LOOKUP_TABLE LABELS utils) - target_compile_options(${TEST_REVERSE_SYMBOL_LOOKUP_TABLE} PRIVATE ${RANS_TEST_ARCH}) o2_add_test(EncodeDecode - NAME ransEncodeDecode + NAME EncodeDecode SOURCES test/test_ransEncodeDecode.cxx PUBLIC_LINK_LIBRARIES O2::rANS COMPONENT_NAME rANS - TARGETVARNAME TEST_ENCODE_DECODE LABELS utils) - target_compile_options(${TEST_ENCODE_DECODE} PRIVATE ${RANS_TEST_ARCH}) - -o2_add_test(Serialize - NAME ransSerialize - SOURCES test/test_ransSerialize.cxx - PUBLIC_LINK_LIBRARIES O2::rANS - COMPONENT_NAME rANS - TARGETVARNAME TEST_SERIALIZE - LABELS utils) - target_compile_options(${TEST_SERIALIZE} PRIVATE ${RANS_TEST_ARCH}) - -if(${RANS_ENABLE_JSON}) - if (TARGET benchmark::benchmark) - - o2_add_header_only_library(libransBenchmark - TARGETVARNAME LIB_RANS_BENCHMARK - INTERFACE_LINK_LIBRARIES O2::rANS RapidJSON::RapidJSON benchmark::benchmark TBB::tbb) - target_compile_options(${LIB_RANS_BENCHMARK} INTERFACE -O3 ${RANS_OPTIONS}) - if(${ENABLE_VTUNE_PROFILER}) - target_compile_definitions(${LIB_RANS_BENCHMARK} INTERFACE ENABLE_VTUNE_PROFILER) - target_link_libraries(${LIB_RANS_BENCHMARK} INTERFACE PkgConfig::Vtune) - endif() - - o2_add_executable(CombinedIterator - SOURCES benchmarks/bench_ransCombinedIterator.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(Histogram - SOURCES benchmarks/bench_ransHistogram.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(Encode - SOURCES benchmarks/bench_ransEncode.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(Decode - SOURCES benchmarks/bench_ransDecode.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(DecodeScaling - SOURCES benchmarks/bench_ransDecodeScaling.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(EncodeImpl - SOURCES benchmarks/bench_ransEncodeImpl.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(Streaming - SOURCES benchmarks/bench_ransStreaming.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(Pack - SOURCES benchmarks/bench_ransPack.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - - o2_add_executable(UnPack - SOURCES benchmarks/bench_ransUnpack.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark) - o2_add_executable(TPCEncodeDecode - SOURCES benchmarks/bench_ransTPC.cxx - COMPONENT_NAME rANS - IS_BENCHMARK - PUBLIC_LINK_LIBRARIES O2::libransBenchmark Boost::program_options) - endif() +if (TARGET benchmark::benchmark) +o2_add_executable(CombinedIterator + SOURCES benchmarks/bench_ransCombinedIterator.cxx + COMPONENT_NAME rANS + IS_BENCHMARK + PUBLIC_LINK_LIBRARIES O2::rANS benchmark::benchmark) endif() o2_add_executable(rans-encode-decode-8 - TARGETVARNAME RANS_ENCODE_DECODE_8 + TARGETVARNAME targetName SOURCES run/bin-encode-decode.cxx PUBLIC_LINK_LIBRARIES O2::rANS Boost::program_options) -target_compile_definitions(${RANS_ENCODE_DECODE_8} PRIVATE -DSOURCE_T=uint8_t) -target_compile_options(${RANS_ENCODE_DECODE_8} PRIVATE -O3 ${RANS_TARGET_ARCH}) +target_compile_definitions(${targetName} PRIVATE -DSOURCE_T=uint8_t) o2_add_executable(rans-encode-decode-16 - TARGETVARNAME RANS_ENCODE_DECODE_16 + TARGETVARNAME targetName SOURCES run/bin-encode-decode.cxx PUBLIC_LINK_LIBRARIES O2::rANS Boost::program_options) -target_compile_definitions(${RANS_ENCODE_DECODE_16} PRIVATE -DSOURCE_T=uint16_t) -target_compile_options(${RANS_ENCODE_DECODE_16} PRIVATE -O3 ${RANS_TARGET_ARCH}) +target_compile_definitions(${targetName} PRIVATE -DSOURCE_T=uint16_t) o2_add_executable(rans-encode-decode-32 - TARGETVARNAME RANS_ENCODE_DECODE_32 + TARGETVARNAME targetName SOURCES run/bin-encode-decode.cxx PUBLIC_LINK_LIBRARIES O2::rANS Boost::program_options) -target_compile_definitions(${RANS_ENCODE_DECODE_32} PRIVATE -DSOURCE_T=uint32_t) -target_compile_options(${RANS_ENCODE_DECODE_32} PRIVATE -O3 ${RANS_TARGET_ARCH}) +target_compile_definitions(${targetName} PRIVATE -DSOURCE_T=uint32_t) diff --git a/Utilities/rANS/benchmarks/bench_ransCombinedIterator.cxx b/Utilities/rANS/benchmarks/bench_ransCombinedIterator.cxx index a26c3cbe1d769..dce88855518cf 100644 --- a/Utilities/rANS/benchmarks/bench_ransCombinedIterator.cxx +++ b/Utilities/rANS/benchmarks/bench_ransCombinedIterator.cxx @@ -1,4 +1,4 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. // See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. // All rights not expressly granted are reserved. // @@ -11,15 +11,14 @@ /// @file bench_ransCombinedIterator.cxx /// @author Michael Lettrich -/// @brief compares performance of on-the-fly merging of data vs copy and merge. +/// @since Nov 18, 2020 +/// @brief #include #include -#include "rANS/iterator.h" - -using namespace o2::rans; +#include "rANS/utils.h" static void BM_Array_Read_Copy(benchmark::State& state) { @@ -52,8 +51,8 @@ static void BM_Array_Read_Iterator(benchmark::State& state) return *iterB + (*iterA << 16); }; - const CombinedInputIterator begin(a.begin(), b.begin(), readOP); - const CombinedInputIterator end(a.end(), b.end(), readOP); + const o2::rans::utils::CombinedInputIterator begin(a.begin(), b.begin(), readOP); + const o2::rans::utils::CombinedInputIterator end(a.end(), b.end(), readOP); auto cIter = c.begin(); @@ -101,7 +100,7 @@ static void BM_Array_Write_Iterator(benchmark::State& state) *iterB = value & ((1 << shift) - 1); }; - auto out = CombinedOutputIteratorFactory::makeIter(a.begin(), b.begin(), writeOP); + auto out = o2::rans::utils::CombinedOutputIteratorFactory::makeIter(a.begin(), b.begin(), writeOP); for (auto iter = c.begin(); iter != c.end(); ++iter) { *out = *iter + 1; diff --git a/Utilities/rANS/benchmarks/bench_ransDecode.cxx b/Utilities/rANS/benchmarks/bench_ransDecode.cxx deleted file mode 100644 index 6e85d7c7cd50c..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransDecode.cxx +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransEncode.cxx -/// @author Michael Lettrich -/// @brief compares performance of different encoders - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/factory.h" -#include "rANS/histogram.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include "helpers.h" - -using namespace o2::rans; - -inline constexpr size_t MessageSize = 1ull << 22; - -// template -// class SourceMessageProxyBinomial -// { -// public: -// SourceMessageProxyBinomial(size_t messageSize) -// { -// if (mSourceMessage.empty()) { -// std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; -// const size_t draws = std::min(1ul << 27, static_cast(std::numeric_limits::max())); -// const double probability = 0.5; -// std::binomial_distribution dist(draws, probability); -// const size_t sourceSize = messageSize / sizeof(source_T) + 1; -// mSourceMessage.resize(sourceSize); -// #ifdef __cpp_lib_execution -// std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -// #else -// std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -// #endif -// } -// } - -// const auto& get() const { return mSourceMessage; }; - -// private: -// std::vector mSourceMessage{}; -// }; - -// inline const SourceMessageProxyBinomial sourceMessageBinomial8{MessageSize}; -// inline const SourceMessageProxyBinomial sourceMessageBinomial16{MessageSize}; -// inline const SourceMessageProxyBinomial sourceMessageBinomial32{MessageSize}; - -template -class SourceMessageProxyUniform -{ - public: - SourceMessageProxyUniform(size_t messageSize) - { - if (mSourceMessage.empty()) { - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - const size_t min = 0; - const double max = std::min(1ul << 27, static_cast(std::numeric_limits::max())); - std::uniform_int_distribution dist(min, max); - const size_t sourceSize = messageSize / sizeof(source_T) + 1; - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - } - } - - const auto& get() const { return mSourceMessage; }; - - private: - std::vector mSourceMessage{}; -}; - -inline const SourceMessageProxyUniform sourceMessageUniform8{MessageSize}; -inline const SourceMessageProxyUniform sourceMessageUniform16{MessageSize}; -inline const SourceMessageProxyUniform sourceMessageUniform32{MessageSize}; - -template -void ransDecodeBenchmark(benchmark::State& st, Args&&... args) -{ - - auto args_tuple = std::make_tuple(std::move(args)...); - - const auto& inputData = std::get<0>(args_tuple).get(); - - using input_data_type = std::remove_cv_t>; - using source_type = typename input_data_type::value_type; - - EncodeBuffer encodeBuffer{inputData.size()}; - DecodeBuffer decodeBuffer{inputData.size()}; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - const auto renormedHistogram = renorm(histogram, metrics, RenormingPolicy::Auto, 10); - - auto encoder = makeDenseEncoder<>::fromRenormed(renormedHistogram); - encodeBuffer.encodeBufferEnd = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data()); - - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams()); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - if (!(decodeBuffer == inputData)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = metrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["SymbolTablePrecision"] = renormedHistogram.getRenormingBits(); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["ExpectedCodewordLength"] = computeExpectedCodewordLength(histogram, renormedHistogram); - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["CompressedSize"] = std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(typename decltype(encoder)::stream_type); - st.counters["Compression"] = st.counters["SourceSize"] / static_cast(st.counters["CompressedSize"]); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); - st.counters["CompressionWRTEntropy"] = st.counters["CompressedSize"] / st.counters["LowerBound"]; -}; - -// BENCHMARK_CAPTURE(ransDecodeBenchmark, decode_binomial_8, sourceMessageBinomial8); -// BENCHMARK_CAPTURE(ransDecodeBenchmark, decode_binomial_16, sourceMessageBinomial16); -// BENCHMARK_CAPTURE(ransDecodeBenchmark, decode_binomial_32, sourceMessageBinomial32); - -BENCHMARK_CAPTURE(ransDecodeBenchmark, decode_uniform_8, sourceMessageUniform8); -BENCHMARK_CAPTURE(ransDecodeBenchmark, decode_uniform_16, sourceMessageUniform16); -BENCHMARK_CAPTURE(ransDecodeBenchmark, decode_uniform_32, sourceMessageUniform32); - -BENCHMARK_MAIN(); \ No newline at end of file diff --git a/Utilities/rANS/benchmarks/bench_ransDecodeScaling.cxx b/Utilities/rANS/benchmarks/bench_ransDecodeScaling.cxx deleted file mode 100644 index 4b8c1c01f98be..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransDecodeScaling.cxx +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransEncode.cxx -/// @author Michael Lettrich -/// @brief compares performance of different encoders - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/factory.h" -#include "rANS/histogram.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include "helpers.h" - -using namespace o2::rans; - -inline constexpr size_t MessageSize = 1ull << 22; - -template -class SourceMessageUniform -{ - public: - SourceMessageUniform(size_t messageSize, size_t max) : mMax{max} - { - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - std::uniform_int_distribution dist(0, max); - const size_t sourceSize = messageSize / sizeof(source_T) + 1; - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - } - - const auto& get() const { return mSourceMessage; }; - size_t getMax() const { return mMax; }; - - private: - size_t mMax{}; - std::vector mSourceMessage{}; -}; - -SourceMessageUniform sourceMessage{0, 0}; - -void ransDecodeBenchmark(benchmark::State& st) -{ - - using source_type = uint32_t; - size_t max = utils::pow2(st.range(0)); - - if (max != sourceMessage.getMax()) { - sourceMessage = SourceMessageUniform{MessageSize, max}; - } - const auto& inputData = sourceMessage.get(); - EncodeBuffer - encodeBuffer{inputData.size()}; - DecodeBuffer decodeBuffer{inputData.size()}; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - const auto renormedHistogram = renorm(histogram, metrics, RenormingPolicy::Auto, 10); - - auto encoder = makeDenseEncoder<>::fromRenormed(renormedHistogram); - encodeBuffer.encodeBufferEnd = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data()); - - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams()); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - if (!(decodeBuffer == inputData)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = metrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["SymbolTablePrecision"] = renormedHistogram.getRenormingBits(); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["ExpectedCodewordLength"] = computeExpectedCodewordLength(histogram, renormedHistogram); - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["CompressedSize"] = std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(typename decltype(encoder)::stream_type); - st.counters["Compression"] = st.counters["SourceSize"] / static_cast(st.counters["CompressedSize"]); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); - st.counters["CompressionWRTEntropy"] = st.counters["CompressedSize"] / st.counters["LowerBound"]; -}; - -BENCHMARK(ransDecodeBenchmark)->DenseRange(8, 27, 1); - -BENCHMARK_MAIN(); \ No newline at end of file diff --git a/Utilities/rANS/benchmarks/bench_ransEncode.cxx b/Utilities/rANS/benchmarks/bench_ransEncode.cxx deleted file mode 100644 index 67eb78dfdb127..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransEncode.cxx +++ /dev/null @@ -1,351 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransEncode.cxx -/// @author Michael Lettrich -/// @brief compares performance of different encoders - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/factory.h" -#include "rANS/histogram.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include "helpers.h" - -using namespace o2::rans; - -inline constexpr size_t MessageSize = 1ull << 22; - -template -class SourceMessageProxy -{ - public: - SourceMessageProxy(size_t messageSize) - { - if (mSourceMessage.empty()) { - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - const size_t draws = std::min(utils::pow2(20), static_cast(std::numeric_limits::max())); - const double probability = 0.5; - std::binomial_distribution dist(draws, probability); - const size_t sourceSize = messageSize / sizeof(source_T) + 1; - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - } - } - - const auto& get() const { return mSourceMessage; }; - - private: - std::vector mSourceMessage{}; -}; - -inline const SourceMessageProxy sourceMessage8{MessageSize}; -inline const SourceMessageProxy sourceMessage16{MessageSize}; -inline const SourceMessageProxy sourceMessage32{MessageSize}; - -template -const auto& getMessage() -{ - if constexpr (std::is_same_v) { - return sourceMessage8.get(); - } else if constexpr (std::is_same_v) { - return sourceMessage16.get(); - } else { - return sourceMessage32.get(); - } -}; - -template -void ransCompressionBenchmark(benchmark::State& st) -{ - using source_type = source_T; - - const auto& inputData = getMessage(); - EncodeBuffer encodeBuffer{inputData.size()}; - DecodeBuffer decodeBuffer{inputData.size()}; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - const auto renormedHistogram = renorm(histogram, metrics, RenormingPolicy::Auto, 10); - auto encoder = makeDenseEncoder::fromRenormed(renormedHistogram); - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - benchmark::DoNotOptimize(encodeBuffer.encodeBufferEnd = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data())); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams()); - if (!(decodeBuffer == inputData)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = metrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["SymbolTablePrecision"] = renormedHistogram.getRenormingBits(); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["ExpectedCodewordLength"] = computeExpectedCodewordLength(histogram, renormedHistogram); - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["CompressedSize"] = std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(typename decltype(encoder)::stream_type); - st.counters["Compression"] = st.counters["SourceSize"] / static_cast(st.counters["CompressedSize"]); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); - st.counters["CompressionWRTEntropy"] = st.counters["CompressedSize"] / st.counters["LowerBound"]; -}; - -template -void ransLiteralCompressionBenchmark(benchmark::State& st) -{ - using source_type = source_T; - - const auto& inputData = getMessage(); - EncodeBuffer encodeBuffer{inputData.size()}; - encodeBuffer.literals.resize(inputData.size(), 0); - encodeBuffer.literalsEnd = encodeBuffer.literals.data(); - DecodeBuffer decodeBuffer{inputData.size()}; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - const auto renormedHistogram = renorm(histogram, metrics); - auto encoder = makeDenseEncoder::fromRenormed(renormedHistogram); - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - encodeBuffer.literalsEnd = encodeBuffer.literals.data(); - benchmark::DoNotOptimize(std::tie(encodeBuffer.encodeBufferEnd, encodeBuffer.literalsEnd) = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data(), encodeBuffer.literalsEnd)); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams(), encodeBuffer.literalsEnd); - if (!(decodeBuffer == inputData)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = metrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["SymbolTablePrecision"] = renormedHistogram.getRenormingBits(); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["ExpectedCodewordLength"] = computeExpectedCodewordLength(histogram, renormedHistogram); - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["CompressedSize"] = std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(typename decltype(encoder)::stream_type); - st.counters["Compression"] = st.counters["SourceSize"] / static_cast(st.counters["CompressedSize"]); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); - st.counters["CompressionWRTEntropy"] = st.counters["CompressedSize"] / st.counters["LowerBound"]; -}; - -template -void ransAdaptiveCompressionBenchmark(benchmark::State& st) -{ - using source_type = source_T; - - const auto& inputData = getMessage(); - EncodeBuffer encodeBuffer{inputData.size()}; - DecodeBuffer decodeBuffer{inputData.size()}; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - auto adaptiveHistogram = makeAdaptiveHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - Metrics adaptiveMetrics{adaptiveHistogram}; - const auto renormedHistogram = renorm(histogram, metrics, RenormingPolicy::Auto, 10); - const auto renormedAdaptiveHistogram = renorm(std::move(adaptiveHistogram), adaptiveMetrics, RenormingPolicy::Auto, 10); - - auto encoder = makeAdaptiveEncoder::fromRenormed(renormedAdaptiveHistogram); - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - benchmark::DoNotOptimize(encodeBuffer.encodeBufferEnd = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data())); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams()); - if (!(decodeBuffer == inputData)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = adaptiveMetrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["SymbolTablePrecision"] = renormedAdaptiveHistogram.getRenormingBits(); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["ExpectedCodewordLength"] = computeExpectedCodewordLength(histogram, renormedHistogram); - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["CompressedSize"] = std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(typename decltype(encoder)::stream_type); - st.counters["Compression"] = st.counters["SourceSize"] / static_cast(st.counters["CompressedSize"]); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); - st.counters["CompressionWRTEntropy"] = st.counters["CompressedSize"] / st.counters["LowerBound"]; -}; - -template -void ransAdaptiveLiteralCompressionBenchmark(benchmark::State& st) -{ - using source_type = source_T; - - const auto& inputData = getMessage(); - EncodeBuffer encodeBuffer{inputData.size()}; - encodeBuffer.literals.resize(inputData.size(), 0); - encodeBuffer.literalsEnd = encodeBuffer.literals.data(); - DecodeBuffer decodeBuffer{inputData.size()}; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - auto adaptiveHistogram = makeAdaptiveHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - Metrics adaptiveMetrics{adaptiveHistogram}; - const auto renormedHistogram = renorm(histogram, metrics); - const auto renormedAdaptiveHistogram = renorm(std::move(adaptiveHistogram), adaptiveMetrics); - - auto encoder = makeAdaptiveEncoder::fromRenormed(renormedAdaptiveHistogram); - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - encodeBuffer.literalsEnd = encodeBuffer.literals.data(); - benchmark::DoNotOptimize(std::tie(encodeBuffer.encodeBufferEnd, encodeBuffer.literalsEnd) = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data(), encodeBuffer.literalsEnd)); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams(), encodeBuffer.literalsEnd); - if (!(decodeBuffer == inputData)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = adaptiveMetrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["SymbolTablePrecision"] = renormedAdaptiveHistogram.getRenormingBits(); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["ExpectedCodewordLength"] = computeExpectedCodewordLength(histogram, renormedHistogram); - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["CompressedSize"] = std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(typename decltype(encoder)::stream_type); - st.counters["Compression"] = st.counters["SourceSize"] / static_cast(st.counters["CompressedSize"]); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); - st.counters["CompressionWRTEntropy"] = st.counters["CompressedSize"] / st.counters["LowerBound"]; -}; - -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); - -BENCHMARK(ransAdaptiveCompressionBenchmark); -//######################################################################################## - -#ifdef RANS_SINGLE_STREAM -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); - -BENCHMARK(ransAdaptiveCompressionBenchmark); -#endif /* RANS_SINGLE_STREAM */ - -//######################################################################################## - -#ifdef RANS_SSE -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); - -BENCHMARK(ransAdaptiveCompressionBenchmark); -#endif /* RANS SSE */ - -// //######################################################################################## - -#ifdef RANS_AVX2 -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); -BENCHMARK(ransCompressionBenchmark); - -BENCHMARK(ransAdaptiveCompressionBenchmark); -#endif /* RANS_AVX2 */ - -//######################################################################################## - -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); - -BENCHMARK(ransAdaptiveLiteralCompressionBenchmark); - -//######################################################################################## - -#ifdef RANS_SINGLE_STREAM -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); - -BENCHMARK(ransAdaptiveLiteralCompressionBenchmark); -#endif /* RANS_SINGLE_STREAM */ - -//######################################################################################## - -#ifdef RANS_SSE -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); - -BENCHMARK(ransAdaptiveLiteralCompressionBenchmark); -#endif /* RANS_SSE */ - -//######################################################################################## - -#ifdef RANS_AVX2 -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); -BENCHMARK(ransLiteralCompressionBenchmark); - -BENCHMARK(ransAdaptiveLiteralCompressionBenchmark); -#endif /* RANS_AVX2 */ - -BENCHMARK_MAIN(); \ No newline at end of file diff --git a/Utilities/rANS/benchmarks/bench_ransEncodeImpl.cxx b/Utilities/rANS/benchmarks/bench_ransEncodeImpl.cxx deleted file mode 100644 index 1896d5b4ffe81..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransEncodeImpl.cxx +++ /dev/null @@ -1,379 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransEncodeImpl.cxx -/// @author Michael Lettrich -/// @brief benchmarks different encoding kernels without the need to renorm and stream data - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include -#include - -#include "rANS/factory.h" -#include "rANS/histogram.h" - -#include "rANS/internal/common/utils.h" -#include "rANS/internal/common/simdtypes.h" -#include "rANS/internal/common/simdops.h" -#include "rANS/internal/encode/simdKernel.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -using count_t = uint32_t; -using ransState_t = uint64_t; -using stream_t = uint32_t; - -#ifdef RANS_SINGLE_STREAM -__extension__ using uint128_t = unsigned __int128; -#endif /* RANS_SINGLE_STREAM */ - -using namespace o2::rans; -using namespace o2::rans::internal; -using namespace o2::rans::utils; - -inline constexpr size_t MessageSize = 1ull << 22; -inline constexpr size_t LowerBound = 1ul << 20; -inline constexpr size_t StreamBits = toBits(); - -template -class SymbolTableData -{ - public: - explicit SymbolTableData(size_t messageSize) - { - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - const size_t draws = std::min(1ul << 20, static_cast(std::numeric_limits::max())); - const double probability = 0.5; - std::binomial_distribution dist(draws, probability); - const size_t sourceSize = messageSize / sizeof(source_T); - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(mSourceMessage)); - Metrics metrics{histogram}; - mRenormedFrequencies = renorm(histogram, metrics); - - double_t expectationValue = std::accumulate(mRenormedFrequencies.begin(), mRenormedFrequencies.end(), 0.0, [this](const double_t& a, const count_t& b) { - double_t prb = static_cast(b) / static_cast(mRenormedFrequencies.getNumSamples()); - return a + b * prb; - }); - - mState = ((LowerBound >> mRenormedFrequencies.getRenormingBits()) << StreamBits) * expectationValue; - }; - - const auto& getSourceMessage() const { return mSourceMessage; }; - const auto& getRenormedFrequencies() const { return mRenormedFrequencies; }; - - ransState_t getState() const { return mState; }; - - private: - std::vector mSourceMessage{}; - RenormedDenseHistogram mRenormedFrequencies{}; - ransState_t mState{}; -}; - -const SymbolTableData Data8(MessageSize); -const SymbolTableData Data16(MessageSize); -const SymbolTableData Data32(MessageSize); - -template -const auto& getData() -{ - if constexpr (std::is_same_v) { - return Data8; - } else if constexpr (std::is_same_v) { - return Data16; - } else { - return Data32; - } -}; - -template -struct SimpleFixture : public benchmark::Fixture { - using source_t = source_T; - using symbol_t = Symbol; - - void SetUp(const ::benchmark::State& state) final - { - const auto& sourceMessage = getData().getSourceMessage(); - DenseSymbolTable symbolTable{getData().getRenormedFrequencies()}; - for (auto& symbol : sourceMessage) { - mSymbols.push_back(symbolTable[symbol]); - } - } - - void TearDown(const ::benchmark::State& state) final - { - mSymbols.clear(); - } - std::vector mSymbols{}; - ransState_t mState = getData().getState(); - size_t mRenormingBits = getData().getRenormedFrequencies().getRenormingBits(); -}; - -template -struct Fixture : public benchmark::Fixture { - using source_t = source_T; - using symbol_t = PrecomputedSymbol; - - void SetUp(const ::benchmark::State& state) final - { - const auto& sourceMessage = getData().getSourceMessage(); - DenseSymbolTable symbolTable{getData().getRenormedFrequencies()}; - for (auto& symbol : sourceMessage) { - mSymbols.push_back(symbolTable[symbol]); - } - } - - void TearDown(const ::benchmark::State& state) final - { - mSymbols.clear(); - } - std::vector mSymbols{}; - ransState_t mState = getData().getState(); - size_t mRenormingBits = getData().getRenormedFrequencies().getRenormingBits(); -}; - -#ifdef RANS_SIMD - -template -struct SIMDFixture : public benchmark::Fixture { - using source_t = source_T; - using symbol_t = Symbol; - - void SetUp(const ::benchmark::State& state) final - { - mState = simd::setAll(getData().getState()); - mNSamples = simd::setAll(static_cast(pow2(getData().getRenormedFrequencies().getRenormingBits()))); - - const auto& sourceMessage = getData().getSourceMessage(); - DenseSymbolTable symbolTable{getData().getRenormedFrequencies()}; - for (size_t i = 0; i < sourceMessage.size(); i += nElems) { - if constexpr (width_V == simd::SIMDWidth::SSE) { - mSymbols.push_back({ - &symbolTable[sourceMessage[i]], - &symbolTable[sourceMessage[i + 1]], - }); - } - if constexpr (width_V == simd::SIMDWidth::AVX) { - mSymbols.push_back({ - &symbolTable[sourceMessage[i]], - &symbolTable[sourceMessage[i + 1]], - &symbolTable[sourceMessage[i + 2]], - &symbolTable[sourceMessage[i + 3]], - }); - } - } - } - - void TearDown(const ::benchmark::State& state) final - { - mSymbols.clear(); - } - - static constexpr size_t nElems = simd::getElementCount(width_V); - std::vector> mSymbols{}; - simd::simdI_t mState; - simd::simdD_t mNSamples; -}; -#endif /* RANS_SIMD */ - -#ifdef RANS_SINGLE_STREAM -ransState_t encode(ransState_t state, const PrecomputedSymbol& symbol) -{ - // x = C(s,x) - ransState_t quotient = static_cast((static_cast(state) * symbol.getReciprocalFrequency()) >> 64); - quotient = quotient >> symbol.getReciprocalShift(); - - return state + symbol.getCumulative() + quotient * symbol.getFrequencyComplement(); -}; -#endif /* RANS_SINGLE_STREAM */ - -ransState_t simpleEncode(ransState_t state, size_t symbolTablePrecision, const Symbol& symbol) -{ - // x = C(s,x) - return ((state / symbol.getFrequency()) << symbolTablePrecision) + symbol.getCumulative() + (state % symbol.getFrequency()); -}; - -#ifdef RANS_SIMD -template -inline auto SIMDEncode(simd::simdI_t states, simd::simdD_t nSamples, gsl::span(width_V)> symbols) -{ - simd::simdIsse_t frequencies; - simd::simdIsse_t cumulativeFrequencies; - simd::aosToSoa(symbols, &frequencies, &cumulativeFrequencies); - return simd::ransEncode(states, simd::int32ToDouble(frequencies), simd::int32ToDouble(cumulativeFrequencies), nSamples); -}; -#endif /* RANS_SIMD */ - -template -static void ransSimpleEncodeBenchmark(benchmark::State& st, SimpleFixture& fixture) -{ - for (auto _ : st) { - for (size_t i = 0; i < fixture.mSymbols.size(); ++i) { - ransState_t newState = simpleEncode(fixture.mState, fixture.mRenormingBits, fixture.mSymbols[i]); - benchmark::DoNotOptimize(newState); - } - }; - - st.SetItemsProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size()); - st.SetBytesProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size() * sizeof(source_T)); -}; - -#ifdef RANS_SINGLE_STREAM -template -static void ransEncodeBenchmark(benchmark::State& st, Fixture& fixture) -{ - for (auto _ : st) { - for (size_t i = 0; i < fixture.mSymbols.size(); ++i) { - ransState_t newState = encode(fixture.mState, fixture.mSymbols[i]); - benchmark::DoNotOptimize(newState); - } - }; - - st.SetItemsProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size()); - st.SetBytesProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size() * sizeof(source_T)); -}; -#endif /* RANS_SINGLE_STREAM */ - -#ifdef RANS_SIMD -template -static void ransSIMDEncodeBenchmark(benchmark::State& st, SIMDFixture& fixture) -{ -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - for (size_t i = 0; i < fixture.mSymbols.size(); ++i) { - auto newStates = SIMDEncode(fixture.mState, fixture.mNSamples, fixture.mSymbols[i]); - benchmark::DoNotOptimize(newStates); - benchmark::ClobberMemory(); - } - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - st.SetItemsProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size()); - st.SetBytesProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size() * sizeof(source_T)); -}; -#endif /* RANS_SIMD */ - -BENCHMARK_TEMPLATE_DEFINE_F(SimpleFixture, simpleEncode_8, uint8_t) -(benchmark::State& st) -{ - ransSimpleEncodeBenchmark(st, *this); -}; -BENCHMARK_TEMPLATE_DEFINE_F(SimpleFixture, simpleEncode_16, uint16_t) -(benchmark::State& st) -{ - ransSimpleEncodeBenchmark(st, *this); -}; -BENCHMARK_TEMPLATE_DEFINE_F(SimpleFixture, simpleEncode_32, uint32_t) -(benchmark::State& st) -{ - ransSimpleEncodeBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(Fixture, encode_8, uint8_t) -(benchmark::State& st) -{ - ransEncodeBenchmark(st, *this); -}; -BENCHMARK_TEMPLATE_DEFINE_F(Fixture, encode_16, uint16_t) -(benchmark::State& st) -{ - ransEncodeBenchmark(st, *this); -}; -BENCHMARK_TEMPLATE_DEFINE_F(Fixture, encode_32, uint32_t) -(benchmark::State& st) -{ - ransEncodeBenchmark(st, *this); -}; - -#ifdef RANS_SSE -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, encodeSSE_8, uint8_t, simd::SIMDWidth::SSE) -(benchmark::State& st) -{ - ransSIMDEncodeBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, encodeSSE_16, uint16_t, simd::SIMDWidth::SSE) -(benchmark::State& st) -{ - ransSIMDEncodeBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, encodeSSE_32, uint32_t, simd::SIMDWidth::SSE) -(benchmark::State& st) -{ - ransSIMDEncodeBenchmark(st, *this); -}; -#endif /*RANS_SSE*/ - -#ifdef RANS_AVX2 -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, encodeAVX_8, uint8_t, simd::SIMDWidth::AVX) -(benchmark::State& st) -{ - ransSIMDEncodeBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, encodeAVX_16, uint16_t, simd::SIMDWidth::AVX) -(benchmark::State& st) -{ - ransSIMDEncodeBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, encodeAVX_32, uint32_t, simd::SIMDWidth::AVX) -(benchmark::State& st) -{ - ransSIMDEncodeBenchmark(st, *this); -}; -#endif /* RANS_AVX2 */ - -BENCHMARK_REGISTER_F(SimpleFixture, simpleEncode_8); -BENCHMARK_REGISTER_F(SimpleFixture, simpleEncode_16); -BENCHMARK_REGISTER_F(SimpleFixture, simpleEncode_32); - -#ifdef RANS_SINGLE_STREAM -BENCHMARK_REGISTER_F(Fixture, encode_8); -BENCHMARK_REGISTER_F(Fixture, encode_16); -BENCHMARK_REGISTER_F(Fixture, encode_32); -#endif /* RANS_SINGLE_STREAM */ - -#ifdef RANS_SSE -BENCHMARK_REGISTER_F(SIMDFixture, encodeSSE_8); -BENCHMARK_REGISTER_F(SIMDFixture, encodeSSE_16); -BENCHMARK_REGISTER_F(SIMDFixture, encodeSSE_32); -#endif /* RANS_SSE */ - -#ifdef RANS_AVX2 -BENCHMARK_REGISTER_F(SIMDFixture, encodeAVX_8); -BENCHMARK_REGISTER_F(SIMDFixture, encodeAVX_16); -BENCHMARK_REGISTER_F(SIMDFixture, encodeAVX_32); -#endif /* RANS_SSE */ - -BENCHMARK_MAIN(); diff --git a/Utilities/rANS/benchmarks/bench_ransHistogram.cxx b/Utilities/rANS/benchmarks/bench_ransHistogram.cxx deleted file mode 100644 index ca925ea742bdb..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransHistogram.cxx +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransEncode.cxx -/// @author Michael Lettrich -/// @brief compares performance of different encoders - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/factory.h" -#include "rANS/internal/containers/DenseHistogram.h" -#include "rANS/internal/containers/AdaptiveHistogram.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include "helpers.h" - -using namespace o2::rans; - -inline constexpr size_t MessageSize = 1ull << 22; - -template -class SourceMessageProxyBinomial -{ - public: - SourceMessageProxyBinomial() = default; - - const auto& get(size_t messageSize) - { - if (mSourceMessage.empty()) { - LOG(info) << "generating Binomial distribution"; - - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - const size_t draws = std::min(1ul << 27, static_cast(std::numeric_limits::max())); - const double probability = 0.5; - std::binomial_distribution dist(draws, probability); - const size_t sourceSize = messageSize / sizeof(source_T) + 1; - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - } - - return mSourceMessage; - }; - - private: - std::vector mSourceMessage{}; -}; - -inline SourceMessageProxyBinomial sourceMessageBinomial32{}; - -template -class SourceMessageProxyUniform -{ - public: - SourceMessageProxyUniform() = default; - - const auto& get(size_t messageSize) - { - if (mSourceMessage.empty()) { - LOG(info) << "generating Uniform distribution"; - - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - const size_t min = 0; - const double max = std::min(1ul << 27, static_cast(std::numeric_limits::max())); - std::uniform_int_distribution dist(min, max); - const size_t sourceSize = messageSize / sizeof(source_T) + 1; - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - } - return mSourceMessage; - }; - - private: - std::vector mSourceMessage{}; -}; - -inline SourceMessageProxyUniform sourceMessageUniform32{}; - -template -void ransMakeHistogramBenchmark(benchmark::State& st, Args&&... args) -{ - - auto args_tuple = std::make_tuple(std::move(args)...); - - const auto& inputData = std::get<0>(args_tuple).get(MessageSize); - - using histogram_type = std::remove_cv_t(args_tuple))>>; - using input_data_type = std::remove_cv_t>; - using source_type = typename input_data_type::value_type; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - histogram_type hist{}; - // hist.addSamples(gsl::span(inputData)); - hist.addSamples(inputData.begin(), inputData.end()); - benchmark::DoNotOptimize(hist); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - bool isSame = true; - histogram_type hist{}; - hist.addSamples(gsl::span(inputData)); - - for (std::ptrdiff_t symbol = histogram.getOffset(); symbol != histogram.getOffset() + histogram.size(); ++symbol) { - if (histogram[symbol] > 0) { - LOG_IF(info, histogram[symbol] != hist[symbol]) << fmt::format("[{}]: {} != {}", symbol, hist[symbol], histogram[symbol]); - isSame = isSame && (histogram[symbol] == hist[symbol]); - } - } - - if (!(isSame)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = metrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["HistogramSize"] = hist.size() * sizeof(source_type); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); -}; - -template -void ransAccessHistogramBenchmark(benchmark::State& st, Args&&... args) -{ - - auto args_tuple = std::make_tuple(std::move(args)...); - const auto& inputData = std::get<0>(args_tuple).get(MessageSize); - - using histogram_type = std::remove_cv_t(args_tuple))>>; - using input_data_type = std::remove_cv_t>; - using source_type = typename input_data_type::value_type; - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(inputData)); - Metrics metrics{histogram}; - - histogram_type hist{}; - hist.addSamples(gsl::span(inputData)); - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - for (auto& symbol : inputData) { - const uint32_t t = hist[symbol]; - benchmark::DoNotOptimize(t); - } - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - bool isSame = true; - for (std::ptrdiff_t symbol = histogram.getOffset(); symbol != histogram.getOffset() + histogram.size(); ++symbol) { - if (histogram[symbol] > 0) { - LOG_IF(info, histogram[symbol] != hist[symbol]) << fmt::format("[{}]: {} != {}", symbol, hist[symbol], histogram[symbol]); - isSame = isSame && (histogram[symbol] == hist[symbol]); - } - } - - if (!(isSame)) { - st.SkipWithError("Missmatch between encoded and decoded Message"); - } - - const auto& datasetProperties = metrics.getDatasetProperties(); - st.SetItemsProcessed(static_cast(inputData.size()) * static_cast(st.iterations())); - st.SetBytesProcessed(static_cast(inputData.size()) * sizeof(source_type) * static_cast(st.iterations())); - st.counters["AlphabetRangeBits"] = datasetProperties.alphabetRangeBits; - st.counters["nUsedAlphabetSymbols"] = datasetProperties.nUsedAlphabetSymbols; - st.counters["HistogramSize"] = hist.size() * sizeof(source_type); - st.counters["Entropy"] = datasetProperties.entropy; - st.counters["SourceSize"] = inputData.size() * sizeof(source_type); - st.counters["LowerBound"] = inputData.size() * (static_cast(st.counters["Entropy"]) / 8); -}; - -BENCHMARK_CAPTURE(ransMakeHistogramBenchmark, makeHistogram_Vector_binomial_32, std::reference_wrapper(sourceMessageBinomial32), DenseHistogram{}); -BENCHMARK_CAPTURE(ransMakeHistogramBenchmark, makeHistogram_Vector_uniform_32, std::reference_wrapper(sourceMessageUniform32), DenseHistogram{}); - -BENCHMARK_CAPTURE(ransMakeHistogramBenchmark, makeHistogram_SparseVector_binomial_32, std::reference_wrapper(sourceMessageBinomial32), AdaptiveHistogram{}); -BENCHMARK_CAPTURE(ransMakeHistogramBenchmark, makeHistogram_SparseVector_uniform_32, std::reference_wrapper(sourceMessageUniform32), AdaptiveHistogram{}); - -BENCHMARK_CAPTURE(ransAccessHistogramBenchmark, accessHistogram_Vector_binomial_32, std::reference_wrapper(sourceMessageBinomial32), DenseHistogram{}); -BENCHMARK_CAPTURE(ransAccessHistogramBenchmark, accessHistogram_Vector_uniform_32, std::reference_wrapper(sourceMessageUniform32), DenseHistogram{}); - -BENCHMARK_CAPTURE(ransAccessHistogramBenchmark, accessHistogram_SparseVector_binomial_32, std::reference_wrapper(sourceMessageBinomial32), AdaptiveHistogram{}); -BENCHMARK_CAPTURE(ransAccessHistogramBenchmark, accessHistogram_SparseVector_uniform_32, std::reference_wrapper(sourceMessageUniform32), AdaptiveHistogram{}); - -BENCHMARK_MAIN(); \ No newline at end of file diff --git a/Utilities/rANS/benchmarks/bench_ransPack.cxx b/Utilities/rANS/benchmarks/bench_ransPack.cxx deleted file mode 100644 index 188ab481dd4ab..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransPack.cxx +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransPack.cxx -/// @author Michael Lettrich -/// @brief benchmarks packing algorithms and compares it to memcpy - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/internal/pack/pack.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include "helpers.h" - -using namespace o2::rans; - -inline constexpr size_t MessageSize = 1ull << 22; - -using source_type = uint32_t; - -template -std::vector makeRandomUniformVector(size_t nelems, source_T min = std::numeric_limits::max(), source_T max = std::numeric_limits::max()) -{ - std::vector result(nelems, 0); - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - std::uniform_int_distribution dist(min, max); - -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, result.begin(), result.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(result.begin(), result.end(), [&dist, &mt]() { return dist(mt); }); -#endif - - return result; -}; - -static void copyBenchmark(benchmark::State& state) -{ - std::vector src = makeRandomUniformVector(MessageSize); - std::vector dst(MessageSize, 0); - for (auto _ : state) { - std::copy(src.begin(), src.end(), dst.begin()); - }; - state.SetItemsProcessed(src.size() * state.iterations()); - state.SetBytesProcessed(src.size() * sizeof(source_type) * state.iterations()); -}; - -static void packingBenchmark(benchmark::State& state) -{ - size_t packingBits = state.range(0); - - std::vector src = makeRandomUniformVector(MessageSize, 0, utils::pow2(packingBits) - 1); - std::vector dst(MessageSize, 0); - for (auto _ : state) { - BitPtr iter{dst.data()}; - for (auto i : src) { - iter = internal::pack(iter, i, packingBits); - } - }; - state.SetItemsProcessed(src.size() * state.iterations()); - state.SetBytesProcessed(src.size() * sizeof(uint32_t) * state.iterations()); - - std::vector unpacked(MessageSize, 0); - - BitPtr iter{dst.data()}; - for (size_t i = 0; i < src.size(); ++i) { - unpacked[i] = internal::unpack(iter, packingBits); - iter += packingBits; - } - if (!std::equal(unpacked.begin(), unpacked.end(), src.begin())) { - state.SkipWithError("error in packing"); - } -}; - -static void fastPackBenchmark(benchmark::State& state) -{ - size_t packingBits = state.range(0); - - std::vector src = makeRandomUniformVector(MessageSize, 0, utils::pow2(packingBits) - 1); - std::vector dst(MessageSize, 0); -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : state) { - pack(src.data(), src.size(), dst.data(), packingBits, 0u); - }; -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - state.SetItemsProcessed(src.size() * state.iterations()); - state.SetBytesProcessed(src.size() * sizeof(uint32_t) * state.iterations()); - - std::vector unpacked(MessageSize, 0); - - BitPtr iter{dst.data()}; - for (size_t i = 0; i < src.size(); ++i) { - unpacked[i] = internal::unpack(iter, packingBits); - // LOGP(info, "[{}]{:0" + std::to_string(packingBits) + "b}", i, unpacked[i]); - iter += packingBits; - } - size_t i = 0; - if (!std::equal(unpacked.begin(), unpacked.end(), src.begin(), [&i](auto a, auto b) -> bool { - if (a != b) { - LOGP(info, "[{}]{:0x}!={:0x}", i++, a, b); - } - return a == b; - })) { - state.SkipWithError("error in packing"); - } -}; - -BENCHMARK(copyBenchmark); -BENCHMARK(packingBenchmark)->DenseRange(1, 32, 1); -BENCHMARK(fastPackBenchmark)->DenseRange(1, 32, 1); -BENCHMARK_MAIN(); diff --git a/Utilities/rANS/benchmarks/bench_ransStreaming.cxx b/Utilities/rANS/benchmarks/bench_ransStreaming.cxx deleted file mode 100644 index 04e592aea2403..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransStreaming.cxx +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransStreaming.cxx -/// @author Michael Lettrich -/// @brief benchmarks streaming out data from rANS state to memory - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/factory.h" -#include "rANS/histogram.h" - -#include "rANS/internal/common/utils.h" -#include "rANS/internal/common/simdtypes.h" -#include "rANS/internal/common/simdops.h" -#include "rANS/internal/encode/simdKernel.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -using count_t = uint32_t; -using ransState_t = uint64_t; -using stream_t = uint32_t; - -using namespace o2::rans; -using namespace o2::rans::internal; -using namespace o2::rans::utils; - -inline constexpr size_t MessageSize = 1ull << 22; -inline constexpr size_t LowerBound = 1ul << 20; -inline constexpr size_t StreamBits = toBits(); - -template -class RenormingData -{ - public: - explicit RenormingData(size_t messageSize) - { - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - const size_t draws = std::min(1ul << 20, static_cast(std::numeric_limits::max())); - const double probability = 0.5; - std::binomial_distribution dist(draws, probability); - const size_t sourceSize = messageSize / sizeof(source_T); - mSourceMessage.resize(sourceSize); -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(mSourceMessage.begin(), mSourceMessage.end(), [&dist, &mt]() { return dist(mt); }); -#endif - - const auto histogram = makeDenseHistogram::fromSamples(gsl::span(mSourceMessage)); - Metrics metrics{histogram}; - mRenormedHistogram = renorm(histogram, metrics); - - double_t expectationValue = std::accumulate(mRenormedHistogram.begin(), mRenormedHistogram.end(), 0.0, [this](const double_t& a, const count_t& b) { - double_t prb = static_cast(b) / static_cast(mRenormedHistogram.getNumSamples()); - return a + b * prb; - }); - - mState = ((LowerBound >> mRenormedHistogram.getRenormingBits()) << StreamBits) * expectationValue; - }; - - const auto& getSourceMessage() const { return mSourceMessage; }; - const auto& getRenormedHistogram() const { return mRenormedHistogram; }; - - ransState_t getState() const { return mState; }; - - private: - std::vector mSourceMessage{}; - RenormedDenseHistogram mRenormedHistogram{}; - ransState_t mState{}; -}; - -const RenormingData Data8(MessageSize); -const RenormingData Data16(MessageSize); -const RenormingData Data32(MessageSize); - -template -const auto& getData() -{ - if constexpr (std::is_same_v) { - return Data8; - } else if constexpr (std::is_same_v) { - return Data16; - } else { - return Data32; - } -}; - -template -struct Fixture : public benchmark::Fixture { - using source_t = source_T; - - void SetUp(const ::benchmark::State& state) final - { - const auto& sourceMessage = getData().getSourceMessage(); - const auto& renormedHistogram = getData().getRenormedHistogram(); - - for (auto& symbol : sourceMessage) { - mFrequencies.push_back(renormedHistogram[symbol]); - } - } - - void TearDown(const ::benchmark::State& state) final - { - mFrequencies.clear(); - } - - std::vector mFrequencies{}; - ransState_t mState = getData().getState(); - size_t mRenormingBits = getData().getRenormedHistogram().getRenormingBits(); -}; - -#ifdef RANS_SIMD -template -struct SIMDFixture : public benchmark::Fixture { - - using source_t = source_T; - - void - SetUp(const ::benchmark::State& state) final - { - mState[0] = simd::setAll(getData().getState()); - mState[1] = simd::setAll(getData().getState()); - - const auto& sourceMessage = getData().getSourceMessage(); - const auto& renormedHistogram = getData().getRenormedHistogram(); - - for (size_t i = 0; i < sourceMessage.size(); i += 2 * nElems) { - if constexpr (width_V == simd::SIMDWidth::SSE) { - mFrequencies.push_back({{simd::epi32_t{renormedHistogram[sourceMessage[i + 0]], - renormedHistogram[sourceMessage[i + 1]], - 0x0u, - 0x0u}, - simd::epi32_t{renormedHistogram[sourceMessage[i + 2]], - renormedHistogram[sourceMessage[i + 3]], - 0x0u, - 0x0u}}}); - } - if constexpr (width_V == simd::SIMDWidth::AVX) { - mFrequencies.push_back({{simd::epi32_t{renormedHistogram[sourceMessage[i + 0]], - renormedHistogram[sourceMessage[i + 1]], - renormedHistogram[sourceMessage[i + 2]], - renormedHistogram[sourceMessage[i + 3]]}, - simd::epi32_t{renormedHistogram[sourceMessage[i + 4]], - renormedHistogram[sourceMessage[i + 5]], - renormedHistogram[sourceMessage[i + 6]], - renormedHistogram[sourceMessage[i + 7]]}}}); - } - } - } - - void TearDown(const ::benchmark::State& state) final - { - mFrequencies.clear(); - } - - static constexpr size_t nElems = simd::getElementCount(width_V); - std::vector, 2>> mFrequencies{}; - simd::simdI_t mState[2]; - uint8_t mRenormingBits = getData().getRenormedHistogram().getRenormingBits(); -}; -#endif /* RANS_SIMD */ - -template -inline std::tuple renorm(ransState_t state, stream_IT outputIter, count_t frequency, size_t symbolTablePrecision) -{ - ransState_t maxState = ((LowerBound >> symbolTablePrecision) << StreamBits) * frequency; // this turns into a shift. - if (state >= maxState) { - *(++outputIter) = static_cast(state); - state >>= StreamBits; - } - - return std::make_tuple(state, outputIter); -}; - -template -static void ransRenormingBenchmark(benchmark::State& st, fixture_T& fixture) -{ - std::vector out(fixture.mFrequencies.size() * 4); - - for (auto _ : st) { - auto outIter = out.data(); - ransState_t newState = fixture.mState; - for (size_t i = 0; i < fixture.mFrequencies.size(); ++i) { - std::tie(newState, outIter) = renorm(fixture.mState, outIter, fixture.mFrequencies[i], fixture.mRenormingBits); - } - benchmark::ClobberMemory(); - }; - - st.SetItemsProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size()); - st.SetBytesProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size() * sizeof(typename fixture_T::source_t)); -}; - -#ifdef RANS_SIMD -template -static void ransRenormingBenchmarkSIMD(benchmark::State& st, fixture_T& fixture) -{ - std::vector out(fixture.mFrequencies.size() * 4); - -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - for (auto _ : st) { - simd::simdIsse_t frequencies[2]; - auto outIter = out.data(); - auto newState = fixture.mState; - for (size_t i = 0; i < fixture.mFrequencies.size(); ++i) { - frequencies[0] = load(fixture.mFrequencies[i][0]); - frequencies[1] = load(fixture.mFrequencies[i][1]); - outIter = simd::ransRenorm(fixture.mState, - frequencies, - fixture.mRenormingBits, - outIter, - newState); - } - benchmark::ClobberMemory(); - }; -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - - st.SetItemsProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size()); - st.SetBytesProcessed(int64_t(st.iterations()) * getData().getSourceMessage().size() * sizeof(typename fixture_T::source_t)); -}; -#endif /* RANS_SIMD */ - -BENCHMARK_TEMPLATE_DEFINE_F(Fixture, renorm_8, uint8_t) -(benchmark::State& st) -{ - ransRenormingBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(Fixture, renorm_16, uint16_t) -(benchmark::State& st) -{ - ransRenormingBenchmark(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(Fixture, renorm_32, uint32_t) -(benchmark::State& st) -{ - ransRenormingBenchmark(st, *this); -}; - -#ifdef RANS_SSE -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, renormSSE_8, uint8_t, simd::SIMDWidth::SSE) -(benchmark::State& st) -{ - ransRenormingBenchmarkSIMD(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, renormSSE_16, uint16_t, simd::SIMDWidth::SSE) -(benchmark::State& st) -{ - ransRenormingBenchmarkSIMD(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, renormSSE_32, uint32_t, simd::SIMDWidth::SSE) -(benchmark::State& st) -{ - ransRenormingBenchmarkSIMD(st, *this); -}; -#endif /* RANS_SSE */ - -#ifdef RANS_AVX2 -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, renormAVX_8, uint8_t, simd::SIMDWidth::AVX) -(benchmark::State& st) -{ - ransRenormingBenchmarkSIMD(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, renormAVX_16, uint16_t, simd::SIMDWidth::AVX) -(benchmark::State& st) -{ - ransRenormingBenchmarkSIMD(st, *this); -}; - -BENCHMARK_TEMPLATE_DEFINE_F(SIMDFixture, renormAVX_32, uint32_t, simd::SIMDWidth::AVX) -(benchmark::State& st) -{ - ransRenormingBenchmarkSIMD(st, *this); -}; -#endif /* RANS_AVX2 */ - -BENCHMARK_REGISTER_F(Fixture, renorm_8); -BENCHMARK_REGISTER_F(Fixture, renorm_16); -BENCHMARK_REGISTER_F(Fixture, renorm_32); - -#ifdef RANS_SSE -BENCHMARK_REGISTER_F(SIMDFixture, renormSSE_8); -BENCHMARK_REGISTER_F(SIMDFixture, renormSSE_16); -BENCHMARK_REGISTER_F(SIMDFixture, renormSSE_32); -#endif /* RANS_SSE */ - -#ifdef RANS_AVX2 -BENCHMARK_REGISTER_F(SIMDFixture, renormAVX_8); -BENCHMARK_REGISTER_F(SIMDFixture, renormAVX_16); -BENCHMARK_REGISTER_F(SIMDFixture, renormAVX_32); -#endif /* RANS_AVX2 */ - -BENCHMARK_MAIN(); diff --git a/Utilities/rANS/benchmarks/bench_ransTPC.cxx b/Utilities/rANS/benchmarks/bench_ransTPC.cxx deleted file mode 100644 index daaa106342c15..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransTPC.cxx +++ /dev/null @@ -1,390 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransTPC.h -/// @author Michael Lettrich -/// @brief main benchmark executable that performs compression/decompression of TPC data while taking detailed metrics. - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "rANS/factory.h" -#include "rANS/histogram.h" -#include "rANS/serialize.h" - -#include "helpers.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -namespace bpo = boost::program_options; -using namespace o2::rans; - -using coder_types = boost::mp11::mp_list -#ifdef RANS_SINGLE_STREAM - , - std::integral_constant -#endif /* RANS_SINGLE_STREAM */ -#ifdef RANS_SSE - , - std::integral_constant -#endif /* RANS_SSE */ -#ifdef RANS_AVX2 - , - std::integral_constant -#endif /* RANS_AVX2 */ - >; - -// using coder_types = boost::mp11::mp_list>; - -std::string toString(CoderTag tag) -{ - switch (tag) { - case CoderTag::Compat: - return {"Compat"}; - break; - case CoderTag::SingleStream: - return {"SingleStream"}; - break; - case CoderTag::SSE: - return {"SSE"}; - break; - case CoderTag::AVX2: - return {"AVX2"}; - break; - default: - throw Exception("Invalid"); - break; - }; -}; - -class TimingDecorator -{ - public: - using jsonWriter_type = rapidjson::Writer; - - TimingDecorator() = default; - TimingDecorator(jsonWriter_type& writer) : mWriter{&writer} {}; - - template >, bool> = true> - decltype(auto) timeAndLog(const std::string& keyName, const std::string& logMessage, F functor) - { - mTimer.start(); - decltype(auto) ret = functor(); - mTimer.stop(); - mWriter->Key(keyName.c_str()); - const double_t msDuration = mTimer.getDurationMS(); - mWriter->Double(msDuration); - LOGP(info, "{} in {} ms", logMessage, msDuration); - - return ret; - }; - - template >, bool> = true> - void timeAndLog(const std::string& keyName, const std::string& logMessage, F functor) - { - mTimer.start(); - functor(); - mTimer.stop(); - mWriter->Key(keyName.c_str()); - const double_t msDuration = mTimer.getDurationMS(); - mWriter->Double(msDuration); - LOGP(info, "{} in {} ms", logMessage, msDuration); - }; - - private: - jsonWriter_type* mWriter{}; - utils::RANSTimer mTimer{}; -}; - -// std::ofstream ofFrequencies{"frequencies.json"}; -// rapidjson::OStreamWrapper streamFrequencies{ofFrequencies}; -// rapidjson::Writer writerFrequencies{streamFrequencies}; - -// std::ofstream ofRenormed{"renormed.json"}; -// rapidjson::OStreamWrapper streamRenormed{ofRenormed}; -// rapidjson::Writer writerRenormed{streamRenormed}; - -template -void ransEncodeDecode(const std::string& name, const std::vector& inputData, rapidjson::Writer& writer) -{ - using source_type = source_T; - utils::RANSTimer timer{}; - TimingDecorator t{writer}; - - writer.Key(name.c_str()); - writer.StartObject(); - - EncodeBuffer encodeBuffer{inputData.size()}; - encodeBuffer.literals.resize(inputData.size(), 0); - encodeBuffer.literalsEnd = encodeBuffer.literals.data(); - DecodeBuffer decodeBuffer{inputData.size()}; - - writer.Key("Timing"); - writer.StartObject(); - - LOGP(info, "processing: {} (nItems: {}, size: {} MiB)", name, inputData.size(), inputData.size() * sizeof(source_type) / 1024.0 / 1024.0); - auto histogram = t.timeAndLog( - "FrequencyTable", "Built Frequency Table", [&]() { return makeDenseHistogram::fromSamples(gsl::span(inputData)); }); - - // writerFrequencies.Key(name.c_str()); - // toJSON(histogram, writerFrequencies); - - auto tmpHist = histogram; - Metrics metrics{}; - RenormedDenseHistogram renormedHistogram{}; - t.timeAndLog("Renorming", "Renormed Frequency Table", [&]() mutable { - metrics = Metrics{histogram}; - renormedHistogram = renorm(std::move(tmpHist), metrics); - }); - // writerRenormed.Key(name.c_str()); - // toJSON(renormedFrequencyTable, writerRenormed); - - auto encoder = t.timeAndLog("Encoder", "Built Encoder", [&]() { return makeDenseEncoder::fromRenormed(renormedHistogram); }); - - t.timeAndLog("Encoding", "Encoded", [&]() mutable { -#ifdef ENABLE_VTUNE_PROFILER - __itt_resume(); -#endif - if (renormedHistogram.hasIncompressibleSymbol()) { - std::tie(encodeBuffer.encodeBufferEnd, encodeBuffer.literalsEnd) = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data(), encodeBuffer.literalsEnd); - } else { - encodeBuffer.encodeBufferEnd = encoder.process(inputData.data(), inputData.data() + inputData.size(), encodeBuffer.buffer.data()); - } -#ifdef ENABLE_VTUNE_PROFILER - __itt_pause(); -#endif - }); - LOGP(info, "Encoded {} Bytes", inputData.size() * sizeof(source_type)); - - std::vector dict(histogram.size() * sizeof(uint64_t), 0); - auto dictEnd = t.timeAndLog("WriteDict", "Serialized Dict", [&]() { return compressRenormedDictionary(encoder.getSymbolTable(), dict.data()); }); - LOGP(info, "Serialized Dict of {} Bytes", std::distance(dict.data(), dictEnd)); - auto recoveredHistogram = t.timeAndLog("ReadDict", "Read Dict", [&]() { - const source_type min = encoder.getSymbolTable().getOffset(); - const source_type max = min + std::max(static_cast(encoder.getSymbolTable().size())-1,0); - return readRenormedDictionary(dict.data(), dictEnd,min,max,renormedHistogram.getRenormingBits()); }); - auto decoder = makeDecoder<>::fromRenormed(renormedHistogram); - auto recoveredDecoder = makeDecoder<>::fromRenormed(recoveredHistogram); - - // if (!(std::equal(decoder.getSymbolTable().begin(), decoder.getSymbolTable().end(), recoveredDecoder.getSymbolTable().begin()) && - // (decoder.getSymbolTable().getEscapeSymbol() == recoveredDecoder.getSymbolTable().getEscapeSymbol()))) { - // LOGP(warning, "Missmatch between original and decoded Dictionary"); - // } - - if (encodeBuffer.literalsEnd == encodeBuffer.literals.data()) { - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams()); - } else { - decoder.process(encodeBuffer.encodeBufferEnd, decodeBuffer.buffer.data(), inputData.size(), encoder.getNStreams(), encodeBuffer.literalsEnd); - } - - if (!(decodeBuffer == inputData)) { - LOGP(warning, "Missmatch between original and decoded Message"); - } - LOG(info) << "finished: " << name; - - writer.EndObject(); // Timing - - const auto& datasetProperties = metrics.getDatasetProperties(); - - // Frequency Table - // ########################## - writer.Key("FrequencyTable"); - writer.StartObject(); - writer.Key("nSamples"); - writer.Uint64(histogram.getNumSamples()); - writer.Key("Min"); - writer.Int(datasetProperties.min); - writer.Key("Max"); - writer.Int(datasetProperties.max); - writer.Key("alphabetRangeBits"); - writer.Int(datasetProperties.alphabetRangeBits); - writer.Key("nUsedAlphabetSymbols"); - writer.Uint(datasetProperties.nUsedAlphabetSymbols); - writer.Key("IncompressibleFrequency"); - writer.Uint(0); - writer.EndObject(); // FrequencyTable - - // RescaledFrequencies - //########################## - const Metrics renormedMetrics{histogram}; - const auto& renormedDatasetProperties = renormedMetrics.getDatasetProperties(); - - writer.Key("RescaledFrequencies"); - writer.StartObject(); - writer.Key("nSamples"); - writer.Uint64(renormedHistogram.getNumSamples()); - writer.Key("Min"); - writer.Int(renormedDatasetProperties.min); - writer.Key("Max"); - writer.Int(renormedDatasetProperties.max); - writer.Key("alphabetRangeBits"); - writer.Int(renormedDatasetProperties.alphabetRangeBits); - writer.Key("nUsedAlphabetSymbols"); - writer.Uint(renormedDatasetProperties.nUsedAlphabetSymbols); - writer.Key("IncompressibleFrequency"); - writer.Uint(renormedHistogram.getIncompressibleSymbolFrequency()); - writer.Key("RenormingBits"); - writer.Uint(renormedHistogram.getRenormingBits()); - writer.EndObject(); // RescaledFrequencies - - // Message Properties - //########################## - writer.Key("Message"); - writer.StartObject(); - writer.Key("Size"); - writer.Uint64(inputData.size()); - writer.Key("SymbolSize"); - writer.Uint(sizeof(source_type)); - writer.Key("Entropy"); - writer.Double(datasetProperties.entropy); - writer.Key("ExpectedCodewordLength"); - writer.Double(computeExpectedCodewordLength<>(histogram, renormedHistogram)); - writer.EndObject(); // Message - - // Compression Properties - //########################## - writer.Key("Compression"); - writer.StartObject(); - writer.Key("EncodeBufferSize"); - writer.Uint64(std::distance(encodeBuffer.buffer.data(), encodeBuffer.encodeBufferEnd) * sizeof(uint32_t)); - writer.Key("LiteralSize"); - writer.Uint64(std::distance(encodeBuffer.literals.data(), encodeBuffer.literalsEnd) * sizeof(source_type)); - writer.Key("DictSize"); - writer.Uint64(std::distance(dict.data(), dictEnd)); - writer.EndObject(); // Compression - - writer.EndObject(); // Encode/Decode Run -}; - -template -void encodeTPC(const std::string& name, const TPCCompressedClusters& compressedClusters, bool mergeColumns, rapidjson::Writer& writer) -{ - writer.Key(name.c_str()); - writer.StartObject(); - ransEncodeDecode("qTotA", compressedClusters.qTotA, writer); - ransEncodeDecode("qMaxA", compressedClusters.qMaxA, writer); - ransEncodeDecode("flagsA", compressedClusters.flagsA, writer); - ransEncodeDecode("rowDiffA", compressedClusters.rowDiffA, writer); - ransEncodeDecode("sliceLegDiffA", compressedClusters.sliceLegDiffA, writer); - ransEncodeDecode("padResA", compressedClusters.padResA, writer); - ransEncodeDecode("timeResA", compressedClusters.timeResA, writer); - ransEncodeDecode("sigmaPadA", compressedClusters.sigmaPadA, writer); - ransEncodeDecode("sigmaTimeA", compressedClusters.sigmaTimeA, writer); - ransEncodeDecode("qPtA", compressedClusters.qPtA, writer); - ransEncodeDecode("rowA", compressedClusters.rowA, writer); - ransEncodeDecode("sliceA", compressedClusters.sliceA, writer); - ransEncodeDecode("timeA", compressedClusters.timeA, writer); - ransEncodeDecode("padA", compressedClusters.padA, writer); - ransEncodeDecode("qTotU", compressedClusters.qTotU, writer); - ransEncodeDecode("qMaxU", compressedClusters.qMaxU, writer); - ransEncodeDecode("flagsU", compressedClusters.flagsU, writer); - ransEncodeDecode("padDiffU", compressedClusters.padDiffU, writer); - ransEncodeDecode("timeDiffU", compressedClusters.timeDiffU, writer); - ransEncodeDecode("sigmaPadU", compressedClusters.sigmaPadU, writer); - ransEncodeDecode("sigmaTimeU", compressedClusters.sigmaTimeU, writer); - ransEncodeDecode("nTrackClusters", compressedClusters.nTrackClusters, writer); - ransEncodeDecode("nSliceRowClusters", compressedClusters.nSliceRowClusters, writer); - - writer.EndObject(); -}; - -using encoder_types = boost::mp11::mp_product; - -int main(int argc, char* argv[]) -{ - bpo::options_description options("Allowed options"); - // clang-format off - options.add_options() - ("help,h", "print usage message") - ("in,i",bpo::value(), "file to process") - ("out,o",bpo::value(), "json output file") - ("mode,m",bpo::value(), "compressor processing mode") - ("log_severity,l",bpo::value(), "severity of FairLogger"); - // clang-format on - - bpo::variables_map vm; - bpo::store(bpo::parse_command_line(argc, argv, options), vm); - bpo::notify(vm); - - if (vm.count("help")) { - std::cout << options << "\n"; - return 0; - } - - const std::string inFile = [&]() { - if (vm.count("in")) { - return vm["in"].as(); - } else { - LOG(error) << "missing path to input file"; - exit(1); - } - }(); - - const std::string outFile = [&]() { - if (vm.count("out")) { - return vm["out"].as(); - } else { - return std::string("out.json"); - } - }(); - - if (vm.count("log_severity")) { - fair::Logger::SetConsoleSeverity(vm["log_severity"].as().c_str()); - } - - std::ofstream of{outFile}; - if (!of) { - std::runtime_error(fmt::format("could not open output file at path {}", inFile)); - } - - // writerFrequencies.StartObject(); - // writerRenormed.StartObject(); - - rapidjson::OStreamWrapper stream{of}; - rapidjson::Writer writer{stream}; - writer.StartObject(); - - TPCCompressedClusters compressedClusters = readFile(inFile); - LOG(info) << "loaded Compressed Clusters from file"; - LOG(info) << "######################################################"; - boost::mp11::mp_for_each([&](auto L) { - using coder_type = boost::mp11::mp_at_c; - constexpr CoderTag coderTag = coder_type::value; - const std::string encoderTitle = toString(coderTag); - - LOGP(info, "start rANS {}/Decode", encoderTitle); - encodeTPC(encoderTitle, compressedClusters, false, writer); - LOG(info) << "######################################################"; - }); - writer.EndObject(); - writer.Flush(); - of.close(); - - // writerFrequencies.EndObject(); - // writerFrequencies.Flush(); - // ofFrequencies.close(); - // writerRenormed.EndObject(); - // writerRenormed.Flush(); - // ofRenormed.close(); -}; \ No newline at end of file diff --git a/Utilities/rANS/benchmarks/bench_ransUnpack.cxx b/Utilities/rANS/benchmarks/bench_ransUnpack.cxx deleted file mode 100644 index 9af732f5b58c5..0000000000000 --- a/Utilities/rANS/benchmarks/bench_ransUnpack.cxx +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file bench_ransUnPack.h -/// @author Michael Lettrich -/// @brief benchmark unpacking of data compared to memcpy - -#include -#include -#include -#include -#include -#ifdef __cpp_lib_execution -#include -#endif -#include - -#include - -#include "rANS/internal/pack/pack.h" - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include "helpers.h" - -using namespace o2::rans; - -inline constexpr size_t MessageSize = 1ull << 22; - -using source_type = uint32_t; - -template -std::vector makeRandomUniformVector(size_t nelems, source_T min = std::numeric_limits::max(), source_T max = std::numeric_limits::max()) -{ - std::vector result(nelems, 0); - std::mt19937 mt(0); // same seed we want always the same distrubution of random numbers; - std::uniform_int_distribution dist(min, max); - -#ifdef __cpp_lib_execution - std::generate(std::execution::par_unseq, result.begin(), result.end(), [&dist, &mt]() { return dist(mt); }); -#else - std::generate(result.begin(), result.end(), [&dist, &mt]() { return dist(mt); }); -#endif - return result; -}; - -static void copyBenchmark(benchmark::State& state) -{ - std::vector src = makeRandomUniformVector(MessageSize); - std::vector dst(MessageSize, 0); - for (auto _ : state) { - std::copy(src.begin(), src.end(), dst.begin()); - }; - state.SetItemsProcessed(src.size() * state.iterations()); - state.SetBytesProcessed(src.size() * sizeof(source_type) * state.iterations()); -}; - -static void unpackingBenchmark(benchmark::State& state) -{ - size_t packingBits = state.range(0); - - std::vector src = makeRandomUniformVector(MessageSize, 0, utils::pow2(packingBits) - 1); - std::vector dst(MessageSize, 0); - - BitPtr iter{dst.data()}; - for (auto i : src) { - iter = internal::pack(iter, i, packingBits); - } - - std::vector unpacked(MessageSize, 0); - - for (auto _ : state) { - BitPtr iter{dst.data()}; - for (size_t i = 0; i < src.size(); ++i) { - unpacked[i] = internal::unpack(iter, packingBits); - iter += packingBits; - } - } - - state.SetItemsProcessed(src.size() * state.iterations()); - state.SetBytesProcessed(src.size() * sizeof(uint32_t) * state.iterations()); - - if (!std::equal(unpacked.begin(), unpacked.end(), src.begin())) { - state.SkipWithError("error in packing"); - } -}; - -BENCHMARK(copyBenchmark); -BENCHMARK(unpackingBenchmark)->DenseRange(1, 32, 1); -BENCHMARK_MAIN(); diff --git a/Utilities/rANS/benchmarks/helpers.h b/Utilities/rANS/benchmarks/helpers.h deleted file mode 100644 index a06531310b19f..0000000000000 --- a/Utilities/rANS/benchmarks/helpers.h +++ /dev/null @@ -1,316 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file helpers.h -/// @author Michael Lettrich -/// @brief common functionality for rANS benchmarks. - -#ifndef RANS_BENCHMARKS_HELPERS_H_ -#define RANS_BENCHMARKS_HELPERS_H_ - -#ifdef ENABLE_VTUNE_PROFILER -#include -#endif - -#include -#include -#include -#include - -#include -#include - -#include "rANS/internal/common/exceptions.h" - -#include -#ifdef __cpp_lib_execution -#include -#endif - -struct TPCCompressedClusters { - - TPCCompressedClusters() = default; - TPCCompressedClusters(size_t nTracks, - size_t nAttachedClusters, - size_t nUnattachedClusters, - size_t nAttachedClustersReduced) : nTracks(nTracks), - nAttachedClusters(nAttachedClusters), - nUnattachedClusters(nUnattachedClusters), - nAttachedClustersReduced(nAttachedClustersReduced) - { - qTotA.resize(this->nAttachedClusters); - qMaxA.resize(this->nAttachedClusters); - flagsA.resize(this->nAttachedClusters); - rowDiffA.resize(this->nAttachedClustersReduced); - sliceLegDiffA.resize(this->nAttachedClustersReduced); - padResA.resize(this->nAttachedClustersReduced); - timeResA.resize(this->nAttachedClustersReduced); - sigmaPadA.resize(this->nAttachedClusters); - sigmaTimeA.resize(this->nAttachedClusters); - - qPtA.resize(this->nTracks); - rowA.resize(this->nTracks); - sliceA.resize(this->nTracks); - timeA.resize(this->nTracks); - padA.resize(this->nTracks); - - qTotU.resize(this->nUnattachedClusters); - qMaxU.resize(this->nUnattachedClusters); - flagsU.resize(this->nUnattachedClusters); - padDiffU.resize(this->nUnattachedClusters); - timeDiffU.resize(this->nUnattachedClusters); - sigmaPadU.resize(this->nUnattachedClusters); - sigmaTimeU.resize(this->nUnattachedClusters); - - nTrackClusters.resize(this->nTracks); - nSliceRowClusters.resize(this->nSliceRows); - }; - - size_t nTracks = 0; - size_t nAttachedClusters = 0; - size_t nUnattachedClusters = 0; - size_t nAttachedClustersReduced = 0; - size_t nSliceRows = 36 * 152; - - std::vector qTotA{}; - std::vector qMaxA{}; - std::vector flagsA{}; - std::vector rowDiffA{}; - std::vector sliceLegDiffA{}; - std::vector padResA{}; - std::vector timeResA{}; - std::vector sigmaPadA{}; - std::vector sigmaTimeA{}; - - std::vector qPtA{}; - std::vector rowA{}; - std::vector sliceA{}; - std::vector timeA{}; - std::vector padA{}; - - std::vector qTotU{}; - std::vector qMaxU{}; - std::vector flagsU{}; - std::vector padDiffU{}; - std::vector timeDiffU{}; - std::vector sigmaPadU{}; - std::vector sigmaTimeU{}; - - std::vector nTrackClusters{}; - std::vector nSliceRowClusters{}; -}; - -class TPCJsonHandler : public rapidjson::BaseReaderHandler, TPCJsonHandler> -{ - private: - class CurrentVector - { - public: - CurrentVector() = default; - - template - CurrentVector(std::vector& vec) : mVectorConcept{std::make_unique>(vec)} {}; - - inline void push_back(unsigned i) { mVectorConcept->push_back(i); }; - - struct VectorConcept { - virtual ~VectorConcept() = default; - virtual void push_back(unsigned i) = 0; - }; - - template - struct VectorWrapper : VectorConcept { - VectorWrapper(std::vector& vector) : mVector(vector){}; - - void push_back(unsigned i) override { mVector.push_back(static_cast(i)); }; - - private: - std::vector& mVector{}; - }; - - std::unique_ptr mVectorConcept{}; - }; - - public: - bool Null() { return true; }; - bool Bool(bool b) { return true; }; - bool Int(int i) { return this->Uint(static_cast(i)); }; - bool Uint(unsigned i) - { - mCurrentVector.push_back(i); - return true; - }; - bool Int64(int64_t i) { return this->Uint(static_cast(i)); }; - bool Uint64(uint64_t i) { return this->Uint(static_cast(i)); }; - bool Double(double d) { return this->Uint(static_cast(d)); }; - bool RawNumber(const Ch* str, rapidjson::SizeType length, bool copy) { return true; }; - bool String(const Ch* str, rapidjson::SizeType length, bool copy) { return true; }; - bool StartObject() { return true; }; - bool Key(const Ch* str, rapidjson::SizeType length, bool copy) - { - if (str == std::string{"qTotA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.qTotA}; - } else if (str == std::string{"qMaxA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.qMaxA}; - } else if (str == std::string{"flagsA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.flagsA}; - } else if (str == std::string{"rowDiffA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.rowDiffA}; - } else if (str == std::string{"sliceLegDiffA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.sliceLegDiffA}; - } else if (str == std::string{"padResA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.padResA}; - } else if (str == std::string{"timeResA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.timeResA}; - } else if (str == std::string{"sigmaPadA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.sigmaPadA}; - } else if (str == std::string{"sigmaTimeA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.sigmaTimeA}; - } else if (str == std::string{"qPtA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.qPtA}; - } else if (str == std::string{"rowA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.rowA}; - } else if (str == std::string{"sliceA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.sliceA}; - } else if (str == std::string{"timeA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.timeA}; - } else if (str == std::string{"padA"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.padA}; - } else if (str == std::string{"qTotU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.qTotU}; - } else if (str == std::string{"qMaxU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.qMaxU}; - } else if (str == std::string{"flagsU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.flagsU}; - } else if (str == std::string{"padDiffU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.padDiffU}; - } else if (str == std::string{"timeDiffU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.timeDiffU}; - } else if (str == std::string{"sigmaPadU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.sigmaPadU}; - } else if (str == std::string{"sigmaTimeU"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.sigmaTimeU}; - } else if (str == std::string{"nTrackClusters"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.nTrackClusters}; - } else if (str == std::string{"nSliceRowClusters"}) { - LOGP(info, "parsing {}", str); - mCurrentVector = CurrentVector{mCompressedClusters.nSliceRowClusters}; - } else { - throw o2::rans::IOError(fmt::format("invalid key: {}", str)); - return false; - } - return true; - }; - bool EndObject(rapidjson::SizeType memberCount) - { - LOGP(info, "parsed {} objects", memberCount); - return true; - }; - bool StartArray() - { - return true; - }; - bool EndArray(rapidjson::SizeType elementCount) - { - LOGP(info, "parsed {} elements", elementCount); - return true; - }; - - TPCCompressedClusters release() && { return std::move(mCompressedClusters); }; - - private: - TPCCompressedClusters mCompressedClusters; - CurrentVector mCurrentVector; -}; - -TPCCompressedClusters readFile(const std::string& filename) -{ - TPCCompressedClusters compressedClusters; - std::ifstream is(filename, std::ios_base::in); - if (is) { - rapidjson::IStreamWrapper isWrapper{is}; - // rapidjson::Document document; - TPCJsonHandler handler; - rapidjson::Reader reader; - reader.Parse(isWrapper, handler); - - compressedClusters = std::move(handler).release(); - - compressedClusters.nAttachedClusters = compressedClusters.qTotA.size(); - compressedClusters.nAttachedClustersReduced = compressedClusters.rowDiffA.size(); - compressedClusters.nUnattachedClusters = compressedClusters.qTotA.size(); - compressedClusters.nTracks = compressedClusters.nTrackClusters.size(); - is.close(); - } else { - throw o2::rans::IOError(fmt::format("Could not open file {}", filename)); - } - return compressedClusters; -}; - -template -struct EncodeBuffer { - - EncodeBuffer() = default; - EncodeBuffer(size_t sourceSize) : buffer(2 * sourceSize), literals() - { - literals.reserve(sourceSize + 32); - literalsEnd = literals.data(); - encodeBufferEnd = buffer.data(); - }; - - std::vector buffer{}; - std::vector literals{}; - stream_T* encodeBufferEnd{buffer.data()}; - source_T* literalsEnd{literals.data()}; -}; - -template -struct DecodeBuffer { - - DecodeBuffer() = default; - DecodeBuffer(size_t sourceSize) : buffer(sourceSize){}; - - template - bool operator==(const T& correct) - { -#ifdef __cpp_lib_execution - return std::equal(std::execution::par_unseq, buffer.begin(), buffer.end(), std::begin(correct), std::end(correct)); -#else - return std::equal(buffer.begin(), buffer.end(), std::begin(correct), std::end(correct)); -#endif - } - - std::vector buffer{}; -}; - -#endif /* RANS_BENCHMARKS_HELPERS_H_ */ \ No newline at end of file diff --git a/Utilities/rANS/include/rANS/Decoder.h b/Utilities/rANS/include/rANS/Decoder.h new file mode 100644 index 0000000000000..0a2fdbb871006 --- /dev/null +++ b/Utilities/rANS/include/rANS/Decoder.h @@ -0,0 +1,101 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file Decoder.h +/// @author Michael Lettrich +/// @since 2020-04-06 +/// @brief Decoder - decode a rANS encoded state back into source symbols + +#ifndef RANS_DECODER_H +#define RANS_DECODER_H + +#include +#include +#include +#include +#include + +#include + +#include "rANS/internal/Decoder.h" +#include "rANS/internal/DecoderBase.h" + +namespace o2 +{ +namespace rans +{ + +template +class Decoder : public internal::DecoderBase +{ + public: + using internal::DecoderBase::DecoderBase; + + template , bool> = true> + void process(stream_IT inputEnd, source_IT outputBegin, size_t messageLength) const; + + private: + using ransDecoder_t = typename internal::DecoderBase::ransDecoder_t; +}; + +template +template , bool>> +void Decoder::process(stream_IT inputEnd, source_IT outputBegin, size_t messageLength) const +{ + using namespace internal; + LOG(trace) << "start decoding"; + RANSTimer t; + t.start(); + + if (messageLength == 0) { + LOG(warning) << "Empty message passed to decoder, skipping decode process"; + return; + } + + stream_IT inputIter = inputEnd; + source_IT it = outputBegin; + + // make Iter point to the last last element + --inputIter; + + ransDecoder_t rans0{this->mSymbolTable.getPrecision()}; + ransDecoder_t rans1{this->mSymbolTable.getPrecision()}; + + inputIter = rans0.init(inputIter); + inputIter = rans1.init(inputIter); + + for (size_t i = 0; i < (messageLength & ~1); i += 2) { + const int64_t s0 = this->mReverseLUT[rans0.get()]; + const int64_t s1 = this->mReverseLUT[rans1.get()]; + *it++ = s0; + *it++ = s1; + inputIter = rans0.advanceSymbol(inputIter, this->mSymbolTable[s0]); + inputIter = rans1.advanceSymbol(inputIter, this->mSymbolTable[s1]); + } + + // last byte, if message length was odd + if (messageLength & 1) { + const int64_t s0 = this->mReverseLUT[rans0.get()]; + *it = s0; + inputIter = rans0.advanceSymbol(inputIter, this->mSymbolTable[s0]); + } + t.stop(); + LOG(debug1) << "Decoder::" << __func__ << " { DecodedSymbols: " << messageLength << "," + << "processedBytes: " << messageLength * sizeof(source_T) << "," + << " inclusiveTimeMS: " << t.getDurationMS() << "," + << " BandwidthMiBPS: " << std::fixed << std::setprecision(2) << (messageLength * sizeof(source_T) * 1.0) / (t.getDurationS() * 1.0 * (1 << 20)) << "}"; + + LOG(trace) << "done decoding"; +} +} // namespace rans +} // namespace o2 + +#endif /* RANS_DECODER_H */ diff --git a/Utilities/rANS/include/rANS/DedupDecoder.h b/Utilities/rANS/include/rANS/DedupDecoder.h new file mode 100644 index 0000000000000..33f883bd29713 --- /dev/null +++ b/Utilities/rANS/include/rANS/DedupDecoder.h @@ -0,0 +1,106 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file Decoder.h +/// @author Michael Lettrich +/// @since 2020-04-06 +/// @brief Decoder - decode a rANS encoded state back into source symbols + +#ifndef RANS_DEDUPDECODER_H +#define RANS_DEDUPDECODER_H + +#include +#include +#include +#include +#include + +#include + +#include "rANS/internal/DecoderSymbol.h" +#include "rANS/internal/ReverseSymbolLookupTable.h" +#include "rANS/internal/SymbolTable.h" +#include "rANS/internal/Decoder.h" +#include "rANS/internal/DecoderBase.h" + +namespace o2 +{ +namespace rans +{ + +template +class DedupDecoder : public internal::DecoderBase +{ + + public: + using duplicatesMap_t = std::map; + + // inherit constructors; + using internal::DecoderBase::DecoderBase; + + template , bool> = true> + void process(stream_IT inputEnd, source_IT outputBegin, size_t messageLength, duplicatesMap_t& duplicates) const; + + private: + using ransDecoder_t = typename internal::DecoderBase::ransDecoder_t; +}; + +template +template , bool>> +void DedupDecoder::process(stream_IT inputEnd, source_IT outputBegin, size_t messageLength, duplicatesMap_t& duplicates) const +{ + using namespace internal; + LOG(trace) << "start decoding"; + RANSTimer t; + t.start(); + + if (messageLength == 0) { + LOG(warning) << "Empty message passed to decoder, skipping decode process"; + return; + } + + stream_IT inputIter = inputEnd; + source_IT it = outputBegin; + + // make Iter point to the last last element + --inputIter; + + ransDecoder_t rans{this->mSymbolTable.getPrecision()}; + inputIter = rans.init(inputIter); + + for (size_t i = 0; i < (messageLength); i++) { + const auto s = (this->mReverseLUT)[rans.get()]; + + // deduplication + auto duplicatesIter = duplicates.find(i); + if (duplicatesIter != duplicates.end()) { + LOG(trace) << "pos[" << i << "]: restoring " << duplicatesIter->second << " duplicates of symbol " << (char)s; + for (unsigned int d = 0; d < duplicatesIter->second; d++) { + *it++ = s; + i++; + } + } + *it++ = s; + inputIter = rans.advanceSymbol(inputIter, (this->mSymbolTable)[s]); + } + + t.stop(); + LOG(debug1) << "Decoder::" << __func__ << " { DecodedSymbols: " << messageLength << "," + << "processedBytes: " << messageLength * sizeof(source_T) << "," + << " inclusiveTimeMS: " << t.getDurationMS() << "," + << " BandwidthMiBPS: " << std::fixed << std::setprecision(2) << (messageLength * sizeof(source_T) * 1.0) / (t.getDurationS() * 1.0 * (1 << 20)) << "}"; + + LOG(trace) << "done decoding"; +} +} // namespace rans +} // namespace o2 + +#endif /* RANS_DEDUPDECODER_H */ diff --git a/Utilities/rANS/include/rANS/DedupEncoder.h b/Utilities/rANS/include/rANS/DedupEncoder.h new file mode 100644 index 0000000000000..d51c912468ccb --- /dev/null +++ b/Utilities/rANS/include/rANS/DedupEncoder.h @@ -0,0 +1,136 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file Encoder.h +/// @author Michael Lettrich +/// @since 2020-04-06 +/// @brief Encoder - code symbol into a rANS encoded state + +#ifndef INCLUDE_RANS_DEDUPENCODER_H_ +#define INCLUDE_RANS_DEDUPENCODER_H_ + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "rANS/internal/EncoderBase.h" +#include "rANS/internal/EncoderSymbol.h" +#include "rANS/internal/helper.h" +#include "rANS/internal/SymbolTable.h" + +namespace o2 +{ +namespace rans +{ + +template +class DedupEncoder : public internal::EncoderBase +{ + public: + // inherit constructors; + using internal::EncoderBase::EncoderBase; + + using duplicatesMap_t = std::map; + + template , bool> = true> + stream_IT process(source_IT inputBegin, source_IT inputEnd, stream_IT outputBegin, duplicatesMap_t& duplicates) const; + + private: + using ransCoder_t = typename internal::EncoderBase::ransCoder_t; +}; + +template +template , bool>> +stream_IT DedupEncoder::process(source_IT inputBegin, source_IT inputEnd, stream_IT outputBegin, duplicatesMap_t& duplicates) const +{ + using namespace internal; + LOG(trace) << "start encoding"; + RANSTimer t; + t.start(); + + if (inputBegin == inputEnd) { + LOG(warning) << "passed empty message to encoder, skip encoding"; + return outputBegin; + } + ransCoder_t rans{this->mSymbolTable.getPrecision()}; + + stream_IT outputIter = outputBegin; + source_IT inputIT = inputEnd; + + const auto inputBufferSize = std::distance(inputBegin, inputEnd); + + auto encode = [&inputBegin, &duplicates, this](source_IT symbolIter, stream_IT outputIter, ransCoder_t& coder) { + const source_T symbol = *symbolIter; + const auto& encoderSymbol = (this->mSymbolTable)[symbol]; + + // dedup step: + auto dedupIT = symbolIter; + // advance on source by one. + --dedupIT; + size_t numDuplicates = 0; + + // find out how many duplicates we have + while (*dedupIT == symbol) { + --dedupIT; + ++numDuplicates; + } + + // if we have a duplicate treat it. + if (numDuplicates > 0) { + const auto pos = std::distance(inputBegin, symbolIter) - 1; + LOG(trace) << "pos[" << pos << "]: found " << numDuplicates << " duplicates of symbol " << (char)symbol; + duplicates.emplace(pos, numDuplicates); + } + + return std::pair(++dedupIT, coder.putSymbol(outputIter, encoderSymbol)); + }; + + while (inputIT != inputBegin) { // NB: working in reverse! + std::tie(inputIT, outputIter) = encode(--inputIT, outputIter, rans); + } + outputIter = rans.flush(outputIter); + // first iterator past the range so that sizes, distances and iterators work correctly. + ++outputIter; + + t.stop(); + LOG(debug1) << "Encoder::" << __func__ << " {ProcessedBytes: " << inputBufferSize * sizeof(source_T) << "," + << " inclusiveTimeMS: " << t.getDurationMS() << "," + << " BandwidthMiBPS: " << std::fixed << std::setprecision(2) << (inputBufferSize * sizeof(source_T) * 1.0) / (t.getDurationS() * 1.0 * (1 << 20)) << "}"; + +// advanced diagnostics for debug builds +#if !defined(NDEBUG) + + const auto inputBufferSizeB = inputBufferSize * sizeof(source_T); + + LOG(debug2) + << "EncoderProperties: {" + << "sourceTypeB: " << sizeof(source_T) << ", " + << "streamTypeB: " << sizeof(stream_T) << ", " + << "coderTypeB: " << sizeof(coder_T) << ", " + << "symbolTablePrecision: " << this->mSymbolTable.getPrecision() << ", " + << "inputBufferSizeB: " << inputBufferSizeB << "}"; +#endif + + LOG(trace) << "done encoding"; + + return outputIter; +}; + +} // namespace rans +} // namespace o2 + +#endif /* INCLUDE_RANS_DEDUPENCODER_H_ */ diff --git a/Utilities/rANS/include/rANS/Encoder.h b/Utilities/rANS/include/rANS/Encoder.h new file mode 100644 index 0000000000000..aa703f7aa3722 --- /dev/null +++ b/Utilities/rANS/include/rANS/Encoder.h @@ -0,0 +1,122 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file Encoder.h +/// @author Michael Lettrich +/// @since 2020-04-06 +/// @brief Encoder - code symbol into a rANS encoded state + +#ifndef RANS_ENCODER_H +#define RANS_ENCODER_H + +#include +#include +#include + +#include +#include + +#include "rANS/internal/EncoderBase.h" +#include "rANS/internal/Encoder.h" +#include "rANS/internal/EncoderSymbol.h" +#include "rANS/internal/helper.h" +#include "rANS/internal/SymbolTable.h" +#include "rANS/FrequencyTable.h" + +namespace o2 +{ +namespace rans +{ + +template +class Encoder : public internal::EncoderBase +{ + + public: + // inherit constructors; + using internal::EncoderBase::EncoderBase; + + template , bool> = true> + const stream_IT process(source_IT inputBegin, source_IT inputEnd, stream_IT outputBegin) const; + + private: + using ransCoder_t = typename internal::EncoderBase::ransCoder_t; +}; + +template +template , bool>> +const stream_IT Encoder::process(source_IT inputBegin, source_IT inputEnd, stream_IT outputBegin) const +{ + using namespace internal; + LOG(trace) << "start encoding"; + RANSTimer t; + t.start(); + + if (inputBegin == inputEnd) { + LOG(warning) << "passed empty message to encoder, skip encoding"; + return outputBegin; + } + + ransCoder_t rans0{this->mSymbolTable.getPrecision()}; + ransCoder_t rans1{this->mSymbolTable.getPrecision()}; + + stream_IT outputIter = outputBegin; + source_IT inputIT = inputEnd; + + const auto inputBufferSize = std::distance(inputBegin, inputEnd); + + auto encode = [this](source_IT symbolIter, stream_IT outputIter, ransCoder_t& coder) { + const source_T symbol = *symbolIter; + const auto& encoderSymbol = (this->mSymbolTable)[symbol]; + return coder.putSymbol(outputIter, encoderSymbol); + }; + + // odd number of bytes? + if (inputBufferSize & 1) { + outputIter = encode(--inputIT, outputIter, rans0); + } + + while (inputIT != inputBegin) { // NB: working in reverse! + outputIter = encode(--inputIT, outputIter, rans1); + outputIter = encode(--inputIT, outputIter, rans0); + } + outputIter = rans1.flush(outputIter); + outputIter = rans0.flush(outputIter); + // first iterator past the range so that sizes, distances and iterators work correctly. + ++outputIter; + + t.stop(); + LOG(debug1) << "Encoder::" << __func__ << " {ProcessedBytes: " << inputBufferSize * sizeof(source_T) << "," + << " inclusiveTimeMS: " << t.getDurationMS() << "," + << " BandwidthMiBPS: " << std::fixed << std::setprecision(2) << (inputBufferSize * sizeof(source_T) * 1.0) / (t.getDurationS() * 1.0 * (1 << 20)) << "}"; + +// advanced diagnostics for debug builds +#if !defined(NDEBUG) + + const auto inputBufferSizeB = inputBufferSize * sizeof(source_T); + + LOG(debug2) << "EncoderProperties: {" + << "sourceTypeB: " << sizeof(source_T) << ", " + << "streamTypeB: " << sizeof(stream_T) << ", " + << "coderTypeB: " << sizeof(coder_T) << ", " + << "symbolTablePrecision: " << this->mSymbolTable.getPrecision() << ", " + << "inputBufferSizeB: " << inputBufferSizeB << "}"; +#endif + + LOG(trace) << "done encoding"; + + return outputIter; +}; + +} // namespace rans +} // namespace o2 + +#endif /* RANS_ENCODER_H */ diff --git a/Utilities/rANS/include/rANS/FrequencyTable.h b/Utilities/rANS/include/rANS/FrequencyTable.h new file mode 100644 index 0000000000000..c768c00ef27d9 --- /dev/null +++ b/Utilities/rANS/include/rANS/FrequencyTable.h @@ -0,0 +1,363 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file FrequencyTable.h +/// @author Michael Lettrich +/// @since 2019-05-08 +/// @brief Histogram to depict frequencies of source symbols for rANS compression. + +#ifndef INCLUDE_RANS_FREQUENCYTABLE_H_ +#define INCLUDE_RANS_FREQUENCYTABLE_H_ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "rANS/definitions.h" +#include "rANS/internal/helper.h" +#include "rANS/utils/HistogramView.h" + +namespace o2 +{ +namespace rans +{ + +class FrequencyTable +{ + public: + using iterator_t = count_t*; + using constIterator_t = const count_t*; + + // Constructors + + // TODO(milettri): fix once ROOT cling respects the standard http://wg21.link/p1286r2 + FrequencyTable() noexcept {}; // NOLINT + + FrequencyTable(symbol_t min, symbol_t max) : mFrequencyTable(max - min + 1, 0), mOffset{min} { assert(max >= min); }; + + template , bool> = true> + FrequencyTable(Freq_IT begin, Freq_IT end, symbol_t min, count_t incompressibleSymbolFrequency = 0); + + // accessors + + inline count_t operator[](symbol_t symbol) const { return getSymbol(symbol); }; + + count_t at(size_t index) const; + + inline const count_t* data() const noexcept { return mFrequencyTable.data(); }; + + inline constIterator_t cbegin() const noexcept { return data(); }; + + inline constIterator_t cend() const noexcept { return data() + size(); }; + + inline constIterator_t begin() const noexcept { return cbegin(); }; + + inline constIterator_t end() const noexcept { return cend(); }; + + inline iterator_t begin() noexcept { return const_cast(cbegin()); }; + + inline iterator_t end() noexcept { return const_cast(cend()); }; + + inline size_t size() const noexcept { return mFrequencyTable.size(); }; + + inline bool empty() const noexcept { return mFrequencyTable.empty(); }; + + size_t getNUsedAlphabetSymbols() const; + + inline size_t getAlphabetRangeBits() const noexcept { return internal::numBitsForNSymbols(size() + this->hasIncompressibleSymbols()); }; + + inline symbol_t getMinSymbol() const noexcept { return mOffset; }; + inline symbol_t getMaxSymbol() const noexcept { return mOffset + std::max(0l, static_cast(mFrequencyTable.size()) - 1l); }; + + inline count_t getIncompressibleSymbolFrequency() const noexcept { return mIncompressibleSymbolFrequency; }; + + inline size_t getNumSamples() const noexcept { return mNumSamples + this->getIncompressibleSymbolFrequency(); }; + + // operations + template , bool> = true> + FrequencyTable& addSamples(Source_IT begin, Source_IT end, bool extendTable = true); + + template , bool> = true> + FrequencyTable& addSamples(Source_IT begin, Source_IT end, symbol_t min, symbol_t max, bool extendTable = true); + + template , bool> = true> + FrequencyTable& addFrequencies(Freq_IT begin, Freq_IT end, symbol_t min, bool extendTable = true); + + FrequencyTable& operator+(FrequencyTable& other); + + histogram_t release() && noexcept; + + FrequencyTable& resize(symbol_t min, symbol_t max, bool truncate = false); + + inline FrequencyTable& resize(size_t newSize, bool truncate = false) { return resize(this->getMinSymbol(), this->getMinSymbol() + newSize, truncate); }; + + FrequencyTable& trim(); + + inline bool hasIncompressibleSymbols() const noexcept { return this->getIncompressibleSymbolFrequency() > 0; }; + + private: + const count_t& getSymbol(symbol_t symbol) const; + count_t& getSymbol(symbol_t symbol); + + count_t frequencyCountingDecorator(count_t frequency) noexcept; + + symbol_t sampleCountingDecorator(symbol_t sample) noexcept; + + histogram_t mFrequencyTable{}; + symbol_t mOffset{}; + size_t mNumSamples{}; + count_t mIncompressibleSymbolFrequency{}; +}; // namespace rans + +template +double_t computeEntropy(IT begin, IT end, symbol_t min); + +double_t computeEntropy(const FrequencyTable& table); + +count_t computeRenormingPrecision(const FrequencyTable& frequencyTable); + +std::ostream& operator<<(std::ostream& out, const FrequencyTable& fTable); + +} // namespace rans +} // namespace o2 + +// IMPL +/////////////////////////////////////////////////////////////////////////////////////////// + +namespace o2 +{ +namespace rans +{ + +template , bool>> +FrequencyTable::FrequencyTable(Freq_IT begin, Freq_IT end, symbol_t min, count_t incompressibleSymbolFrequency) +{ + auto histogram = utils::HistogramView{begin, end, min}; + this->addFrequencies(histogram.begin(), histogram.end(), histogram.getMin(), true); + mIncompressibleSymbolFrequency = incompressibleSymbolFrequency; +} + +template , bool>> +inline FrequencyTable& FrequencyTable::addSamples(Source_IT begin, Source_IT end, bool extendTable) +{ + if (begin != end) { + const auto& [minIter, maxIter] = std::minmax_element(begin, end); + addSamples(begin, end, *minIter, *maxIter, extendTable); + } else { + LOG(warning) << "Passed empty message to " << __func__; // RS this is ok for empty columns + } + return *this; +} + +template , bool>> +FrequencyTable& FrequencyTable::addSamples(Source_IT begin, Source_IT end, symbol_t min, symbol_t max, bool extendTable) +{ + LOG(trace) << "start adding samples"; + internal::RANSTimer t; + t.start(); + + if (begin == end) { + LOG(warning) << "Passed empty message to " << __func__; // RS this is ok for empty columns + } else { + if (extendTable) { + this->resize(min, max); + // add new symbols + std::for_each(begin, end, [this](symbol_t symbol) { ++this->getSymbol(this->sampleCountingDecorator(symbol)); }); + } else { + // add new symbols but all that are out of range are set to incompressible + std::for_each(begin, end, [this](symbol_t symbol) { + if (this->empty() || symbol < this->getMinSymbol() || symbol > this->getMaxSymbol()) { + ++this->mIncompressibleSymbolFrequency; + } else { + ++this->getSymbol(this->sampleCountingDecorator(symbol)); + } + }); + } + } + + t.stop(); + LOG(debug1) << __func__ << " inclusive time (ms): " << t.getDurationMS(); + +#if !defined(NDEBUG) + LOG(debug2) << *this; +#endif + LOG(trace) << "done adding samples"; + return *this; +} + +template , bool>> +FrequencyTable& FrequencyTable::addFrequencies(Freq_IT begin, Freq_IT end, symbol_t min, bool extendTable) +{ + LOG(trace) << "start adding frequencies"; + internal::RANSTimer t; + t.start(); + + auto thisHistogram = utils::HistogramView{mFrequencyTable.begin(), mFrequencyTable.end(), mOffset}; + auto addedHistogram = utils::trim(utils::HistogramView{begin, end, min}); + if (addedHistogram.empty()) { + LOG(warning) << "Passed empty FrequencyTable to " << __func__; // RS this is ok for empty columns + } else { + + const symbol_t newMin = std::min(thisHistogram.getMin(), addedHistogram.getMin()); + const symbol_t newMax = std::max(thisHistogram.getMax(), addedHistogram.getMax()); + + // resize table + const bool needsExtend = thisHistogram.empty() || (newMin < thisHistogram.getMin()) || (newMax > thisHistogram.getMax()); + + if (needsExtend && extendTable) { + + if (thisHistogram.empty()) { + mFrequencyTable = histogram_t(addedHistogram.size()); + std::transform(addedHistogram.begin(), addedHistogram.end(), mFrequencyTable.begin(), [this](count_t frequency) { + return this->frequencyCountingDecorator(frequency); + }); + mOffset = addedHistogram.getOffset(); + } else { + const symbol_t newSize = newMax - newMin + 1; + histogram_t newFreequencyTable(newSize, 0); + auto newHistogram = utils::HistogramView{newFreequencyTable.begin(), newFreequencyTable.end(), newMin}; + auto histogramOverlap = utils::intersection(newHistogram, thisHistogram); + assert(!histogramOverlap.empty()); + assert(histogramOverlap.size() == thisHistogram.size()); + std::copy(thisHistogram.begin(), thisHistogram.end(), histogramOverlap.begin()); + + histogramOverlap = utils::intersection(newHistogram, addedHistogram); + assert(!histogramOverlap.empty()); + assert(histogramOverlap.size() == addedHistogram.size()); + std::transform(addedHistogram.begin(), addedHistogram.end(), histogramOverlap.begin(), histogramOverlap.begin(), [this](const count_t& a, const count_t& b) { return internal::safeadd(this->frequencyCountingDecorator(a), b); }); + + this->mFrequencyTable = std::move(newFreequencyTable); + this->mOffset = newHistogram.getOffset(); + } + } else { + thisHistogram = utils::HistogramView{mFrequencyTable.begin(), mFrequencyTable.end(), mOffset}; + + // left incompressible tail + auto leftTail = utils::leftTail(addedHistogram, thisHistogram); + if (!leftTail.empty()) { + mIncompressibleSymbolFrequency += std::accumulate(leftTail.begin(), leftTail.end(), 0); + } + + // intersection + auto overlapAdded = utils::intersection(addedHistogram, thisHistogram); + auto overlapThis = utils::intersection(thisHistogram, addedHistogram); + if (!overlapAdded.empty()) { + assert(overlapAdded.getMin() == overlapThis.getMin()); + assert(overlapAdded.size() == overlapThis.size()); + std::transform(overlapAdded.begin(), overlapAdded.end(), overlapThis.begin(), overlapThis.begin(), [this](const count_t& a, const count_t& b) { return internal::safeadd(this->frequencyCountingDecorator(a), b); }); + } + + // right incompressible tail + auto rightTail = utils::rightTail(addedHistogram, thisHistogram); + if (!rightTail.empty()) { + mIncompressibleSymbolFrequency += std::accumulate(rightTail.begin(), rightTail.end(), 0); + } + } + } + t.stop(); + LOG(debug1) << __func__ << " inclusive time (ms): " << t.getDurationMS(); + +#if !defined(NDEBUG) + LOG(debug2) << *this; +#endif + + LOG(trace) << "done adding frequencies"; + return *this; +} + +inline auto FrequencyTable::at(size_t index) const -> count_t +{ + assert(index < size()); + return mFrequencyTable[index]; +}; + +inline FrequencyTable& FrequencyTable::operator+(FrequencyTable& other) +{ + addFrequencies(other.cbegin(), other.cend(), other.getMinSymbol(), true); + return *this; +} + +inline auto FrequencyTable::release() && noexcept -> histogram_t +{ + auto frequencies = std::move(mFrequencyTable); + *this = FrequencyTable(); + + return frequencies; +}; + +inline size_t FrequencyTable::getNUsedAlphabetSymbols() const +{ + return std::count_if(mFrequencyTable.begin(), mFrequencyTable.end(), [](count_t count) { return count > 0; }) + static_cast(this->hasIncompressibleSymbols()); +}; + +inline auto FrequencyTable::getSymbol(symbol_t symbol) const -> const count_t& +{ + // negative numbers cause overflow thus we get away with one comparison only + const size_t index = static_cast(symbol - this->getMinSymbol()); + assert(index < mFrequencyTable.size()); + return mFrequencyTable[index]; +} + +inline auto FrequencyTable::getSymbol(symbol_t symbol) -> count_t& +{ + return const_cast(static_cast(*this).getSymbol(symbol)); +} + +inline count_t FrequencyTable::frequencyCountingDecorator(count_t frequency) noexcept +{ + mNumSamples += frequency; + return frequency; +}; + +inline symbol_t FrequencyTable::sampleCountingDecorator(symbol_t sample) noexcept +{ + ++mNumSamples; + return sample; +}; + +template +double_t computeEntropy(IT begin, IT end, symbol_t min) +{ + double_t numSamples = std::accumulate(begin, end, 0); + return std::accumulate(begin, end, 0, [numSamples](double_t entropy, count_t frequency) { + const double_t p = static_cast(frequency) / static_cast(numSamples); + const double_t length = p == 0 ? 0 : std::log2(p); + return entropy -= p * length; + }); +}; + +template , bool> = true> +inline FrequencyTable makeFrequencyTableFromSamples(Source_IT begin, Source_IT end, bool extendTable = true) +{ + FrequencyTable frequencyTable{}; + frequencyTable.addSamples(begin, end, extendTable); + return frequencyTable; +} + +template , bool> = true> +inline FrequencyTable makeFrequencyTableFromSamples(Source_IT begin, Source_IT end, symbol_t min, symbol_t max, bool extendTable = true) +{ + FrequencyTable frequencyTable{min, max}; + frequencyTable.addSamples(begin, end, extendTable); + return frequencyTable; +} + +} // namespace rans +} // namespace o2 + +#endif /* INCLUDE_RANS_FREQUENCYTABLE_H_ */ diff --git a/Utilities/rANS/include/rANS/LiteralDecoder.h b/Utilities/rANS/include/rANS/LiteralDecoder.h new file mode 100644 index 0000000000000..bcd1585191a16 --- /dev/null +++ b/Utilities/rANS/include/rANS/LiteralDecoder.h @@ -0,0 +1,125 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file Decoder.h +/// @author Michael Lettrich +/// @since 2020-04-06 +/// @brief Decoder - decode a rANS encoded state back into source symbols + +#ifndef RANS_LITERALDECODER_H +#define RANS_LITERALDECODER_H + +#include "Decoder.h" + +#include +#include +#include +#include +#include + +#include + +#include "rANS/internal/DecoderSymbol.h" +#include "rANS/internal/ReverseSymbolLookupTable.h" +#include "rANS/internal/SymbolTable.h" +#include "rANS/internal/Decoder.h" +#include "rANS/internal/DecoderBase.h" + +namespace o2 +{ +namespace rans +{ + +template +class LiteralDecoder : public internal::DecoderBase +{ + public: + using internal::DecoderBase::DecoderBase; + + template , bool> = true> + void process(stream_IT inputEnd, source_IT outputBegin, size_t messageLength, std::vector& literals) const; + + private: + using ransDecoder_t = typename internal::DecoderBase::ransDecoder_t; +}; + +template +template , bool>> +void LiteralDecoder::process(stream_IT inputEnd, source_IT outputBegin, size_t messageLength, std::vector& literals) const +{ + using namespace internal; + LOG(trace) << "start decoding"; + RANSTimer t; + +#ifdef O2_RANS_PRINT_PROCESSED_DATA + JSONArrayLogger arrayLogger{}; +#endif + + t.start(); + + if (messageLength == 0) { + LOG(warning) << "Empty message passed to decoder, skipping decode process"; + return; + } + + stream_IT inputIter = inputEnd; + source_IT it = outputBegin; + + auto decode = [&, this](ransDecoder_t& decoder) { + const auto cumul = decoder.get(); + const auto streamSymbol = (this->mReverseLUT)[cumul]; + source_T symbol = streamSymbol; + if (this->mSymbolTable.isEscapeSymbol(streamSymbol)) { + symbol = literals.back(); + literals.pop_back(); + } + +#ifdef O2_RANS_PRINT_PROCESSED_DATA + arrayLogger << symbol; +#endif + + return std::make_tuple(symbol, decoder.advanceSymbol(inputIter, (this->mSymbolTable)[streamSymbol])); + }; + + // make Iter point to the last last element + --inputIter; + + ransDecoder_t rans0{this->mSymbolTable.getPrecision()}; + ransDecoder_t rans1{this->mSymbolTable.getPrecision()}; + inputIter = rans0.init(inputIter); + inputIter = rans1.init(inputIter); + + for (size_t i = 0; i < (messageLength & ~1); i += 2) { + std::tie(*it++, inputIter) = decode(rans0); + std::tie(*it++, inputIter) = decode(rans1); + } + + // last byte, if message length was odd + if (messageLength & 1) { + std::tie(*it++, inputIter) = decode(rans0); + } + t.stop(); + +#ifdef O2_RANS_PRINT_PROCESSED_DATA + LOG(info) << "decoderOutput:" << arrayLogger; +#endif + + LOG(debug1) << "Decoder::" << __func__ << " { DecodedSymbols: " << messageLength << "," + << "processedBytes: " << messageLength * sizeof(source_T) << "," + << " inclusiveTimeMS: " << t.getDurationMS() << "," + << " BandwidthMiBPS: " << std::fixed << std::setprecision(2) << (messageLength * sizeof(source_T) * 1.0) / (t.getDurationS() * 1.0 * (1 << 20)) << "}"; + + LOG(trace) << "done decoding"; +} +} // namespace rans +} // namespace o2 + +#endif /* RANS_LITERALDECODER_H */ diff --git a/Utilities/rANS/include/rANS/LiteralEncoder.h b/Utilities/rANS/include/rANS/LiteralEncoder.h new file mode 100644 index 0000000000000..3d3e2b2542c9f --- /dev/null +++ b/Utilities/rANS/include/rANS/LiteralEncoder.h @@ -0,0 +1,138 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file Encoder.h +/// @author Michael Lettrich +/// @since 2020-04-06 +/// @brief Encoder - code symbol into a rANS encoded state + +#ifndef RANS_LITERAL_ENCODER_H +#define RANS_LITERAL_ENCODER_H + +#include +#include +#include + +#include +#include + +#include "rANS/internal/EncoderBase.h" +#include "rANS/internal/EncoderSymbol.h" +#include "rANS/internal/helper.h" +#include "rANS/internal/SymbolTable.h" + +namespace o2 +{ +namespace rans +{ + +template +class LiteralEncoder : public internal::EncoderBase +{ + + public: + // inherit constructors; + using internal::EncoderBase::EncoderBase; + + template , bool> = true> + stream_IT process(source_IT inputBegin, source_IT inputEnd, stream_IT outputBegin, std::vector& literals) const; + + private: + using ransCoder_t = typename internal::EncoderBase::ransCoder_t; +}; + +template +template , bool>> +stream_IT LiteralEncoder::process(source_IT inputBegin, source_IT inputEnd, stream_IT outputBegin, std::vector& literals) const +{ + using namespace internal; + LOG(trace) << "start encoding"; + RANSTimer t; + +#ifdef O2_RANS_PRINT_PROCESSED_DATA + JSONArrayLogger arrayLogger{true}; +#endif + + t.start(); + + if (inputBegin == inputEnd) { + LOG(warning) << "passed empty message to encoder, skip encoding"; + return outputBegin; + } + + ransCoder_t rans0{this->mSymbolTable.getPrecision()}; + ransCoder_t rans1{this->mSymbolTable.getPrecision()}; + + stream_IT outputIter = outputBegin; + source_IT inputIT = inputEnd; + + const auto inputBufferSize = std::distance(inputBegin, inputEnd); + + auto encode = [&, this](source_IT symbolIter, stream_IT outputIter, ransCoder_t& coder) { + const source_T symbol = *symbolIter; + const auto& encoderSymbol = (this->mSymbolTable)[symbol]; + if (this->mSymbolTable.isEscapeSymbol(symbol)) { + literals.push_back(symbol); + } + +#ifdef O2_RANS_PRINT_PROCESSED_DATA + arrayLogger << symbol; +#endif + + return coder.putSymbol(outputIter, encoderSymbol); + }; + + // odd number of bytes? + if (inputBufferSize & 1) { + outputIter = encode(--inputIT, outputIter, rans0); + } + + while (inputIT != inputBegin) { // NB: working in reverse! + outputIter = encode(--inputIT, outputIter, rans1); + outputIter = encode(--inputIT, outputIter, rans0); + } + outputIter = rans1.flush(outputIter); + outputIter = rans0.flush(outputIter); + // first iterator past the range so that sizes, distances and iterators work correctly. + ++outputIter; + + t.stop(); + +#ifdef O2_RANS_PRINT_PROCESSED_DATA + LOG(info) << "encoderInput:" << arrayLogger; +#endif + + LOG(debug1) << "Encoder::" << __func__ << " {ProcessedBytes: " << inputBufferSize * sizeof(source_T) << "," + << " inclusiveTimeMS: " << t.getDurationMS() << "," + << " BandwidthMiBPS: " << std::fixed << std::setprecision(2) << (inputBufferSize * sizeof(source_T) * 1.0) / (t.getDurationS() * 1.0 * (1 << 20)) << "}"; + +// advanced diagnostics for debug builds +#if !defined(NDEBUG) + + const auto inputBufferSizeB = inputBufferSize * sizeof(source_T); + + LOG(debug2) << "EncoderProperties: {" + << "sourceTypeB: " << sizeof(source_T) << ", " + << "streamTypeB: " << sizeof(stream_T) << ", " + << "coderTypeB: " << sizeof(coder_T) << ", " + << "symbolTablePrecision: " << this->mSymbolTable.getPrecision() << ", " + << "inputBufferSizeB: " << inputBufferSizeB << "}"; +#endif + + LOG(trace) << "done encoding"; + + return outputIter; +}; + +} // namespace rans +} // namespace o2 + +#endif /* RANS_LITERAL_ENCODER_H */ diff --git a/Utilities/rANS/include/rANS/RenormedFrequencyTable.h b/Utilities/rANS/include/rANS/RenormedFrequencyTable.h new file mode 100644 index 0000000000000..dff3cde5f42ff --- /dev/null +++ b/Utilities/rANS/include/rANS/RenormedFrequencyTable.h @@ -0,0 +1,94 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file RenormedFrequencyTable.h +/// @author Michael Lettrich +/// @since 2019-05-08 +/// @brief Histogram to depict frequencies of source symbols for rANS compression. + +#ifndef INCLUDE_RANS_RENORMEDFREQUENCYTABLE_H_ +#define INCLUDE_RANS_RENORMEDFREQUENCYTABLE_H_ + +#include + +#include "rANS/FrequencyTable.h" + +namespace o2 +{ +namespace rans +{ + +class RenormedFrequencyTable +{ + public: + using iterator_t = typename FrequencyTable::iterator_t; + using constIterator_t = typename FrequencyTable::constIterator_t; + + // TODO(milettri): fix once ROOT cling respects the standard http://wg21.link/p1286r2 + RenormedFrequencyTable() noexcept {}; // NOLINT + + inline RenormedFrequencyTable(FrequencyTable frequencyTable, size_t renormingBits) : mRenormingBits(renormingBits), mFrequencyTable{std::move(frequencyTable)} + { + if (mFrequencyTable.getNumSamples() != internal::pow2(this->getRenormingBits())) { + throw std::runtime_error{fmt::format("FrequencyTable needs to be renormed to {} Bits.", this->getRenormingBits())}; + } + if (!mFrequencyTable.hasIncompressibleSymbols()) { + throw std::runtime_error{fmt::format("FrequencyTable needs to have an incompressible symbol.")}; + } + }; + + inline size_t getRenormingBits() const noexcept { return mRenormingBits; }; + + inline bool isRenormedTo(size_t nBits) const noexcept { return nBits == this->getRenormingBits(); }; + + inline const count_t* data() const noexcept { return mFrequencyTable.data(); }; + + inline constIterator_t begin() const noexcept { return mFrequencyTable.begin(); }; + + inline constIterator_t end() const noexcept { return mFrequencyTable.end(); }; + + inline constIterator_t cbegin() const noexcept { return this->begin(); }; + + inline constIterator_t cend() const noexcept { return this->end(); }; + + inline count_t operator[](size_t index) const { return mFrequencyTable[index]; }; + + inline count_t at(size_t index) const { return mFrequencyTable.at(index); }; + + inline symbol_t getMinSymbol() const noexcept { return mFrequencyTable.getMinSymbol(); }; + + inline symbol_t getMaxSymbol() const noexcept { return mFrequencyTable.getMaxSymbol(); }; + + inline size_t size() const noexcept { return mFrequencyTable.size(); }; + + inline bool empty() const noexcept { return mFrequencyTable.empty(); }; + + inline size_t getNUsedAlphabetSymbols() const { return mFrequencyTable.getNUsedAlphabetSymbols(); }; + + inline size_t getAlphabetRangeBits() const noexcept { return mFrequencyTable.getAlphabetRangeBits(); }; + + inline count_t getIncompressibleSymbolFrequency() const noexcept { return mFrequencyTable.getIncompressibleSymbolFrequency(); }; + + inline size_t getNumSamples() const noexcept { return mFrequencyTable.getNumSamples(); }; + + private: + size_t mRenormingBits{}; + FrequencyTable mFrequencyTable{}; +}; + +RenormedFrequencyTable renorm(FrequencyTable oldTable, size_t newPrecision = 0); + +RenormedFrequencyTable renormCutoffIncompressible(FrequencyTable oldTable, uint8_t newPrecision = 0, uint8_t lowProbabilityCutoffBits = 3); + +} // namespace rans +} // namespace o2 + +#endif /* INCLUDE_RANS_RENORMEDFREQUENCYTABLE_H_ */ diff --git a/Utilities/rANS/include/rANS/compat.h b/Utilities/rANS/include/rANS/compat.h deleted file mode 100644 index f94be92bed472..0000000000000 --- a/Utilities/rANS/include/rANS/compat.h +++ /dev/null @@ -1,295 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file compat.h -/// @author Michael Lettrich -/// @brief functionality to maintain compatibility with previous version of this library - -#ifndef RANS_COMPAT_H_ -#define RANS_COMPAT_H_ - -#include - -#include "rANS/internal/common/typetraits.h" - -#include "rANS/internal/containers/DenseHistogram.h" -#include "rANS/internal/containers/RenormedHistogram.h" -#include "rANS/internal/containers/DenseSymbolTable.h" -#include "rANS/internal/containers/Symbol.h" -#include "rANS/internal/containers/LowRangeDecoderTable.h" - -#include "rANS/internal/encode/Encoder.h" -#include "rANS/internal/encode/SingleStreamEncoderImpl.h" - -#include "rANS/internal/decode/Decoder.h" -#include "rANS/internal/decode/DecoderImpl.h" - -#include "rANS/factory.h" - -namespace o2::rans::compat -{ - -namespace defaults -{ -namespace internal -{ -inline constexpr size_t RenormingLowerBound = 31; -} // namespace internal - -struct CoderPreset { - inline static constexpr size_t nStreams = 2; - inline static constexpr size_t renormingLowerBound = internal::RenormingLowerBound; -}; - -} // namespace defaults - -namespace compatImpl -{ -inline constexpr uint32_t MinRenormThreshold = 10; -inline constexpr uint32_t MaxRenormThreshold = 20; -} // namespace compatImpl - -inline size_t computeRenormingPrecision(size_t nUsedAlphabetSymbols) -{ - const uint32_t minBits = o2::rans::utils::log2UInt(nUsedAlphabetSymbols); - const uint32_t estimate = minBits * 3u / 2u; - const uint32_t maxThreshold = std::max(minBits, compatImpl::MaxRenormThreshold); - const uint32_t minThreshold = std::max(estimate, compatImpl::MinRenormThreshold); - - return std::min(minThreshold, maxThreshold); -}; - -template -RenormedDenseHistogram renorm(DenseHistogram histogram, size_t newPrecision = 0) -{ - using namespace o2::rans::internal; - - constexpr size_t IncompressibleSymbolFrequency = 1; - - if (histogram.empty()) { - LOG(warning) << "rescaling empty histogram"; - } - - size_t nUsedAlphabetSymbols = countNUsedAlphabetSymbols(histogram); - - if (newPrecision == 0) { - newPrecision = computeRenormingPrecision(nUsedAlphabetSymbols); - } - - const size_t alphabetSize = histogram.size() + 1; // +1 for incompressible symbol - std::vector cumulativeFrequencies(alphabetSize + 1); // +1 to store total cumulative frequencies - cumulativeFrequencies[0] = 0; - std::inclusive_scan(histogram.begin(), histogram.end(), ++cumulativeFrequencies.begin(), std::plus<>(), 0ull); - cumulativeFrequencies.back() = histogram.getNumSamples() + IncompressibleSymbolFrequency; - - auto getFrequency = [&cumulativeFrequencies](count_t i) { - assert(cumulativeFrequencies[i + 1] >= cumulativeFrequencies[i]); - return cumulativeFrequencies[i + 1] - cumulativeFrequencies[i]; - }; - - // we will memorize only those entries which can be used - const auto sortIdx = [&]() { - std::vector indices; - indices.reserve(nUsedAlphabetSymbols + 1); - - for (size_t i = 0; i < alphabetSize; ++i) { - if (getFrequency(i) != 0) { - indices.push_back(i); - } - } - - std::sort(indices.begin(), indices.end(), [&](count_t i, count_t j) { return getFrequency(i) < getFrequency(j); }); - return indices; - }(); - - // resample distribution based on cumulative frequencies - const count_t newCumulatedFrequency = utils::pow2(newPrecision); - size_t nSamples = histogram.getNumSamples() + IncompressibleSymbolFrequency; - assert(newCumulatedFrequency >= nUsedAlphabetSymbols); - size_t needsShift = 0; - for (size_t i = 0; i < sortIdx.size(); i++) { - if (static_cast(getFrequency(sortIdx[i])) * (newCumulatedFrequency - needsShift) / nSamples >= 1) { - break; - } - needsShift++; - } - - size_t shift = 0; - auto beforeUpdate = cumulativeFrequencies[0]; - for (size_t i = 0; i < alphabetSize; i++) { - auto& nextCumulative = cumulativeFrequencies[i + 1]; - uint64_t oldFrequeny = nextCumulative - beforeUpdate; - if (oldFrequeny && oldFrequeny * (newCumulatedFrequency - needsShift) / nSamples < 1) { - shift++; - } - beforeUpdate = cumulativeFrequencies[i + 1]; - nextCumulative = (static_cast(newCumulatedFrequency - needsShift) * nextCumulative) / nSamples + shift; - } - assert(shift == needsShift); - - // verify -#if !defined(NDEBUG) - assert(cumulativeFrequencies.front() == 0); - assert(cumulativeFrequencies.back() == newCumulatedFrequency); - size_t i = 0; - for (auto frequency : histogram) { - if (frequency == 0) { - assert(cumulativeFrequencies[i + 1] == cumulativeFrequencies[i]); - } else { - assert(cumulativeFrequencies[i + 1] > cumulativeFrequencies[i]); - } - ++i; - } -#endif - - typename RenormedDenseHistogram::container_type rescaledFrequencies(histogram.size(), histogram.getOffset()); - - assert(cumulativeFrequencies.size() == histogram.size() + 2); - // calculate updated frequencies - for (size_t i = 0; i < histogram.size(); ++i) { - rescaledFrequencies(i) = getFrequency(i); - } - const typename RenormedDenseHistogram::value_type incompressibleSymbolFrequency = getFrequency(histogram.size()); - - return RenormedDenseHistogram{std::move(rescaledFrequencies), newPrecision, incompressibleSymbolFrequency}; -}; - -class makeEncoder -{ - - public: - template - [[nodiscard]] inline static constexpr decltype(auto) fromRenormed(const RenormedHistogramConcept& renormed) - { - using namespace o2::rans::internal; - using source_type = typename RenormedHistogramConcept::source_type; - using symbol_type = internal::PrecomputedSymbol; - using coder_command = SingleStreamEncoderImpl; - using symbolTable_type = DenseSymbolTable; - using encoderType = Encoder; - - return encoderType{renormed}; - }; - - template - [[nodiscard]] inline static decltype(auto) fromHistogram(DenseHistogram histogram, size_t renormingPrecision = 0) - { - const auto renormedHistogram = o2::rans::compat::renorm(std::move(histogram), renormingPrecision); - return makeEncoder::fromRenormed(renormedHistogram); - }; - - template - [[nodiscard]] inline static decltype(auto) fromSamples(source_IT begin, source_IT end, size_t renormingPrecision = 0) - { - auto histogram = makeDenseHistogram::fromSamples(begin, end); - - return makeEncoder::fromHistogram(std::move(histogram), renormingPrecision); - }; - - template - [[nodiscard]] inline static decltype(auto) fromSamples(gsl::span range, size_t renormingPrecision = 0) - { - auto histogram = makeDenseHistogram::template fromSamples(range); - return makeEncoder::fromHistogram(std::move(histogram), renormingPrecision); - }; - - private: - static constexpr CoderTag mCoderTag = CoderTag::SingleStream; - static constexpr size_t mNstreams = defaults::CoderPreset::nStreams; - static constexpr size_t mRenormingLowerBound = defaults::CoderPreset::renormingLowerBound; -}; - -class makeDecoder -{ - - using this_type = makeDecoder; - - public: - template - [[nodiscard]] inline static constexpr decltype(auto) fromRenormed(const RenormedHistogramConcept& renormed) - { - using namespace internal; - - using source_type = typename RenormedHistogramConcept::source_type; - using coder_type = DecoderImpl; - using decoder_type = Decoder; - - return decoder_type{renormed}; - }; - - template - [[nodiscard]] inline static decltype(auto) fromHistogram(DenseHistogram histogram, size_t renormingPrecision = 0) - { - const auto renormedHistogram = o2::rans::compat::renorm(std::move(histogram), renormingPrecision); - return this_type::fromRenormed(renormedHistogram); - }; - - template - [[nodiscard]] inline static decltype(auto) fromSamples(source_IT begin, source_IT end, size_t renormingPrecision = 0) - { - auto histogram = makeDenseHistogram::fromSamples(begin, end); - return this_type::fromHistogram(std::move(histogram), renormingPrecision); - }; - - template - [[nodiscard]] inline static decltype(auto) fromSamples(gsl::span range, size_t renormingPrecision = 0) - { - auto histogram = makeDenseHistogram::fromSamples(range); - return this_type::fromHistogram(std::move(histogram), renormingPrecision); - }; - - private: - static constexpr CoderTag mCoderTag = CoderTag::SingleStream; - static constexpr size_t mNstreams = defaults::CoderPreset::nStreams; - static constexpr size_t mRenormingLowerBound = defaults::CoderPreset::renormingLowerBound; -}; - -template -inline size_t getAlphabetRangeBits(const DenseHistogram& histogram) noexcept -{ - using namespace o2::rans::internal; - const auto view = trim(makeHistogramView(histogram)); - return internal::numBitsForNSymbols(view.size()); -}; - -template -inline size_t getAlphabetRangeBits(const RenormedDenseHistogram& histogram) noexcept -{ - using namespace o2::rans::internal; - const auto view = trim(makeHistogramView(histogram)); - return internal::numBitsForNSymbols(view.size() + histogram.hasIncompressibleSymbol()); -}; - -template -inline size_t getAlphabetRangeBits(const DenseSymbolTable& symbolTable) noexcept -{ - const bool hasIncompressibleSymbol = symbolTable.getEscapeSymbol().getFrequency() > 0; - return internal::numBitsForNSymbols(symbolTable.size() + hasIncompressibleSymbol); -}; - -inline size_t calculateMaxBufferSizeB(size_t nElements, size_t rangeBits) -{ - constexpr size_t sizeofStreamT = sizeof(uint32_t); - // // RS: w/o safety margin the o2-test-ctf-io produces an overflow in the Encoder::process - // constexpr size_t SaferyMargin = 16; - // return std::ceil(1.20 * (num * rangeBits * 1.0) / (sizeofStreamT * 8.0)) + SaferyMargin; - return nElements * sizeofStreamT; -} - -template -using encoder_type = decltype(makeEncoder::fromRenormed(RenormedDenseHistogram{})); - -template -using decoder_type = decltype(makeDecoder::fromRenormed(RenormedDenseHistogram{})); - -} // namespace o2::rans::compat - -#endif /* RANS_COMPAT_H_ */ \ No newline at end of file diff --git a/Utilities/rANS/include/rANS/decode.h b/Utilities/rANS/include/rANS/decode.h deleted file mode 100644 index f98d746276ce6..0000000000000 --- a/Utilities/rANS/include/rANS/decode.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file decode.h -/// @author michael.lettrich@cern.ch -/// @brief public interface for decoding. - -#ifndef RANS_DECODE_H_ -#define RANS_DECODE_H_ - -#include "rANS/internal/containers/DenseSymbolTable.h" -#include "rANS/internal/containers/Symbol.h" -#include "rANS/internal/decode/Decoder.h" - -#endif /* RANS_DECODE_H_ */ diff --git a/Utilities/rANS/include/rANS/definitions.h b/Utilities/rANS/include/rANS/definitions.h new file mode 100644 index 0000000000000..d650b1b6eecf1 --- /dev/null +++ b/Utilities/rANS/include/rANS/definitions.h @@ -0,0 +1,36 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// @file definitions.h +/// @author Michael Lettrich +/// @since 2021-10-22 +/// @brief Global Definitions used throughout rANS library + +#ifndef INCLUDE_RANS_DEFINITIONS_H_ +#define INCLUDE_RANS_DEFINITIONS_H_ + +#include +#include + +namespace o2 +{ +namespace rans +{ +using symbol_t = int32_t; +using count_t = uint32_t; +using histogram_t = std::vector; + +inline constexpr uint8_t MinRenormThreshold = 10; +inline constexpr uint8_t MaxRenormThreshold = 20; +} // namespace rans +} // namespace o2 + +#endif /* INCLUDE_RANS_DEFINITIONS_H_ */ diff --git a/Utilities/rANS/include/rANS/encode.h b/Utilities/rANS/include/rANS/encode.h deleted file mode 100644 index a62fe1121db90..0000000000000 --- a/Utilities/rANS/include/rANS/encode.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file encode.h -/// @author michael.lettrich@cern.ch -/// @brief public interface for encoding. - -#ifndef RANS_ENCODE_H_ -#define RANS_ENCODE_H_ - -#include "rANS/internal/containers/DenseSymbolTable.h" -#include "rANS/internal/containers/Symbol.h" -#include "rANS/internal/encode/Encoder.h" - -#endif /* RANS_ENCODE_H_ */ diff --git a/Utilities/rANS/include/rANS/factory.h b/Utilities/rANS/include/rANS/factory.h deleted file mode 100644 index 549031ae8ecdb..0000000000000 --- a/Utilities/rANS/include/rANS/factory.h +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2019-2023 CERN and copyright holders of ALICE O2. -// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. -// All rights not expressly granted are reserved. -// -// This software is distributed under the terms of the GNU General Public -// License v3 (GPL Version 3), copied verbatim in the file "COPYING". -// -// In applying this license CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -/// @file factory.h -/// @author michael.lettrich@cern.ch -/// @brief static factory classes for building histograms, encoders and decoders. - -#ifndef RANS_FACTORY_H_ -#define RANS_FACTORY_H_ - -#include "rANS/internal/common/defaults.h" -#include "rANS/internal/common/typetraits.h" -#include "rANS/internal/common/codertraits.h" - -#include "rANS/internal/metrics/Metrics.h" -#include "rANS/internal/transform/renorm.h" - -#include "rANS/internal/containers/DenseHistogram.h" -#include "rANS/internal/containers/DenseSymbolTable.h" - -#include "rANS/internal/containers/AdaptiveHistogram.h" -#include "rANS/internal/containers/AdaptiveSymbolTable.h" - -#include "rANS/internal/containers/SparseSymbolTable.h" -#include "rANS/internal/containers/SparseHistogram.h" - -#include "rANS/internal/containers/RenormedHistogram.h" - -#include "rANS/internal/containers/LowRangeDecoderTable.h" -#include "rANS/internal/containers/HighRangeDecoderTable.h" -#include "rANS/internal/containers/Symbol.h" - -#include "rANS/internal/encode/Encoder.h" -#include "rANS/internal/encode/SingleStreamEncoderImpl.h" -#include "rANS/internal/encode/SIMDEncoderImpl.h" - -#include "rANS/internal/decode/Decoder.h" -#include "rANS/internal/decode/DecoderImpl.h" - -namespace o2::rans -{ - -namespace internal -{ - -template